runtime: tidy GC driver

Change-Id: I0da26e89ae73272e49e82c6549c774e5bc97f64c
Reviewed-on: https://go-review.googlesource.com/5331
Reviewed-by: Austin Clements <austin@google.com>

4 files changed, 96 insertions, 124 deletions

diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go
index b65bf70656..475f97fd05 100644
--- a/src/runtime/malloc.go
+++ b/src/runtime/malloc.go
@@ -655,7 +655,7 @@ func mallocgc(size uintptr, typ *_type, flags uint32) unsafe.Pointer {
 	}
 
 	if shouldtriggergc() {
-		gogc(0)
+		startGC(gcBackgroundMode)
 	} else if shouldhelpgc && atomicloaduint(&bggc.working) == 1 {
 		// bggc.lock not taken since race on bggc.working is benign.
 		// At worse we don't call gchelpwork.
diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go
index 75d6b9158e..079856ed70 100644
--- a/src/runtime/mgc.go
+++ b/src/runtime/mgc.go
@@ -206,9 +206,7 @@ func shouldtriggergc() bool {
 	return triggerratio*(int64(memstats.next_gc)-int64(memstats.heap_alloc)) <= int64(memstats.next_gc) && atomicloaduint(&bggc.working) == 0
 }
 
-var work workdata
-
-type workdata struct {
+var work struct {
 	full    uint64                // lock-free list of full blocks workbuf
 	empty   uint64                // lock-free list of empty blocks workbuf
 	partial uint64                // lock-free list of partially filled blocks workbuf
@@ -226,19 +224,21 @@ type workdata struct {
 
 // GC runs a garbage collection.
 func GC() {
-	gogc(2)
+	startGC(gcForceBlockMode)
 }
 
-// force = 0 - start concurrent GC
-// force = 1 - do STW GC regardless of current heap usage
-// force = 2 - go STW GC and eager sweep
-func gogc(force int32) {
+const (
+	gcBackgroundMode = iota // concurrent GC
+	gcForceMode             // stop-the-world GC now
+	gcForceBlockMode        // stop-the-world GC now and wait for sweep
+)
+
+func startGC(mode int) {
 	// The gc is turned off (via enablegc) until the bootstrap has completed.
 	// Also, malloc gets called in the guts of a number of libraries that might be
 	// holding locks. To avoid deadlocks during stoptheworld, don't bother
 	// trying to run gc while holding a lock. The next mallocgc without a lock
 	// will do the gc instead.
-
 	mp := acquirem()
 	if gp := getg(); gp == mp.g0 || mp.locks > 1 || !memstats.enablegc || panicking != 0 || gcpercent < 0 {
 		releasem(mp)
@@ -247,20 +247,23 @@ func gogc(force int32) {
 	releasem(mp)
 	mp = nil
 
-	if force == 0 {
-		lock(&bggc.lock)
-		if !bggc.started {
-			bggc.working = 1
-			bggc.started = true
-			go backgroundgc()
-		} else if bggc.working == 0 {
-			bggc.working = 1
-			ready(bggc.g)
-		}
-		unlock(&bggc.lock)
-	} else {
-		gcwork(force)
+	if mode != gcBackgroundMode {
+		// special synchronous cases
+		gc(mode)
+		return
 	}
+
+	// trigger concurrent GC
+	lock(&bggc.lock)
+	if !bggc.started {
+		bggc.working = 1
+		bggc.started = true
+		go backgroundgc()
+	} else if bggc.working == 0 {
+		bggc.working = 1
+		ready(bggc.g)
+	}
+	unlock(&bggc.lock)
 }
 
 // State of the background concurrent GC goroutine.
@@ -276,15 +279,15 @@ var bggc struct {
 func backgroundgc() {
 	bggc.g = getg()
 	for {
-		gcwork(0)
+		gc(gcBackgroundMode)
 		lock(&bggc.lock)
 		bggc.working = 0
 		goparkunlock(&bggc.lock, "Concurrent GC wait", traceEvGoBlock)
 	}
 }
 
-func gcwork(force int32) {
-
+func gc(mode int) {
+	// Ok, we're doing it!  Stop everybody else
 	semacquire(&worldsema, false)
 
 	// Pick up the remaining unswept/not being swept spans concurrently
@@ -292,13 +295,11 @@ func gcwork(force int32) {
 		sweep.nbgsweep++
 	}
 
-	// Ok, we're doing it!  Stop everybody else
-
 	mp := acquirem()
 	mp.preemptoff = "gcing"
 	releasem(mp)
 	gctimer.count++
-	if force == 0 {
+	if mode == gcBackgroundMode {
 		gctimer.cycle.sweepterm = nanotime()
 	}
 
@@ -307,31 +308,40 @@ func gcwork(force int32) {
 		traceGCStart()
 	}
 
-	// Pick up the remaining unswept/not being swept spans before we STW
-	for gosweepone() != ^uintptr(0) {
-		sweep.nbgsweep++
-	}
 	systemstack(stoptheworld)
 	systemstack(finishsweep_m) // finish sweep before we start concurrent scan.
-	if force == 0 {            // Do as much work concurrently as possible
-		gcphase = _GCscan
-		systemstack(starttheworld)
-		gctimer.cycle.scan = nanotime()
-		// Do a concurrent heap scan before we stop the world.
-		systemstack(gcscan_m)
-		gctimer.cycle.installmarkwb = nanotime()
-		systemstack(stoptheworld)
-		systemstack(gcinstallmarkwb)
-		systemstack(harvestwbufs)
-		systemstack(starttheworld)
-		gctimer.cycle.mark = nanotime()
-		systemstack(gcmark_m)
-		gctimer.cycle.markterm = nanotime()
-		systemstack(stoptheworld)
-		systemstack(gcinstalloffwb_m)
+
+	if mode == gcBackgroundMode { // Do as much work concurrently as possible
+		systemstack(func() {
+			gcphase = _GCscan
+
+			// Concurrent scan.
+			starttheworld()
+			gctimer.cycle.scan = nanotime()
+			gcscan_m()
+			gctimer.cycle.installmarkwb = nanotime()
+
+			// Sync.
+			stoptheworld()
+			gcphase = _GCmark
+			harvestwbufs()
+
+			// Concurrent mark.
+			starttheworld()
+			gctimer.cycle.mark = nanotime()
+			var gcw gcWork
+			gcDrain(&gcw)
+			gcw.dispose()
+
+			// Begin mark termination.
+			gctimer.cycle.markterm = nanotime()
+			stoptheworld()
+			gcphase = _GCoff
+		})
 	} else {
-		// For non-concurrent GC (force != 0) g stack have not been scanned so
-		// set gcscanvalid such that mark termination scans all stacks.
+		// For non-concurrent GC (mode != gcBackgroundMode)
+		// g stack have not been scanned so set gcscanvalid
+		// such that mark termination scans all stacks.
 		// No races here since we are in a STW phase.
 		for _, gp := range allgs {
 			gp.gcworkdone = false  // set to true in gcphasework
@@ -341,9 +351,10 @@ func gcwork(force int32) {
 
 	startTime := nanotime()
 	if mp != acquirem() {
-		throw("gogc: rescheduled")
+		throw("gcwork: rescheduled")
 	}
 
+	// TODO(rsc): Should the concurrent GC clear pools earlier?
 	clearpools()
 
 	// Run gc on the g0 stack.  We do this so that the g stack
@@ -355,7 +366,6 @@ func gcwork(force int32) {
 	if debug.gctrace > 1 {
 		n = 2
 	}
-	eagersweep := force >= 2
 	for i := 0; i < n; i++ {
 		if i > 0 {
 			// refresh start time if doing a second GC
@@ -363,12 +373,28 @@ func gcwork(force int32) {
 		}
 		// switch to g0, call gc, then switch back
 		systemstack(func() {
-			gc_m(startTime, eagersweep)
+			gc_m(startTime, mode == gcForceBlockMode)
 		})
 	}
 
 	systemstack(func() {
-		gccheckmark_m(startTime, eagersweep)
+		// Called from malloc.go using systemstack.
+		// The world is stopped. Rerun the scan and mark phases
+		// using the bitMarkedCheck bit instead of the
+		// bitMarked bit. If the marking encounters an
+		// bitMarked bit that is not set then we throw.
+		//go:nowritebarrier
+		if debug.gccheckmark == 0 {
+			return
+		}
+
+		if checkmarkphase {
+			throw("gccheckmark_m, entered with checkmarkphase already true")
+		}
+
+		checkmarkphase = true
+		initCheckmarks()
+		gc_m(startTime, mode == gcForceBlockMode) // turns off checkmarkphase + calls clearcheckmarkbits
 	})
 
 	if trace.enabled {
@@ -379,13 +405,13 @@ func gcwork(force int32) {
 	// all done
 	mp.preemptoff = ""
 
-	if force == 0 {
+	if mode == gcBackgroundMode {
 		gctimer.cycle.sweep = nanotime()
 	}
 
 	semrelease(&worldsema)
 
-	if force == 0 {
+	if mode == gcBackgroundMode {
 		if gctimer.verbose > 1 {
 			GCprinttimes()
 		} else if gctimer.verbose > 0 {
@@ -405,76 +431,23 @@ func gcwork(force int32) {
 	}
 }
 
-// For now this must be bracketed with a stoptheworld and a starttheworld to ensure
-// all go routines see the new barrier.
-//go:nowritebarrier
-func gcinstalloffwb_m() {
-	gcphase = _GCoff
-}
-
-// For now this must be bracketed with a stoptheworld and a starttheworld to ensure
-// all go routines see the new barrier.
-//go:nowritebarrier
-func gcinstallmarkwb() {
-	gcphase = _GCmark
-}
-
-// Mark all objects that are known about.
-// This is the concurrent mark phase.
-//go:nowritebarrier
-func gcmark_m() {
-	var gcw gcWork
-	gcDrain(&gcw)
-	gcw.dispose()
-	// TODO add another harvestwbuf and reset work.nwait=0, work.ndone=0, and work.nproc=1
-	// and repeat the above gcDrain.
-}
-
-// Called from malloc.go using systemstack.
-// The world is stopped. Rerun the scan and mark phases
-// using the bitMarkedCheck bit instead of the
-// bitMarked bit. If the marking encounters an
-// bitMarked bit that is not set then we throw.
-//go:nowritebarrier
-func gccheckmark_m(startTime int64, eagersweep bool) {
-	if debug.gccheckmark == 0 {
-		return
-	}
-
-	if checkmarkphase {
-		throw("gccheckmark_m, entered with checkmarkphase already true")
+// STW is in effect at this point.
+//TODO go:nowritebarrier
+func gc_m(start_time int64, eagersweep bool) {
+	if _DebugGCPtrs {
+		print("GC start\n")
 	}
 
-	checkmarkphase = true
-	initCheckmarks()
-	gc_m(startTime, eagersweep) // turns off checkmarkphase + calls clearcheckmarkbits
-}
-
-// Called from malloc.go using systemstack, stopping and starting the world handled in caller.
-//go:nowritebarrier
-func gc_m(start_time int64, eagersweep bool) {
 	_g_ := getg()
 	gp := _g_.m.curg
 	casgstatus(gp, _Grunning, _Gwaiting)
 	gp.waitreason = "garbage collection"
 
-	gc(start_time, eagersweep)
-	casgstatus(gp, _Gwaiting, _Grunning)
-}
-
-// STW is in effect at this point.
-//TODO go:nowritebarrier
-func gc(start_time int64, eagersweep bool) {
-	if _DebugGCPtrs {
-		print("GC start\n")
-	}
-
 	gcphase = _GCmarktermination
 	if debug.allocfreetrace > 0 {
 		tracegc()
 	}
 
-	_g_ := getg()
 	_g_.m.traceback = 2
 	t0 := start_time
 	work.tstart = start_time
@@ -619,6 +592,7 @@ func gc(start_time int64, eagersweep bool) {
 	if debug.gccheckmark > 0 {
 		if !checkmarkphase {
 			// first half of two-pass; don't set up sweep
+			casgstatus(gp, _Gwaiting, _Grunning)
 			return
 		}
 		checkmarkphase = false // done checking marks
@@ -666,16 +640,12 @@ func gc(start_time int64, eagersweep bool) {
 	if _DebugGCPtrs {
 		print("GC end\n")
 	}
+
+	casgstatus(gp, _Gwaiting, _Grunning)
 }
 
 // Hooks for other packages
 
-//go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory
-func runtime_debug_freeOSMemory() {
-	gogc(2) // force GC and do eager sweep
-	systemstack(scavenge_m)
-}
-
 var poolcleanup func()
 
 //go:linkname sync_runtime_registerPoolCleanup sync.runtime_registerPoolCleanup
diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go
index d082f8e622..ba800aacef 100644
--- a/src/runtime/mheap.go
+++ b/src/runtime/mheap.go
@@ -747,8 +747,10 @@ func mHeap_Scavenge(k int32, now, limit uint64) {
 	}
 }
 
-func scavenge_m() {
-	mHeap_Scavenge(-1, ^uint64(0), 0)
+//go:linkname runtime_debug_freeOSMemory runtime/debug.freeOSMemory
+func runtime_debug_freeOSMemory() {
+	startGC(gcForceBlockMode)
+	systemstack(func() { mHeap_Scavenge(-1, ^uint64(0), 0) })
 }
 
 // Initialize a new span with the given start and npages.
diff --git a/src/runtime/proc.go b/src/runtime/proc.go
index 027416a9ec..d251c314d4 100644
--- a/src/runtime/proc.go
+++ b/src/runtime/proc.go
@@ -123,7 +123,7 @@ func forcegchelper() {
 		if debug.gctrace > 0 {
 			println("GC forced")
 		}
-		gogc(1)
+		startGC(gcForceMode)
 	}
 }