runtime: mark newly-mapped memory as scavenged

On most platforms newly-mapped memory is untouched, meaning the pages
backing the region haven't been faulted in yet. However, we mark this
memory as unscavenged which means the background scavenger
aggressively "returns" this memory to the OS if the heap is small.

The only platform where newly-mapped memory is actually unscavenged (and
counts toward the application's RSS) is on Windows, since
(*mheap).sysAlloc commits the reservation. Instead of making a special
case for Windows, I change the requirements a bit for a sysReserve'd
region. It must now be both sysMap'd and sysUsed'd, with sysMap being a
no-op on Windows. Comments about memory allocation have been updated to
include a more up-to-date mental model of which states a region of memory
may be in (at a very low level) and how to transition between these
states.

Now this means we can correctly mark newly-mapped heap memory as
scavenged on every platform, reducing the load on the background
scavenger early on in the application for small heaps. As a result,
heap-growth scavenging is no longer necessary, since any actual RSS
growth will be accounted for on the allocation codepath.

Finally, this change also cleans up grow a little bit to avoid
pretending that it's freeing an in-use span and just does the necessary
operations directly.

Fixes #32012.
Fixes #31966.
Updates #26473.

Change-Id: Ie06061eb638162e0560cdeb0b8993d94cfb4d290
Reviewed-on: https://go-review.googlesource.com/c/go/+/177097
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>

1 files changed, 17 insertions, 18 deletions

diff --git a/src/runtime/mem_windows.go b/src/runtime/mem_windows.go
index f752136706..165062ec27 100644
--- a/src/runtime/mem_windows.go
+++ b/src/runtime/mem_windows.go
@@ -60,24 +60,34 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 }
 
 func sysUsed(v unsafe.Pointer, n uintptr) {
-	r := stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_COMMIT, _PAGE_READWRITE)
-	if r != 0 {
+	p := stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_COMMIT, _PAGE_READWRITE)
+	if p == uintptr(v) {
 		return
 	}
 
 	// Commit failed. See SysUnused.
-	for n > 0 {
-		small := n
+	// Hold on to n here so we can give back a better error message
+	// for certain cases.
+	k := n
+	for k > 0 {
+		small := k
 		for small >= 4096 && stdcall4(_VirtualAlloc, uintptr(v), small, _MEM_COMMIT, _PAGE_READWRITE) == 0 {
 			small /= 2
 			small &^= 4096 - 1
 		}
 		if small < 4096 {
-			print("runtime: VirtualAlloc of ", small, " bytes failed with errno=", getlasterror(), "\n")
-			throw("runtime: failed to commit pages")
+			errno := getlasterror()
+			switch errno {
+			case _ERROR_NOT_ENOUGH_MEMORY, _ERROR_COMMITMENT_LIMIT:
+				print("runtime: VirtualAlloc of ", n, " bytes failed with errno=", errno, "\n")
+				throw("out of memory")
+			default:
+				print("runtime: VirtualAlloc of ", small, " bytes failed with errno=", errno, "\n")
+				throw("runtime: failed to commit pages")
+			}
 		}
 		v = add(v, small)
-		n -= small
+		k -= small
 	}
 }
 
@@ -116,15 +126,4 @@ func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer {
 
 func sysMap(v unsafe.Pointer, n uintptr, sysStat *uint64) {
 	mSysStatInc(sysStat, n)
-	p := stdcall4(_VirtualAlloc, uintptr(v), n, _MEM_COMMIT, _PAGE_READWRITE)
-	if p != uintptr(v) {
-		errno := getlasterror()
-		print("runtime: VirtualAlloc of ", n, " bytes failed with errno=", errno, "\n")
-		switch errno {
-		case _ERROR_NOT_ENOUGH_MEMORY, _ERROR_COMMITMENT_LIMIT:
-			throw("out of memory")
-		default:
-			throw("runtime: cannot map pages in arena address space")
-		}
-	}
 }