runtime/metrics: add additional allocation metrics

This change adds four additional metrics to the runtime/metrics package
to fill in a few gaps with runtime.MemStats that were overlooked. The
biggest one is TotalAlloc, which is impossible to find with the
runtime/metrics package, but also add a few others for convenience and
clarity. For instance, the total number of objects allocated and freed
are technically available via allocs-by-size and frees-by-size, but it's
onerous to get them (one needs to sum the sample counts in the
histograms).

The four additional metrics are:
- /gc/heap/allocs:bytes   -- total bytes allocated (TotalAlloc)
- /gc/heap/allocs:objects -- total objects allocated (Mallocs - [tiny])
- /gc/heap/frees:bytes    -- total bytes frees (TotalAlloc-HeapAlloc)
- /gc/heap/frees:objects  -- total objects freed (Frees - [tiny])

This change also updates the descriptions of allocs-by-size and
frees-by-size to be more precise.

Change-Id: Iec8c1797a584491e3484b198f2e7f325b68954a7
Reviewed-on: https://go-review.googlesource.com/c/go/+/312431
Reviewed-by: Michael Pratt <mpratt@google.com>
Trust: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Go Bot <gobot@golang.org>

1 files changed, 47 insertions, 5 deletions

diff --git a/src/runtime/metrics_test.go b/src/runtime/metrics_test.go
index e405d807e4..5d32ef469c 100644
--- a/src/runtime/metrics_test.go
+++ b/src/runtime/metrics_test.go
@@ -42,6 +42,7 @@ func TestReadMetrics(t *testing.T) {
 	// Check to make sure the values we read line up with other values we read.
 	var allocsBySize *metrics.Float64Histogram
 	var tinyAllocs uint64
+	var mallocs, frees uint64
 	for i := range samples {
 		switch name := samples[i].Name; name {
 		case "/memory/classes/heap/free:bytes":
@@ -87,6 +88,8 @@ func TestReadMetrics(t *testing.T) {
 				}
 			}
 			allocsBySize = hist
+		case "/gc/heap/allocs:bytes":
+			checkUint64(t, name, samples[i].Value.Uint64(), mstats.TotalAlloc)
 		case "/gc/heap/frees-by-size:bytes":
 			hist := samples[i].Value.Float64Histogram()
 			// Skip size class 0 in BySize, because it's always empty and not represented
@@ -101,6 +104,8 @@ func TestReadMetrics(t *testing.T) {
 					t.Errorf("histogram counts do not match BySize for class %d: got %d, want %d", i, c, f)
 				}
 			}
+		case "/gc/heap/frees:bytes":
+			checkUint64(t, name, samples[i].Value.Uint64(), mstats.TotalAlloc-mstats.HeapAlloc)
 		case "/gc/heap/tiny/allocs:objects":
 			// Currently, MemStats adds tiny alloc count to both Mallocs AND Frees.
 			// The reason for this is because MemStats couldn't be extended at the time
@@ -112,6 +117,13 @@ func TestReadMetrics(t *testing.T) {
 			// Check tiny allocation count outside of this loop, by using the allocs-by-size
 			// histogram in order to figure out how many large objects there are.
 			tinyAllocs = samples[i].Value.Uint64()
+			// Because the next two metrics tests are checking against Mallocs and Frees,
+			// we can't check them directly for the same reason: we need to account for tiny
+			// allocations included in Mallocs and Frees.
+		case "/gc/heap/allocs:objects":
+			mallocs = samples[i].Value.Uint64()
+		case "/gc/heap/frees:objects":
+			frees = samples[i].Value.Uint64()
 		case "/gc/heap/objects:objects":
 			checkUint64(t, name, samples[i].Value.Uint64(), mstats.HeapObjects)
 		case "/gc/heap/goal:bytes":
@@ -131,6 +143,10 @@ func TestReadMetrics(t *testing.T) {
 		nonTinyAllocs += c
 	}
 	checkUint64(t, "/gc/heap/tiny/allocs:objects", tinyAllocs, mstats.Mallocs-nonTinyAllocs)
+
+	// Check allocation and free counts.
+	checkUint64(t, "/gc/heap/allocs:objects", mallocs, mstats.Mallocs-tinyAllocs)
+	checkUint64(t, "/gc/heap/frees:objects", frees, mstats.Frees-tinyAllocs)
 }
 
 func TestReadMetricsConsistency(t *testing.T) {
@@ -153,8 +169,10 @@ func TestReadMetricsConsistency(t *testing.T) {
 		got, want uint64
 	}
 	var objects struct {
-		alloc, free *metrics.Float64Histogram
-		total       uint64
+		alloc, free             *metrics.Float64Histogram
+		allocs, frees           uint64
+		allocdBytes, freedBytes uint64
+		total, totalBytes       uint64
 	}
 	var gc struct {
 		numGC  uint64
@@ -180,10 +198,20 @@ func TestReadMetricsConsistency(t *testing.T) {
 		switch samples[i].Name {
 		case "/memory/classes/total:bytes":
 			totalVirtual.got = samples[i].Value.Uint64()
+		case "/memory/classes/heap/objects:bytes":
+			objects.totalBytes = samples[i].Value.Uint64()
 		case "/gc/heap/objects:objects":
 			objects.total = samples[i].Value.Uint64()
+		case "/gc/heap/allocs:bytes":
+			objects.allocdBytes = samples[i].Value.Uint64()
+		case "/gc/heap/allocs:objects":
+			objects.allocs = samples[i].Value.Uint64()
 		case "/gc/heap/allocs-by-size:bytes":
 			objects.alloc = samples[i].Value.Float64Histogram()
+		case "/gc/heap/frees:bytes":
+			objects.freedBytes = samples[i].Value.Uint64()
+		case "/gc/heap/frees:objects":
+			objects.frees = samples[i].Value.Uint64()
 		case "/gc/heap/frees-by-size:bytes":
 			objects.free = samples[i].Value.Float64Histogram()
 		case "/gc/cycles:gc-cycles":
@@ -203,6 +231,12 @@ func TestReadMetricsConsistency(t *testing.T) {
 	if totalVirtual.got != totalVirtual.want {
 		t.Errorf(`"/memory/classes/total:bytes" does not match sum of /memory/classes/**: got %d, want %d`, totalVirtual.got, totalVirtual.want)
 	}
+	if got, want := objects.allocs-objects.frees, objects.total; got != want {
+		t.Errorf("mismatch between object alloc/free tallies and total: got %d, want %d", got, want)
+	}
+	if got, want := objects.allocdBytes-objects.freedBytes, objects.totalBytes; got != want {
+		t.Errorf("mismatch between object alloc/free tallies and total: got %d, want %d", got, want)
+	}
 	if b, c := len(objects.alloc.Buckets), len(objects.alloc.Counts); b != c+1 {
 		t.Errorf("allocs-by-size has wrong bucket or counts length: %d buckets, %d counts", b, c)
 	}
@@ -222,17 +256,25 @@ func TestReadMetricsConsistency(t *testing.T) {
 			}
 		}
 		if !t.Failed() {
-			got, want := uint64(0), objects.total
+			var gotAlloc, gotFree uint64
+			want := objects.total
 			for i := range objects.alloc.Counts {
 				if objects.alloc.Counts[i] < objects.free.Counts[i] {
 					t.Errorf("found more allocs than frees in object dist bucket %d", i)
 					continue
 				}
-				got += objects.alloc.Counts[i] - objects.free.Counts[i]
+				gotAlloc += objects.alloc.Counts[i]
+				gotFree += objects.free.Counts[i]
 			}
-			if got != want {
+			if got := gotAlloc - gotFree; got != want {
 				t.Errorf("object distribution counts don't match count of live objects: got %d, want %d", got, want)
 			}
+			if gotAlloc != objects.allocs {
+				t.Errorf("object distribution counts don't match total allocs: got %d, want %d", gotAlloc, objects.allocs)
+			}
+			if gotFree != objects.frees {
+				t.Errorf("object distribution counts don't match total allocs: got %d, want %d", gotFree, objects.frees)
+			}
 		}
 	}
 	// The current GC has at least 2 pauses per GC.