runtime: optimize timers.cleanHead

goos: linux
goarch: amd64
pkg: time
cpu: AMD Ryzen 9 7950X 16-Core Processor
                                   │ s7base.txt  │               s7.txt                │
                                   │   sec/op    │   sec/op     vs base                │
AdjustTimers10000-32                 291.4µ ± 5%   263.3µ ± 4%   -9.64% (p=0.000 n=10)
AdjustTimers10000SingleThread-32     1.728m ± 5%   1.742m ± 3%        ~ (p=0.796 n=10)
AdjustTimers10000NoReset-32          253.1µ ± 6%   192.2µ ± 2%  -24.07% (p=0.000 n=10)
AdjustTimers10000NoSleep-32          277.3µ ± 3%   237.0µ ± 2%  -14.54% (p=0.000 n=10)
AdjustTimers10000NoResetNoSleep-32   241.4µ ± 2%   185.2µ ± 1%  -23.30% (p=0.000 n=10)

goos: darwin
goarch: arm64
pkg: time
cpu: Apple M3 Pro
                                   │ m3base.txt  │               m3.txt                │
                                   │   sec/op    │   sec/op     vs base                │
AdjustTimers10000-12                 288.1µ ± 1%   272.6µ ± 3%   -5.38% (p=0.000 n=10)
AdjustTimers10000SingleThread-12     1.195m ± 1%   1.126m ± 1%   -5.74% (p=0.000 n=10)
AdjustTimers10000NoReset-12          280.8µ ± 1%   255.1µ ± 2%   -9.14% (p=0.000 n=10)
AdjustTimers10000NoSleep-12          292.5µ ± 1%   250.2µ ± 2%  -14.47% (p=0.000 n=10)
AdjustTimers10000NoResetNoSleep-12   279.8µ ± 2%   230.3µ ± 1%  -17.69% (p=0.000 n=10)

Change-Id: I36edb40ee2cd11ab44d20bff045fa77609dca648
Reviewed-on: https://go-review.googlesource.com/c/go/+/574739
Reviewed-by: Ian Lance Taylor <iant@google.com>
Auto-Submit: Russ Cox <rsc@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>

1 files changed, 20 insertions, 6 deletions

diff --git a/src/runtime/time.go b/src/runtime/time.go
index 96b7b76b07..42f629d168 100644
--- a/src/runtime/time.go
+++ b/src/runtime/time.go
@@ -239,12 +239,8 @@ func (t *timer) updateHeap(ts *timers) (updated bool) {
 		assertLockHeld(&ts.mu)
 	}
 	if t.state&timerZombie != 0 {
-		// Take timer out of heap, applying final t.whenHeap update first.
-		t.state &^= timerHeaped | timerZombie
-		if t.state&timerModified != 0 {
-			t.state &^= timerModified
-			t.whenHeap = t.when
-		}
+		// Take timer out of heap.
+		t.state &^= timerHeaped | timerZombie | timerModified
 		if ts != nil {
 			ts.zombies.Add(-1)
 			ts.deleteMin()
@@ -651,6 +647,24 @@ func (ts *timers) cleanHead() {
 			return
 		}
 
+		// Delete zombies from tail of heap. It requires no heap adjustments at all,
+		// and doing so increases the chances that when we swap out a zombie
+		// in heap[0] for the tail of the heap, we'll get a non-zombie timer,
+		// shortening this loop.
+		n := len(ts.heap)
+		if t := ts.heap[n-1]; t.astate.Load()&timerZombie != 0 {
+			t.lock()
+			if t.state&timerZombie != 0 {
+				t.state &^= timerHeaped | timerZombie | timerModified
+				t.ts = nil
+				ts.zombies.Add(-1)
+				ts.heap[n-1] = nil
+				ts.heap = ts.heap[:n-1]
+			}
+			t.unlock()
+			continue
+		}
+
 		t := ts.heap[0]
 		if t.ts != ts {
 			throw("bad ts")