runtime: fix atomic operations on non-heap addresses

Race detector runtime does not tolerate operations on addresses
that was not previously declared with __tsan_map_shadow
(namely, data, bss and heap). The corresponding address
checks for atomic operations were removed in
https://golang.org/cl/111310044
Restore these checks.
It's tricker than just not calling into race runtime,
because it is the race runtime that makes the atomic
operations themselves (if we do not call into race runtime
we skip the atomic operation itself as well). So instead we call
__tsan_go_ignore_sync_start/end around the atomic operation.
This forces race runtime to skip all other processing
except than doing the atomic operation itself.
Fixes #9136.

LGTM=rsc
R=rsc
CC=golang-codereviews
https://golang.org/cl/179030043

1 files changed, 37 insertions, 6 deletions

diff --git a/src/runtime/race_amd64.s b/src/runtime/race_amd64.s
index bdea28c7c0..a96d9de123 100644
--- a/src/runtime/race_amd64.s
+++ b/src/runtime/race_amd64.s
@@ -138,17 +138,15 @@ TEXT	racecalladdr<>(SB), NOSPLIT, $0-0
 	get_tls(R12)
 	MOVQ	g(R12), R14
 	MOVQ	g_racectx(R14), RARG0	// goroutine context
-	// Check that addr is within [arenastart, arenaend) or within [noptrdata, enoptrbss).
+	// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
 	CMPQ	RARG1, runtime·racearenastart(SB)
 	JB	racecalladdr_data
 	CMPQ	RARG1, runtime·racearenaend(SB)
 	JB	racecalladdr_call
 racecalladdr_data:
-	MOVQ	$runtime·noptrdata(SB), R13
-	CMPQ	RARG1, R13
+	CMPQ	RARG1, runtime·racedatastart(SB)
 	JB	racecalladdr_ret
-	MOVQ	$runtime·enoptrbss(SB), R13
-	CMPQ	RARG1, R13
+	CMPQ	RARG1, runtime·racedataend(SB)
 	JAE	racecalladdr_ret
 racecalladdr_call:
 	MOVQ	AX, AX		// w/o this 6a miscompiles this function
@@ -166,6 +164,7 @@ TEXT	runtime·racefuncenter(SB), NOSPLIT, $0-8
 	MOVQ	callpc+0(FP), RARG1
 	// void __tsan_func_enter(ThreadState *thr, void *pc);
 	MOVQ	$__tsan_func_enter(SB), AX
+	// racecall<> preserves R15
 	CALL	racecall<>(SB)
 	MOVQ	R15, DX	// restore function entry context
 	RET
@@ -306,13 +305,45 @@ TEXT	sync∕atomic·CompareAndSwapPointer(SB), NOSPLIT, $0-0
 TEXT	racecallatomic<>(SB), NOSPLIT, $0-0
 	// Trigger SIGSEGV early.
 	MOVQ	16(SP), R12
-	MOVL	(R12), R12
+	MOVL	(R12), R13
+	// Check that addr is within [arenastart, arenaend) or within [racedatastart, racedataend).
+	CMPQ	R12, runtime·racearenastart(SB)
+	JB	racecallatomic_data
+	CMPQ	R12, runtime·racearenaend(SB)
+	JB	racecallatomic_ok
+racecallatomic_data:
+	CMPQ	R12, runtime·racedatastart(SB)
+	JB	racecallatomic_ignore
+	CMPQ	R12, runtime·racedataend(SB)
+	JAE	racecallatomic_ignore
+racecallatomic_ok:
+	// Addr is within the good range, call the atomic function.
 	get_tls(R12)
 	MOVQ	g(R12), R14
 	MOVQ	g_racectx(R14), RARG0	// goroutine context
 	MOVQ	8(SP), RARG1	// caller pc
 	MOVQ	(SP), RARG2	// pc
 	LEAQ	16(SP), RARG3	// arguments
+	JMP	racecall<>(SB)	// does not return
+racecallatomic_ignore:
+	// Addr is outside the good range.
+	// Call __tsan_go_ignore_sync_begin to ignore synchronization during the atomic op.
+	// An attempt to synchronize on the address would cause crash.
+	MOVQ	AX, R15	// remember the original function
+	MOVQ	$__tsan_go_ignore_sync_begin(SB), AX
+	MOVQ	g(R12), R14
+	MOVQ	g_racectx(R14), RARG0	// goroutine context
+	CALL	racecall<>(SB)
+	MOVQ	R15, AX	// restore the original function
+	// Call the atomic function.
+	MOVQ	g_racectx(R14), RARG0	// goroutine context
+	MOVQ	8(SP), RARG1	// caller pc
+	MOVQ	(SP), RARG2	// pc
+	LEAQ	16(SP), RARG3	// arguments
+	CALL	racecall<>(SB)
+	// Call __tsan_go_ignore_sync_end.
+	MOVQ	$__tsan_go_ignore_sync_end(SB), AX
+	MOVQ	g_racectx(R14), RARG0	// goroutine context
 	JMP	racecall<>(SB)
 
 // void runtime·racecall(void(*f)(...), ...)