aboutsummaryrefslogtreecommitdiff
path: root/src/runtime/asm_amd64.s
diff options
context:
space:
mode:
Diffstat (limited to 'src/runtime/asm_amd64.s')
-rw-r--r--src/runtime/asm_amd64.s248
1 files changed, 170 insertions, 78 deletions
diff --git a/src/runtime/asm_amd64.s b/src/runtime/asm_amd64.s
index b5d01ba73c..517c5a9d3e 100644
--- a/src/runtime/asm_amd64.s
+++ b/src/runtime/asm_amd64.s
@@ -84,9 +84,7 @@ GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8
DATA _rt0_amd64_lib_argv<>(SB)/8, $0
GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8
-// Defined as ABIInternal since it does not use the stack-based Go ABI (and
-// in addition there are no calls to this entry point from Go code).
-TEXT runtime·rt0_go<ABIInternal>(SB),NOSPLIT,$0
+TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
// copy arguments forward on an even stack
MOVQ DI, AX // argc
MOVQ SI, BX // argv
@@ -252,38 +250,26 @@ TEXT runtime·asminit(SB),NOSPLIT,$0-0
// No per-thread init.
RET
+TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
+ CALL runtime·mstart0(SB)
+ RET // not reached
+
/*
* go-routine
*/
-// func gosave(buf *gobuf)
-// save state in Gobuf; setjmp
-TEXT runtime·gosave(SB), NOSPLIT, $0-8
- MOVQ buf+0(FP), AX // gobuf
- LEAQ buf+0(FP), BX // caller's SP
- MOVQ BX, gobuf_sp(AX)
- MOVQ 0(SP), BX // caller's PC
- MOVQ BX, gobuf_pc(AX)
- MOVQ $0, gobuf_ret(AX)
- MOVQ BP, gobuf_bp(AX)
- // Assert ctxt is zero. See func save.
- MOVQ gobuf_ctxt(AX), BX
- TESTQ BX, BX
- JZ 2(PC)
- CALL runtime·badctxt(SB)
- get_tls(CX)
- MOVQ g(CX), BX
- MOVQ BX, gobuf_g(AX)
- RET
-
// func gogo(buf *gobuf)
// restore state from Gobuf; longjmp
-TEXT runtime·gogo(SB), NOSPLIT, $16-8
+TEXT runtime·gogo(SB), NOSPLIT, $0-8
MOVQ buf+0(FP), BX // gobuf
MOVQ gobuf_g(BX), DX
MOVQ 0(DX), CX // make sure g != nil
+ JMP gogo<>(SB)
+
+TEXT gogo<>(SB), NOSPLIT, $0
get_tls(CX)
MOVQ DX, g(CX)
+ MOVQ DX, R14 // set the g register
MOVQ gobuf_sp(BX), SP // restore SP
MOVQ gobuf_ret(BX), AX
MOVQ gobuf_ctxt(BX), DX
@@ -308,7 +294,6 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-8
MOVQ BX, (g_sched+gobuf_pc)(AX)
LEAQ fn+0(FP), BX // caller's SP
MOVQ BX, (g_sched+gobuf_sp)(AX)
- MOVQ AX, (g_sched+gobuf_g)(AX)
MOVQ BP, (g_sched+gobuf_bp)(AX)
// switch to m->g0 & its stack, call fn
@@ -320,6 +305,7 @@ TEXT runtime·mcall(SB), NOSPLIT, $0-8
MOVQ $runtime·badmcall(SB), AX
JMP AX
MOVQ SI, g(CX) // g = m->g0
+ MOVQ SI, R14 // set the g register
MOVQ (g_sched+gobuf_sp)(SI), SP // sp = m->g0->sched.sp
PUSHQ AX
MOVQ DI, DX
@@ -358,19 +344,12 @@ TEXT runtime·systemstack(SB), NOSPLIT, $0-8
// switch stacks
// save our state in g->sched. Pretend to
// be systemstack_switch if the G stack is scanned.
- MOVQ $runtime·systemstack_switch(SB), SI
- MOVQ SI, (g_sched+gobuf_pc)(AX)
- MOVQ SP, (g_sched+gobuf_sp)(AX)
- MOVQ AX, (g_sched+gobuf_g)(AX)
- MOVQ BP, (g_sched+gobuf_bp)(AX)
+ CALL gosave_systemstack_switch<>(SB)
// switch to g0
MOVQ DX, g(CX)
+ MOVQ DX, R14 // set the g register
MOVQ (g_sched+gobuf_sp)(DX), BX
- // make it look like mstart called systemstack on g0, to stop traceback
- SUBQ $8, BX
- MOVQ $runtime·mstart(SB), DX
- MOVQ DX, 0(BX)
MOVQ BX, SP
// call target function
@@ -445,7 +424,6 @@ TEXT runtime·morestack(SB),NOSPLIT,$0-0
// Set g->sched to context in f.
MOVQ 0(SP), AX // f's PC
MOVQ AX, (g_sched+gobuf_pc)(SI)
- MOVQ SI, (g_sched+gobuf_g)(SI)
LEAQ 8(SP), AX // f's SP
MOVQ AX, (g_sched+gobuf_sp)(SI)
MOVQ BP, (g_sched+gobuf_bp)(SI)
@@ -464,8 +442,77 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
MOVL $0, DX
JMP runtime·morestack(SB)
+// REFLECTCALL_USE_REGABI is not defined. It must be defined in conjunction with the
+// register constants in the internal/abi package.
+
+#ifdef REFLECTCALL_USE_REGABI
+// spillArgs stores return values from registers to a *internal/abi.RegArgs in R12.
+TEXT spillArgs<>(SB),NOSPLIT,$0-0
+ MOVQ AX, 0(R12)
+ MOVQ BX, 8(R12)
+ MOVQ CX, 16(R12)
+ MOVQ DI, 24(R12)
+ MOVQ SI, 32(R12)
+ MOVQ R8, 40(R12)
+ MOVQ R9, 48(R12)
+ MOVQ R10, 56(R12)
+ MOVQ R11, 64(R12)
+ MOVQ X0, 72(R12)
+ MOVQ X1, 80(R12)
+ MOVQ X2, 88(R12)
+ MOVQ X3, 96(R12)
+ MOVQ X4, 104(R12)
+ MOVQ X5, 112(R12)
+ MOVQ X6, 120(R12)
+ MOVQ X7, 128(R12)
+ MOVQ X8, 136(R12)
+ MOVQ X9, 144(R12)
+ MOVQ X10, 152(R12)
+ MOVQ X11, 160(R12)
+ MOVQ X12, 168(R12)
+ MOVQ X13, 176(R12)
+ MOVQ X14, 184(R12)
+ RET
+
+// unspillArgs loads args into registers from a *internal/abi.RegArgs in R12.
+TEXT unspillArgs<>(SB),NOSPLIT,$0-0
+ MOVQ 0(R12), AX
+ MOVQ 8(R12), BX
+ MOVQ 16(R12), CX
+ MOVQ 24(R12), DI
+ MOVQ 32(R12), SI
+ MOVQ 40(R12), R8
+ MOVQ 48(R12), R9
+ MOVQ 56(R12), R10
+ MOVQ 64(R12), R11
+ MOVQ 72(R12), X0
+ MOVQ 80(R12), X1
+ MOVQ 88(R12), X2
+ MOVQ 96(R12), X3
+ MOVQ 104(R12), X4
+ MOVQ 112(R12), X5
+ MOVQ 120(R12), X6
+ MOVQ 128(R12), X7
+ MOVQ 136(R12), X8
+ MOVQ 144(R12), X9
+ MOVQ 152(R12), X10
+ MOVQ 160(R12), X11
+ MOVQ 168(R12), X12
+ MOVQ 176(R12), X13
+ MOVQ 184(R12), X14
+ RET
+#else
+// spillArgs stores return values from registers to a pointer in R12.
+TEXT spillArgs<>(SB),NOSPLIT,$0-0
+ RET
+
+// unspillArgs loads args into registers from a pointer in R12.
+TEXT unspillArgs<>(SB),NOSPLIT,$0-0
+ RET
+#endif
+
// reflectcall: call a function with the given argument list
-// func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
+// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
// we don't have variable-sized frames, so we use a small number
// of constant-sized-frame functions to encode a few bits of size in the pc.
// Caution: ugly multiline assembly macros in your future!
@@ -477,8 +524,8 @@ TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
JMP AX
// Note: can't just "JMP NAME(SB)" - bad inlining results.
-TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-32
- MOVLQZX argsize+24(FP), CX
+TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-48
+ MOVLQZX frameSize+32(FP), CX
DISPATCH(runtime·call16, 16)
DISPATCH(runtime·call32, 32)
DISPATCH(runtime·call64, 64)
@@ -510,23 +557,28 @@ TEXT ·reflectcall<ABIInternal>(SB), NOSPLIT, $0-32
JMP AX
#define CALLFN(NAME,MAXSIZE) \
-TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
+TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
NO_LOCAL_POINTERS; \
/* copy arguments to stack */ \
- MOVQ argptr+16(FP), SI; \
- MOVLQZX argsize+24(FP), CX; \
+ MOVQ stackArgs+16(FP), SI; \
+ MOVLQZX stackArgsSize+24(FP), CX; \
MOVQ SP, DI; \
REP;MOVSB; \
+ /* set up argument registers */ \
+ MOVQ regArgs+40(FP), R12; \
+ CALL unspillArgs<>(SB); \
/* call function */ \
MOVQ f+8(FP), DX; \
PCDATA $PCDATA_StackMapIndex, $0; \
- MOVQ (DX), AX; \
- CALL AX; \
- /* copy return values back */ \
- MOVQ argtype+0(FP), DX; \
- MOVQ argptr+16(FP), DI; \
- MOVLQZX argsize+24(FP), CX; \
- MOVLQZX retoffset+28(FP), BX; \
+ MOVQ (DX), R12; \
+ CALL R12; \
+ /* copy register return values back */ \
+ MOVQ regArgs+40(FP), R12; \
+ CALL spillArgs<>(SB); \
+ MOVLQZX stackArgsSize+24(FP), CX; \
+ MOVLQZX stackRetOffset+28(FP), BX; \
+ MOVQ stackArgs+16(FP), DI; \
+ MOVQ stackArgsType+0(FP), DX; \
MOVQ SP, SI; \
ADDQ BX, DI; \
ADDQ BX, SI; \
@@ -538,12 +590,13 @@ TEXT NAME(SB), WRAPPER, $MAXSIZE-32; \
// separate function so it can allocate stack space for the arguments
// to reflectcallmove. It does not follow the Go ABI; it expects its
// arguments in registers.
-TEXT callRet<>(SB), NOSPLIT, $32-0
+TEXT callRet<>(SB), NOSPLIT, $40-0
NO_LOCAL_POINTERS
MOVQ DX, 0(SP)
MOVQ DI, 8(SP)
MOVQ SI, 16(SP)
MOVQ CX, 24(SP)
+ MOVQ R12, 32(SP)
CALL runtime·reflectcallmove(SB)
RET
@@ -604,21 +657,44 @@ TEXT runtime·jmpdefer(SB), NOSPLIT, $0-16
MOVQ 0(DX), BX
JMP BX // but first run the deferred function
-// Save state of caller into g->sched. Smashes R8, R9.
-TEXT gosave<>(SB),NOSPLIT,$0
- get_tls(R8)
- MOVQ g(R8), R8
- MOVQ 0(SP), R9
- MOVQ R9, (g_sched+gobuf_pc)(R8)
+// Save state of caller into g->sched,
+// but using fake PC from systemstack_switch.
+// Must only be called from functions with no locals ($0)
+// or else unwinding from systemstack_switch is incorrect.
+// Smashes R9.
+TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0
+#ifndef GOEXPERIMENT_REGABI
+ get_tls(R14)
+ MOVQ g(R14), R14
+#endif
+ MOVQ $runtime·systemstack_switch(SB), R9
+ MOVQ R9, (g_sched+gobuf_pc)(R14)
LEAQ 8(SP), R9
- MOVQ R9, (g_sched+gobuf_sp)(R8)
- MOVQ $0, (g_sched+gobuf_ret)(R8)
- MOVQ BP, (g_sched+gobuf_bp)(R8)
+ MOVQ R9, (g_sched+gobuf_sp)(R14)
+ MOVQ $0, (g_sched+gobuf_ret)(R14)
+ MOVQ BP, (g_sched+gobuf_bp)(R14)
// Assert ctxt is zero. See func save.
- MOVQ (g_sched+gobuf_ctxt)(R8), R9
+ MOVQ (g_sched+gobuf_ctxt)(R14), R9
TESTQ R9, R9
JZ 2(PC)
- CALL runtime·badctxt(SB)
+ CALL runtime·abort(SB)
+ RET
+
+// func asmcgocall_no_g(fn, arg unsafe.Pointer)
+// Call fn(arg) aligned appropriately for the gcc ABI.
+// Called on a system stack, and there may be no g yet (during needm).
+TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
+ MOVQ fn+0(FP), AX
+ MOVQ arg+8(FP), BX
+ MOVQ SP, DX
+ SUBQ $32, SP
+ ANDQ $~15, SP // alignment
+ MOVQ DX, 8(SP)
+ MOVQ BX, DI // DI = first argument in AMD64 ABI
+ MOVQ BX, CX // CX = first argument in Win64
+ CALL AX
+ MOVQ 8(SP), DX
+ MOVQ DX, SP
RET
// func asmcgocall(fn, arg unsafe.Pointer) int32
@@ -649,7 +725,7 @@ TEXT ·asmcgocall(SB),NOSPLIT,$0-20
// Switch to system stack.
MOVQ m_g0(R8), SI
- CALL gosave<>(SB)
+ CALL gosave_systemstack_switch<>(SB)
MOVQ SI, g(CX)
MOVQ (g_sched+gobuf_sp)(SI), SP
@@ -846,6 +922,7 @@ settls:
TEXT setg_gcc<>(SB),NOSPLIT,$0
get_tls(AX)
MOVQ DI, g(AX)
+ MOVQ DI, R14 // set the g register
RET
TEXT runtime·abort(SB),NOSPLIT,$0-0
@@ -1367,7 +1444,7 @@ TEXT _cgo_topofstack(SB),NOSPLIT,$0
// so as to make it identifiable to traceback (this
// function it used as a sentinel; traceback wants to
// see the func PC, not a wrapper PC).
-TEXT runtime·goexit<ABIInternal>(SB),NOSPLIT,$0-0
+TEXT runtime·goexit<ABIInternal>(SB),NOSPLIT|TOPFRAME,$0-0
BYTE $0x90 // NOP
CALL runtime·goexit1(SB) // does not return
// traceback from goexit1 must hit code range of goexit
@@ -1382,6 +1459,18 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
POPQ R15
RET
+// Initialize special registers then jump to sigpanic.
+// This function is injected from the signal handler for panicking
+// signals. It is quite painful to set X15 in the signal context,
+// so we do it here.
+TEXT ·sigpanic0<ABIInternal>(SB),NOSPLIT,$0-0
+#ifdef GOEXPERIMENT_REGABI
+ get_tls(R14)
+ MOVQ g(R14), R14
+ XORPS X15, X15
+#endif
+ JMP ·sigpanic<ABIInternal>(SB)
+
// gcWriteBarrier performs a heap pointer write and informs the GC.
//
// gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
@@ -1390,24 +1479,28 @@ TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
// It clobbers FLAGS. It does not clobber any general-purpose registers,
// but may clobber others (e.g., SSE registers).
// Defined as ABIInternal since it does not use the stack-based Go ABI.
-TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$120
+TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$112
// Save the registers clobbered by the fast path. This is slightly
// faster than having the caller spill these.
- MOVQ R14, 104(SP)
- MOVQ R13, 112(SP)
+ MOVQ R12, 96(SP)
+ MOVQ R13, 104(SP)
// TODO: Consider passing g.m.p in as an argument so they can be shared
// across a sequence of write barriers.
+#ifdef GOEXPERIMENT_REGABI
+ MOVQ g_m(R14), R13
+#else
get_tls(R13)
MOVQ g(R13), R13
MOVQ g_m(R13), R13
+#endif
MOVQ m_p(R13), R13
- MOVQ (p_wbBuf+wbBuf_next)(R13), R14
+ MOVQ (p_wbBuf+wbBuf_next)(R13), R12
// Increment wbBuf.next position.
- LEAQ 16(R14), R14
- MOVQ R14, (p_wbBuf+wbBuf_next)(R13)
- CMPQ R14, (p_wbBuf+wbBuf_end)(R13)
+ LEAQ 16(R12), R12
+ MOVQ R12, (p_wbBuf+wbBuf_next)(R13)
+ CMPQ R12, (p_wbBuf+wbBuf_end)(R13)
// Record the write.
- MOVQ AX, -16(R14) // Record value
+ MOVQ AX, -16(R12) // Record value
// Note: This turns bad pointer writes into bad
// pointer reads, which could be confusing. We could avoid
// reading from obviously bad pointers, which would
@@ -1415,12 +1508,12 @@ TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$120
// patch this up in the signal handler, or use XCHG to
// combine the read and the write.
MOVQ (DI), R13
- MOVQ R13, -8(R14) // Record *slot
+ MOVQ R13, -8(R12) // Record *slot
// Is the buffer full? (flags set in CMPQ above)
JEQ flush
ret:
- MOVQ 104(SP), R14
- MOVQ 112(SP), R13
+ MOVQ 96(SP), R12
+ MOVQ 104(SP), R13
// Do the write.
MOVQ AX, (DI)
RET
@@ -1450,10 +1543,10 @@ flush:
MOVQ R9, 64(SP)
MOVQ R10, 72(SP)
MOVQ R11, 80(SP)
- MOVQ R12, 88(SP)
+ // R12 already saved
// R13 already saved
- // R14 already saved
- MOVQ R15, 96(SP)
+ // R14 is g
+ MOVQ R15, 88(SP)
// This takes arguments DI and AX
CALL runtime·wbBufFlush(SB)
@@ -1469,8 +1562,7 @@ flush:
MOVQ 64(SP), R9
MOVQ 72(SP), R10
MOVQ 80(SP), R11
- MOVQ 88(SP), R12
- MOVQ 96(SP), R15
+ MOVQ 88(SP), R15
JMP ret
// gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX.
7apu2bq3rhoylwmucxizk54afw5jyda7pho4j5zdcqpwkufke7zdzwad.onion