Text file src/runtime/asm_riscv64.s

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "go_asm.h"
     6  #include "funcdata.h"
     7  #include "textflag.h"
     8  
     9  // func rt0_go()
    10  TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
    11  	// X2 = stack; A0 = argc; A1 = argv
    12  	ADD	$-24, X2
    13  	MOV	A0, 8(X2)	// argc
    14  	MOV	A1, 16(X2)	// argv
    15  
    16  	// create istack out of the given (operating system) stack.
    17  	// _cgo_init may update stackguard.
    18  	MOV	$runtime·g0(SB), g
    19  	MOV	$(-64*1024), T0
    20  	ADD	T0, X2, T1
    21  	MOV	T1, g_stackguard0(g)
    22  	MOV	T1, g_stackguard1(g)
    23  	MOV	T1, (g_stack+stack_lo)(g)
    24  	MOV	X2, (g_stack+stack_hi)(g)
    25  
    26  	// if there is a _cgo_init, call it using the gcc ABI.
    27  	MOV	_cgo_init(SB), T0
    28  	BEQ	T0, ZERO, nocgo
    29  
    30  	MOV	ZERO, A3		// arg 3: not used
    31  	MOV	ZERO, A2		// arg 2: not used
    32  	MOV	$setg_gcc<>(SB), A1	// arg 1: setg
    33  	MOV	g, A0			// arg 0: G
    34  	JALR	RA, T0
    35  
    36  nocgo:
    37  	// update stackguard after _cgo_init
    38  	MOV	(g_stack+stack_lo)(g), T0
    39  	ADD	$const__StackGuard, T0
    40  	MOV	T0, g_stackguard0(g)
    41  	MOV	T0, g_stackguard1(g)
    42  
    43  	// set the per-goroutine and per-mach "registers"
    44  	MOV	$runtime·m0(SB), T0
    45  
    46  	// save m->g0 = g0
    47  	MOV	g, m_g0(T0)
    48  	// save m0 to g0->m
    49  	MOV	T0, g_m(g)
    50  
    51  	CALL	runtime·check(SB)
    52  
    53  	// args are already prepared
    54  	CALL	runtime·args(SB)
    55  	CALL	runtime·osinit(SB)
    56  	CALL	runtime·schedinit(SB)
    57  
    58  	// create a new goroutine to start program
    59  	MOV	$runtime·mainPC(SB), T0		// entry
    60  	ADD	$-16, X2
    61  	MOV	T0, 8(X2)
    62  	MOV	ZERO, 0(X2)
    63  	CALL	runtime·newproc(SB)
    64  	ADD	$16, X2
    65  
    66  	// start this M
    67  	CALL	runtime·mstart(SB)
    68  
    69  	WORD $0 // crash if reached
    70  	RET
    71  
    72  TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
    73  	CALL	runtime·mstart0(SB)
    74  	RET // not reached
    75  
    76  // void setg_gcc(G*); set g called from gcc with g in A0
    77  TEXT setg_gcc<>(SB),NOSPLIT,$0-0
    78  	MOV	A0, g
    79  	CALL	runtime·save_g(SB)
    80  	RET
    81  
    82  // func cputicks() int64
    83  TEXT runtime·cputicks(SB),NOSPLIT,$0-8
    84  	// RDTIME to emulate cpu ticks
    85  	// RDCYCLE reads counter that is per HART(core) based
    86  	// according to the riscv manual, see issue 46737
    87  	RDTIME	A0
    88  	MOV	A0, ret+0(FP)
    89  	RET
    90  
    91  // systemstack_switch is a dummy routine that systemstack leaves at the bottom
    92  // of the G stack. We need to distinguish the routine that
    93  // lives at the bottom of the G stack from the one that lives
    94  // at the top of the system stack because the one at the top of
    95  // the system stack terminates the stack walk (see topofstack()).
    96  TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
    97  	UNDEF
    98  	JALR	RA, ZERO	// make sure this function is not leaf
    99  	RET
   100  
   101  // func systemstack(fn func())
   102  TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   103  	MOV	fn+0(FP), CTXT	// CTXT = fn
   104  	MOV	g_m(g), T0	// T0 = m
   105  
   106  	MOV	m_gsignal(T0), T1	// T1 = gsignal
   107  	BEQ	g, T1, noswitch
   108  
   109  	MOV	m_g0(T0), T1	// T1 = g0
   110  	BEQ	g, T1, noswitch
   111  
   112  	MOV	m_curg(T0), T2
   113  	BEQ	g, T2, switch
   114  
   115  	// Bad: g is not gsignal, not g0, not curg. What is it?
   116  	// Hide call from linker nosplit analysis.
   117  	MOV	$runtime·badsystemstack(SB), T1
   118  	JALR	RA, T1
   119  
   120  switch:
   121  	// save our state in g->sched. Pretend to
   122  	// be systemstack_switch if the G stack is scanned.
   123  	CALL	gosave_systemstack_switch<>(SB)
   124  
   125  	// switch to g0
   126  	MOV	T1, g
   127  	CALL	runtime·save_g(SB)
   128  	MOV	(g_sched+gobuf_sp)(g), T0
   129  	MOV	T0, X2
   130  
   131  	// call target function
   132  	MOV	0(CTXT), T1	// code pointer
   133  	JALR	RA, T1
   134  
   135  	// switch back to g
   136  	MOV	g_m(g), T0
   137  	MOV	m_curg(T0), g
   138  	CALL	runtime·save_g(SB)
   139  	MOV	(g_sched+gobuf_sp)(g), X2
   140  	MOV	ZERO, (g_sched+gobuf_sp)(g)
   141  	RET
   142  
   143  noswitch:
   144  	// already on m stack, just call directly
   145  	// Using a tail call here cleans up tracebacks since we won't stop
   146  	// at an intermediate systemstack.
   147  	MOV	0(CTXT), T1	// code pointer
   148  	ADD	$8, X2
   149  	JMP	(T1)
   150  
   151  TEXT runtime·getcallerpc(SB),NOSPLIT|NOFRAME,$0-8
   152  	MOV	0(X2), T0		// LR saved by caller
   153  	MOV	T0, ret+0(FP)
   154  	RET
   155  
   156  /*
   157   * support for morestack
   158   */
   159  
   160  // Called during function prolog when more stack is needed.
   161  // Called with return address (i.e. caller's PC) in X5 (aka T0),
   162  // and the LR register contains the caller's LR.
   163  //
   164  // The traceback routines see morestack on a g0 as being
   165  // the top of a stack (for example, morestack calling newstack
   166  // calling the scheduler calling newm calling gc), so we must
   167  // record an argument size. For that purpose, it has no arguments.
   168  
   169  // func morestack()
   170  TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
   171  	// Cannot grow scheduler stack (m->g0).
   172  	MOV	g_m(g), A0
   173  	MOV	m_g0(A0), A1
   174  	BNE	g, A1, 3(PC)
   175  	CALL	runtime·badmorestackg0(SB)
   176  	CALL	runtime·abort(SB)
   177  
   178  	// Cannot grow signal stack (m->gsignal).
   179  	MOV	m_gsignal(A0), A1
   180  	BNE	g, A1, 3(PC)
   181  	CALL	runtime·badmorestackgsignal(SB)
   182  	CALL	runtime·abort(SB)
   183  
   184  	// Called from f.
   185  	// Set g->sched to context in f.
   186  	MOV	X2, (g_sched+gobuf_sp)(g)
   187  	MOV	T0, (g_sched+gobuf_pc)(g)
   188  	MOV	RA, (g_sched+gobuf_lr)(g)
   189  	MOV	CTXT, (g_sched+gobuf_ctxt)(g)
   190  
   191  	// Called from f.
   192  	// Set m->morebuf to f's caller.
   193  	MOV	RA, (m_morebuf+gobuf_pc)(A0)	// f's caller's PC
   194  	MOV	X2, (m_morebuf+gobuf_sp)(A0)	// f's caller's SP
   195  	MOV	g, (m_morebuf+gobuf_g)(A0)
   196  
   197  	// Call newstack on m->g0's stack.
   198  	MOV	m_g0(A0), g
   199  	CALL	runtime·save_g(SB)
   200  	MOV	(g_sched+gobuf_sp)(g), X2
   201  	// Create a stack frame on g0 to call newstack.
   202  	MOV	ZERO, -8(X2)	// Zero saved LR in frame
   203  	ADD	$-8, X2
   204  	CALL	runtime·newstack(SB)
   205  
   206  	// Not reached, but make sure the return PC from the call to newstack
   207  	// is still in this function, and not the beginning of the next.
   208  	UNDEF
   209  
   210  // func morestack_noctxt()
   211  TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
   212  	// Force SPWRITE. This function doesn't actually write SP,
   213  	// but it is called with a special calling convention where
   214  	// the caller doesn't save LR on stack but passes it as a
   215  	// register, and the unwinder currently doesn't understand.
   216  	// Make it SPWRITE to stop unwinding. (See issue 54332)
   217  	MOV	X2, X2
   218  
   219  	MOV	ZERO, CTXT
   220  	JMP	runtime·morestack(SB)
   221  
   222  // AES hashing not implemented for riscv64
   223  TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
   224  	JMP	runtime·memhashFallback<ABIInternal>(SB)
   225  TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
   226  	JMP	runtime·strhashFallback<ABIInternal>(SB)
   227  TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
   228  	JMP	runtime·memhash32Fallback<ABIInternal>(SB)
   229  TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
   230  	JMP	runtime·memhash64Fallback<ABIInternal>(SB)
   231  
   232  // func return0()
   233  TEXT runtime·return0(SB), NOSPLIT, $0
   234  	MOV	$0, A0
   235  	RET
   236  
   237  // restore state from Gobuf; longjmp
   238  
   239  // func gogo(buf *gobuf)
   240  TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
   241  	MOV	buf+0(FP), T0
   242  	MOV	gobuf_g(T0), T1
   243  	MOV	0(T1), ZERO // make sure g != nil
   244  	JMP	gogo<>(SB)
   245  
   246  TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
   247  	MOV	T1, g
   248  	CALL	runtime·save_g(SB)
   249  
   250  	MOV	gobuf_sp(T0), X2
   251  	MOV	gobuf_lr(T0), RA
   252  	MOV	gobuf_ret(T0), A0
   253  	MOV	gobuf_ctxt(T0), CTXT
   254  	MOV	ZERO, gobuf_sp(T0)
   255  	MOV	ZERO, gobuf_ret(T0)
   256  	MOV	ZERO, gobuf_lr(T0)
   257  	MOV	ZERO, gobuf_ctxt(T0)
   258  	MOV	gobuf_pc(T0), T0
   259  	JALR	ZERO, T0
   260  
   261  // func procyield(cycles uint32)
   262  TEXT runtime·procyield(SB),NOSPLIT,$0-0
   263  	RET
   264  
   265  // Switch to m->g0's stack, call fn(g).
   266  // Fn must never return. It should gogo(&g->sched)
   267  // to keep running g.
   268  
   269  // func mcall(fn func(*g))
   270  TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
   271  #ifdef GOEXPERIMENT_regabiargs
   272  	MOV	X10, CTXT
   273  #else
   274  	MOV	fn+0(FP), CTXT
   275  #endif
   276  
   277  	// Save caller state in g->sched
   278  	MOV	X2, (g_sched+gobuf_sp)(g)
   279  	MOV	RA, (g_sched+gobuf_pc)(g)
   280  	MOV	ZERO, (g_sched+gobuf_lr)(g)
   281  
   282  	// Switch to m->g0 & its stack, call fn.
   283  	MOV	g, X10
   284  	MOV	g_m(g), T1
   285  	MOV	m_g0(T1), g
   286  	CALL	runtime·save_g(SB)
   287  	BNE	g, X10, 2(PC)
   288  	JMP	runtime·badmcall(SB)
   289  	MOV	0(CTXT), T1			// code pointer
   290  	MOV	(g_sched+gobuf_sp)(g), X2	// sp = m->g0->sched.sp
   291  	// we don't need special macro for regabi since arg0(X10) = g
   292  	ADD	$-16, X2
   293  	MOV	X10, 8(X2)			// setup g
   294  	MOV	ZERO, 0(X2)			// clear return address
   295  	JALR	RA, T1
   296  	JMP	runtime·badmcall2(SB)
   297  
   298  // Save state of caller into g->sched,
   299  // but using fake PC from systemstack_switch.
   300  // Must only be called from functions with no locals ($0)
   301  // or else unwinding from systemstack_switch is incorrect.
   302  // Smashes X31.
   303  TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
   304  	MOV	$runtime·systemstack_switch(SB), X31
   305  	ADD	$8, X31	// get past prologue
   306  	MOV	X31, (g_sched+gobuf_pc)(g)
   307  	MOV	X2, (g_sched+gobuf_sp)(g)
   308  	MOV	ZERO, (g_sched+gobuf_lr)(g)
   309  	MOV	ZERO, (g_sched+gobuf_ret)(g)
   310  	// Assert ctxt is zero. See func save.
   311  	MOV	(g_sched+gobuf_ctxt)(g), X31
   312  	BEQ	ZERO, X31, 2(PC)
   313  	CALL	runtime·abort(SB)
   314  	RET
   315  
   316  // func asmcgocall(fn, arg unsafe.Pointer) int32
   317  // Call fn(arg) on the scheduler stack,
   318  // aligned appropriately for the gcc ABI.
   319  // See cgocall.go for more details.
   320  TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   321  	MOV	fn+0(FP), X5
   322  	MOV	arg+8(FP), X10
   323  
   324  	MOV	X2, X8	// save original stack pointer
   325  	MOV	g, X9
   326  
   327  	// Figure out if we need to switch to m->g0 stack.
   328  	// We get called to create new OS threads too, and those
   329  	// come in on the m->g0 stack already. Or we might already
   330  	// be on the m->gsignal stack.
   331  	MOV	g_m(g), X6
   332  	MOV	m_gsignal(X6), X7
   333  	BEQ	X7, g, g0
   334  	MOV	m_g0(X6), X7
   335  	BEQ	X7, g, g0
   336  
   337  	CALL	gosave_systemstack_switch<>(SB)
   338  	MOV	X7, g
   339  	CALL	runtime·save_g(SB)
   340  	MOV	(g_sched+gobuf_sp)(g), X2
   341  
   342  	// Now on a scheduling stack (a pthread-created stack).
   343  g0:
   344  	// Save room for two of our pointers.
   345  	ADD	$-16, X2
   346  	MOV	X9, 0(X2)	// save old g on stack
   347  	MOV	(g_stack+stack_hi)(X9), X9
   348  	SUB	X8, X9, X8
   349  	MOV	X8, 8(X2)	// save depth in old g stack (can't just save SP, as stack might be copied during a callback)
   350  
   351  	JALR	RA, (X5)
   352  
   353  	// Restore g, stack pointer. X10 is return value.
   354  	MOV	0(X2), g
   355  	CALL	runtime·save_g(SB)
   356  	MOV	(g_stack+stack_hi)(g), X5
   357  	MOV	8(X2), X6
   358  	SUB	X6, X5, X6
   359  	MOV	X6, X2
   360  
   361  	MOVW	X10, ret+16(FP)
   362  	RET
   363  
   364  // func asminit()
   365  TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
   366  	RET
   367  
   368  // reflectcall: call a function with the given argument list
   369  // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   370  // we don't have variable-sized frames, so we use a small number
   371  // of constant-sized-frame functions to encode a few bits of size in the pc.
   372  // Caution: ugly multiline assembly macros in your future!
   373  
   374  #define DISPATCH(NAME,MAXSIZE)	\
   375  	MOV	$MAXSIZE, T1	\
   376  	BLTU	T1, T0, 3(PC)	\
   377  	MOV	$NAME(SB), T2;	\
   378  	JALR	ZERO, T2
   379  // Note: can't just "BR NAME(SB)" - bad inlining results.
   380  
   381  // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   382  TEXT reflect·call(SB), NOSPLIT, $0-0
   383  	JMP	·reflectcall(SB)
   384  
   385  // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   386  TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
   387  	MOVWU	frameSize+32(FP), T0
   388  	DISPATCH(runtime·call16, 16)
   389  	DISPATCH(runtime·call32, 32)
   390  	DISPATCH(runtime·call64, 64)
   391  	DISPATCH(runtime·call128, 128)
   392  	DISPATCH(runtime·call256, 256)
   393  	DISPATCH(runtime·call512, 512)
   394  	DISPATCH(runtime·call1024, 1024)
   395  	DISPATCH(runtime·call2048, 2048)
   396  	DISPATCH(runtime·call4096, 4096)
   397  	DISPATCH(runtime·call8192, 8192)
   398  	DISPATCH(runtime·call16384, 16384)
   399  	DISPATCH(runtime·call32768, 32768)
   400  	DISPATCH(runtime·call65536, 65536)
   401  	DISPATCH(runtime·call131072, 131072)
   402  	DISPATCH(runtime·call262144, 262144)
   403  	DISPATCH(runtime·call524288, 524288)
   404  	DISPATCH(runtime·call1048576, 1048576)
   405  	DISPATCH(runtime·call2097152, 2097152)
   406  	DISPATCH(runtime·call4194304, 4194304)
   407  	DISPATCH(runtime·call8388608, 8388608)
   408  	DISPATCH(runtime·call16777216, 16777216)
   409  	DISPATCH(runtime·call33554432, 33554432)
   410  	DISPATCH(runtime·call67108864, 67108864)
   411  	DISPATCH(runtime·call134217728, 134217728)
   412  	DISPATCH(runtime·call268435456, 268435456)
   413  	DISPATCH(runtime·call536870912, 536870912)
   414  	DISPATCH(runtime·call1073741824, 1073741824)
   415  	MOV	$runtime·badreflectcall(SB), T2
   416  	JALR	ZERO, T2
   417  
   418  #define CALLFN(NAME,MAXSIZE)			\
   419  TEXT NAME(SB), WRAPPER, $MAXSIZE-48;		\
   420  	NO_LOCAL_POINTERS;			\
   421  	/* copy arguments to stack */		\
   422  	MOV	stackArgs+16(FP), A1;			\
   423  	MOVWU	stackArgsSize+24(FP), A2;		\
   424  	MOV	X2, A3;				\
   425  	ADD	$8, A3;				\
   426  	ADD	A3, A2;				\
   427  	BEQ	A3, A2, 6(PC);			\
   428  	MOVBU	(A1), A4;			\
   429  	ADD	$1, A1;				\
   430  	MOVB	A4, (A3);			\
   431  	ADD	$1, A3;				\
   432  	JMP	-5(PC);				\
   433  	/* set up argument registers */		\
   434  	MOV	regArgs+40(FP), X25;		\
   435  	CALL	·unspillArgs(SB);		\
   436  	/* call function */			\
   437  	MOV	f+8(FP), CTXT;			\
   438  	MOV	(CTXT), X25;			\
   439  	PCDATA  $PCDATA_StackMapIndex, $0;	\
   440  	JALR	RA, X25;				\
   441  	/* copy return values back */		\
   442  	MOV	regArgs+40(FP), X25;		\
   443  	CALL	·spillArgs(SB);		\
   444  	MOV	stackArgsType+0(FP), A5;		\
   445  	MOV	stackArgs+16(FP), A1;			\
   446  	MOVWU	stackArgsSize+24(FP), A2;			\
   447  	MOVWU	stackRetOffset+28(FP), A4;		\
   448  	ADD	$8, X2, A3;			\
   449  	ADD	A4, A3; 			\
   450  	ADD	A4, A1;				\
   451  	SUB	A4, A2;				\
   452  	CALL	callRet<>(SB);			\
   453  	RET
   454  
   455  // callRet copies return values back at the end of call*. This is a
   456  // separate function so it can allocate stack space for the arguments
   457  // to reflectcallmove. It does not follow the Go ABI; it expects its
   458  // arguments in registers.
   459  TEXT callRet<>(SB), NOSPLIT, $40-0
   460  	NO_LOCAL_POINTERS
   461  	MOV	A5, 8(X2)
   462  	MOV	A1, 16(X2)
   463  	MOV	A3, 24(X2)
   464  	MOV	A2, 32(X2)
   465  	MOV	X25, 40(X2)
   466  	CALL	runtime·reflectcallmove(SB)
   467  	RET
   468  
   469  CALLFN(·call16, 16)
   470  CALLFN(·call32, 32)
   471  CALLFN(·call64, 64)
   472  CALLFN(·call128, 128)
   473  CALLFN(·call256, 256)
   474  CALLFN(·call512, 512)
   475  CALLFN(·call1024, 1024)
   476  CALLFN(·call2048, 2048)
   477  CALLFN(·call4096, 4096)
   478  CALLFN(·call8192, 8192)
   479  CALLFN(·call16384, 16384)
   480  CALLFN(·call32768, 32768)
   481  CALLFN(·call65536, 65536)
   482  CALLFN(·call131072, 131072)
   483  CALLFN(·call262144, 262144)
   484  CALLFN(·call524288, 524288)
   485  CALLFN(·call1048576, 1048576)
   486  CALLFN(·call2097152, 2097152)
   487  CALLFN(·call4194304, 4194304)
   488  CALLFN(·call8388608, 8388608)
   489  CALLFN(·call16777216, 16777216)
   490  CALLFN(·call33554432, 33554432)
   491  CALLFN(·call67108864, 67108864)
   492  CALLFN(·call134217728, 134217728)
   493  CALLFN(·call268435456, 268435456)
   494  CALLFN(·call536870912, 536870912)
   495  CALLFN(·call1073741824, 1073741824)
   496  
   497  // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
   498  // Must obey the gcc calling convention.
   499  TEXT _cgo_topofstack(SB),NOSPLIT,$8
   500  	// g (X27) and REG_TMP (X31) might be clobbered by load_g.
   501  	// X27 is callee-save in the gcc calling convention, so save it.
   502  	MOV	g, savedX27-8(SP)
   503  
   504  	CALL	runtime·load_g(SB)
   505  	MOV	g_m(g), X5
   506  	MOV	m_curg(X5), X5
   507  	MOV	(g_stack+stack_hi)(X5), X10 // return value in X10
   508  
   509  	MOV	savedX27-8(SP), g
   510  	RET
   511  
   512  // func goexit(neverCallThisFunction)
   513  // The top-most function running on a goroutine
   514  // returns to goexit+PCQuantum.
   515  TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
   516  	MOV	ZERO, ZERO	// NOP
   517  	JMP	runtime·goexit1(SB)	// does not return
   518  	// traceback from goexit1 must hit code range of goexit
   519  	MOV	ZERO, ZERO	// NOP
   520  
   521  // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
   522  // See cgocall.go for more details.
   523  TEXT ·cgocallback(SB),NOSPLIT,$24-24
   524  	NO_LOCAL_POINTERS
   525  
   526  	// Load m and g from thread-local storage.
   527  	MOVBU	runtime·iscgo(SB), X5
   528  	BEQ	ZERO, X5, nocgo
   529  	CALL	runtime·load_g(SB)
   530  nocgo:
   531  
   532  	// If g is nil, Go did not create the current thread.
   533  	// Call needm to obtain one for temporary use.
   534  	// In this case, we're running on the thread stack, so there's
   535  	// lots of space, but the linker doesn't know. Hide the call from
   536  	// the linker analysis by using an indirect call.
   537  	BEQ	ZERO, g, needm
   538  
   539  	MOV	g_m(g), X5
   540  	MOV	X5, savedm-8(SP)
   541  	JMP	havem
   542  
   543  needm:
   544  	MOV	g, savedm-8(SP) // g is zero, so is m.
   545  	MOV	$runtime·needm(SB), X6
   546  	JALR	RA, X6
   547  
   548  	// Set m->sched.sp = SP, so that if a panic happens
   549  	// during the function we are about to execute, it will
   550  	// have a valid SP to run on the g0 stack.
   551  	// The next few lines (after the havem label)
   552  	// will save this SP onto the stack and then write
   553  	// the same SP back to m->sched.sp. That seems redundant,
   554  	// but if an unrecovered panic happens, unwindm will
   555  	// restore the g->sched.sp from the stack location
   556  	// and then systemstack will try to use it. If we don't set it here,
   557  	// that restored SP will be uninitialized (typically 0) and
   558  	// will not be usable.
   559  	MOV	g_m(g), X5
   560  	MOV	m_g0(X5), X6
   561  	MOV	X2, (g_sched+gobuf_sp)(X6)
   562  
   563  havem:
   564  	// Now there's a valid m, and we're running on its m->g0.
   565  	// Save current m->g0->sched.sp on stack and then set it to SP.
   566  	// Save current sp in m->g0->sched.sp in preparation for
   567  	// switch back to m->curg stack.
   568  	// NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
   569  	MOV	m_g0(X5), X6
   570  	MOV	(g_sched+gobuf_sp)(X6), X7
   571  	MOV	X7, savedsp-24(SP)	// must match frame size
   572  	MOV	X2, (g_sched+gobuf_sp)(X6)
   573  
   574  	// Switch to m->curg stack and call runtime.cgocallbackg.
   575  	// Because we are taking over the execution of m->curg
   576  	// but *not* resuming what had been running, we need to
   577  	// save that information (m->curg->sched) so we can restore it.
   578  	// We can restore m->curg->sched.sp easily, because calling
   579  	// runtime.cgocallbackg leaves SP unchanged upon return.
   580  	// To save m->curg->sched.pc, we push it onto the curg stack and
   581  	// open a frame the same size as cgocallback's g0 frame.
   582  	// Once we switch to the curg stack, the pushed PC will appear
   583  	// to be the return PC of cgocallback, so that the traceback
   584  	// will seamlessly trace back into the earlier calls.
   585  	MOV	m_curg(X5), g
   586  	CALL	runtime·save_g(SB)
   587  	MOV	(g_sched+gobuf_sp)(g), X6 // prepare stack as X6
   588  	MOV	(g_sched+gobuf_pc)(g), X7
   589  	MOV	X7, -(24+8)(X6)		// "saved LR"; must match frame size
   590  	// Gather our arguments into registers.
   591  	MOV	fn+0(FP), X7
   592  	MOV	frame+8(FP), X8
   593  	MOV	ctxt+16(FP), X9
   594  	MOV	$-(24+8)(X6), X2	// switch stack; must match frame size
   595  	MOV	X7, 8(X2)
   596  	MOV	X8, 16(X2)
   597  	MOV	X9, 24(X2)
   598  	CALL	runtime·cgocallbackg(SB)
   599  
   600  	// Restore g->sched (== m->curg->sched) from saved values.
   601  	MOV	0(X2), X7
   602  	MOV	X7, (g_sched+gobuf_pc)(g)
   603  	MOV	$(24+8)(X2), X6		// must match frame size
   604  	MOV	X6, (g_sched+gobuf_sp)(g)
   605  
   606  	// Switch back to m->g0's stack and restore m->g0->sched.sp.
   607  	// (Unlike m->curg, the g0 goroutine never uses sched.pc,
   608  	// so we do not have to restore it.)
   609  	MOV	g_m(g), X5
   610  	MOV	m_g0(X5), g
   611  	CALL	runtime·save_g(SB)
   612  	MOV	(g_sched+gobuf_sp)(g), X2
   613  	MOV	savedsp-24(SP), X6	// must match frame size
   614  	MOV	X6, (g_sched+gobuf_sp)(g)
   615  
   616  	// If the m on entry was nil, we called needm above to borrow an m
   617  	// for the duration of the call. Since the call is over, return it with dropm.
   618  	MOV	savedm-8(SP), X5
   619  	BNE	ZERO, X5, droppedm
   620  	MOV	$runtime·dropm(SB), X6
   621  	JALR	RA, X6
   622  droppedm:
   623  
   624  	// Done!
   625  	RET
   626  
   627  TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
   628  	EBREAK
   629  	RET
   630  
   631  TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
   632  	EBREAK
   633  	RET
   634  
   635  // void setg(G*); set g. for use by needm.
   636  TEXT runtime·setg(SB), NOSPLIT, $0-8
   637  	MOV	gg+0(FP), g
   638  	// This only happens if iscgo, so jump straight to save_g
   639  	CALL	runtime·save_g(SB)
   640  	RET
   641  
   642  TEXT ·checkASM(SB),NOSPLIT,$0-1
   643  	MOV	$1, T0
   644  	MOV	T0, ret+0(FP)
   645  	RET
   646  
   647  #ifdef GOEXPERIMENT_regabiargs
   648  // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
   649  TEXT ·spillArgs(SB),NOSPLIT,$0-0
   650  	MOV	X10, (0*8)(X25)
   651  	MOV	X11, (1*8)(X25)
   652  	MOV	X12, (2*8)(X25)
   653  	MOV	X13, (3*8)(X25)
   654  	MOV	X14, (4*8)(X25)
   655  	MOV	X15, (5*8)(X25)
   656  	MOV	X16, (6*8)(X25)
   657  	MOV	X17, (7*8)(X25)
   658  	MOV	X8,  (8*8)(X25)
   659  	MOV	X9,  (9*8)(X25)
   660  	MOV	X18, (10*8)(X25)
   661  	MOV	X19, (11*8)(X25)
   662  	MOV	X20, (12*8)(X25)
   663  	MOV	X21, (13*8)(X25)
   664  	MOV	X22, (14*8)(X25)
   665  	MOV	X23, (15*8)(X25)
   666  	MOVD	F10, (16*8)(X25)
   667  	MOVD	F11, (17*8)(X25)
   668  	MOVD	F12, (18*8)(X25)
   669  	MOVD	F13, (19*8)(X25)
   670  	MOVD	F14, (20*8)(X25)
   671  	MOVD	F15, (21*8)(X25)
   672  	MOVD	F16, (22*8)(X25)
   673  	MOVD	F17, (23*8)(X25)
   674  	MOVD	F8,  (24*8)(X25)
   675  	MOVD	F9,  (25*8)(X25)
   676  	MOVD	F18, (26*8)(X25)
   677  	MOVD	F19, (27*8)(X25)
   678  	MOVD	F20, (28*8)(X25)
   679  	MOVD	F21, (29*8)(X25)
   680  	MOVD	F22, (30*8)(X25)
   681  	MOVD	F23, (31*8)(X25)
   682  	RET
   683  
   684  // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
   685  TEXT ·unspillArgs(SB),NOSPLIT,$0-0
   686  	MOV	(0*8)(X25), X10
   687  	MOV	(1*8)(X25), X11
   688  	MOV	(2*8)(X25), X12
   689  	MOV	(3*8)(X25), X13
   690  	MOV	(4*8)(X25), X14
   691  	MOV	(5*8)(X25), X15
   692  	MOV	(6*8)(X25), X16
   693  	MOV	(7*8)(X25), X17
   694  	MOV	(8*8)(X25), X8
   695  	MOV	(9*8)(X25), X9
   696  	MOV	(10*8)(X25), X18
   697  	MOV	(11*8)(X25), X19
   698  	MOV	(12*8)(X25), X20
   699  	MOV	(13*8)(X25), X21
   700  	MOV	(14*8)(X25), X22
   701  	MOV	(15*8)(X25), X23
   702  	MOVD	(16*8)(X25), F10
   703  	MOVD	(17*8)(X25), F11
   704  	MOVD	(18*8)(X25), F12
   705  	MOVD	(19*8)(X25), F13
   706  	MOVD	(20*8)(X25), F14
   707  	MOVD	(21*8)(X25), F15
   708  	MOVD	(22*8)(X25), F16
   709  	MOVD	(23*8)(X25), F17
   710  	MOVD	(24*8)(X25), F8
   711  	MOVD	(25*8)(X25), F9
   712  	MOVD	(26*8)(X25), F18
   713  	MOVD	(27*8)(X25), F19
   714  	MOVD	(28*8)(X25), F20
   715  	MOVD	(29*8)(X25), F21
   716  	MOVD	(30*8)(X25), F22
   717  	MOVD	(31*8)(X25), F23
   718  	RET
   719  #else
   720  TEXT ·spillArgs(SB),NOSPLIT,$0-0
   721  	RET
   722  
   723  TEXT ·unspillArgs(SB),NOSPLIT,$0-0
   724  	RET
   725  #endif
   726  
   727  // gcWriteBarrier performs a heap pointer write and informs the GC.
   728  //
   729  // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
   730  // - T0 is the destination of the write
   731  // - T1 is the value being written at T0.
   732  // It clobbers R30 (the linker temp register - REG_TMP).
   733  // The act of CALLing gcWriteBarrier will clobber RA (LR).
   734  // It does not clobber any other general-purpose registers,
   735  // but may clobber others (e.g., floating point registers).
   736  TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$208
   737  	// Save the registers clobbered by the fast path.
   738  	MOV	A0, 24*8(X2)
   739  	MOV	A1, 25*8(X2)
   740  	MOV	g_m(g), A0
   741  	MOV	m_p(A0), A0
   742  	MOV	(p_wbBuf+wbBuf_next)(A0), A1
   743  	// Increment wbBuf.next position.
   744  	ADD	$16, A1
   745  	MOV	A1, (p_wbBuf+wbBuf_next)(A0)
   746  	MOV	(p_wbBuf+wbBuf_end)(A0), A0
   747  	MOV	A0, T6		// T6 is linker temp register (REG_TMP)
   748  	// Record the write.
   749  	MOV	T1, -16(A1)	// Record value
   750  	MOV	(T0), A0	// TODO: This turns bad writes into bad reads.
   751  	MOV	A0, -8(A1)	// Record *slot
   752  	// Is the buffer full?
   753  	BEQ	A1, T6, flush
   754  ret:
   755  	MOV	24*8(X2), A0
   756  	MOV	25*8(X2), A1
   757  	// Do the write.
   758  	MOV	T1, (T0)
   759  	RET
   760  
   761  flush:
   762  	// Save all general purpose registers since these could be
   763  	// clobbered by wbBufFlush and were not saved by the caller.
   764  	MOV	T0, 1*8(X2)	// Also first argument to wbBufFlush
   765  	MOV	T1, 2*8(X2)	// Also second argument to wbBufFlush
   766  	// X0 is zero register
   767  	// X1 is LR, saved by prologue
   768  	// X2 is SP
   769  	// X3 is GP
   770  	// X4 is TP
   771  	// X5 is first arg to wbBufFlush (T0)
   772  	// X6 is second arg to wbBufFlush (T1)
   773  	MOV	X7, 3*8(X2)
   774  	MOV	X8, 4*8(X2)
   775  	MOV	X9, 5*8(X2)
   776  	// X10 already saved (A0)
   777  	// X11 already saved (A1)
   778  	MOV	X12, 6*8(X2)
   779  	MOV	X13, 7*8(X2)
   780  	MOV	X14, 8*8(X2)
   781  	MOV	X15, 9*8(X2)
   782  	MOV	X16, 10*8(X2)
   783  	MOV	X17, 11*8(X2)
   784  	MOV	X18, 12*8(X2)
   785  	MOV	X19, 13*8(X2)
   786  	MOV	X20, 14*8(X2)
   787  	MOV	X21, 15*8(X2)
   788  	MOV	X22, 16*8(X2)
   789  	MOV	X23, 17*8(X2)
   790  	MOV	X24, 18*8(X2)
   791  	MOV	X25, 19*8(X2)
   792  	MOV	X26, 20*8(X2)
   793  	// X27 is g.
   794  	MOV	X28, 21*8(X2)
   795  	MOV	X29, 22*8(X2)
   796  	MOV	X30, 23*8(X2)
   797  	// X31 is tmp register.
   798  
   799  	// This takes arguments T0 and T1.
   800  	CALL	runtime·wbBufFlush(SB)
   801  
   802  	MOV	1*8(X2), T0
   803  	MOV	2*8(X2), T1
   804  	MOV	3*8(X2), X7
   805  	MOV	4*8(X2), X8
   806  	MOV	5*8(X2), X9
   807  	MOV	6*8(X2), X12
   808  	MOV	7*8(X2), X13
   809  	MOV	8*8(X2), X14
   810  	MOV	9*8(X2), X15
   811  	MOV	10*8(X2), X16
   812  	MOV	11*8(X2), X17
   813  	MOV	12*8(X2), X18
   814  	MOV	13*8(X2), X19
   815  	MOV	14*8(X2), X20
   816  	MOV	15*8(X2), X21
   817  	MOV	16*8(X2), X22
   818  	MOV	17*8(X2), X23
   819  	MOV	18*8(X2), X24
   820  	MOV	19*8(X2), X25
   821  	MOV	20*8(X2), X26
   822  	MOV	21*8(X2), X28
   823  	MOV	22*8(X2), X29
   824  	MOV	23*8(X2), X30
   825  
   826  	JMP	ret
   827  
   828  // Note: these functions use a special calling convention to save generated code space.
   829  // Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those
   830  // arguments are allocated in the caller's stack frame.
   831  // These stubs write the args into that stack space and then tail call to the
   832  // corresponding runtime handler.
   833  // The tail call makes these stubs disappear in backtraces.
   834  TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
   835  #ifdef GOEXPERIMENT_regabiargs
   836  	MOV	T0, X10
   837  	MOV	T1, X11
   838  #else
   839  	MOV	T0, x+0(FP)
   840  	MOV	T1, y+8(FP)
   841  #endif
   842  	JMP	runtime·goPanicIndex<ABIInternal>(SB)
   843  TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
   844  #ifdef GOEXPERIMENT_regabiargs
   845  	MOV	T0, X10
   846  	MOV	T1, X11
   847  #else
   848  	MOV	T0, x+0(FP)
   849  	MOV	T1, y+8(FP)
   850  #endif
   851  	JMP	runtime·goPanicIndexU<ABIInternal>(SB)
   852  TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
   853  #ifdef GOEXPERIMENT_regabiargs
   854  	MOV	T1, X10
   855  	MOV	T2, X11
   856  #else
   857  	MOV	T1, x+0(FP)
   858  	MOV	T2, y+8(FP)
   859  #endif
   860  	JMP	runtime·goPanicSliceAlen<ABIInternal>(SB)
   861  TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
   862  #ifdef GOEXPERIMENT_regabiargs
   863  	MOV	T1, X10
   864  	MOV	T2, X11
   865  #else
   866  	MOV	T1, x+0(FP)
   867  	MOV	T2, y+8(FP)
   868  #endif
   869  	JMP	runtime·goPanicSliceAlenU<ABIInternal>(SB)
   870  TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
   871  #ifdef GOEXPERIMENT_regabiargs
   872  	MOV	T1, X10
   873  	MOV	T2, X11
   874  #else
   875  	MOV	T1, x+0(FP)
   876  	MOV	T2, y+8(FP)
   877  #endif
   878  	JMP	runtime·goPanicSliceAcap<ABIInternal>(SB)
   879  TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
   880  #ifdef GOEXPERIMENT_regabiargs
   881  	MOV	T1, X10
   882  	MOV	T2, X11
   883  #else
   884  	MOV	T1, x+0(FP)
   885  	MOV	T2, y+8(FP)
   886  #endif
   887  	JMP	runtime·goPanicSliceAcapU<ABIInternal>(SB)
   888  TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
   889  #ifdef GOEXPERIMENT_regabiargs
   890  	MOV	T0, X10
   891  	MOV	T1, X11
   892  #else
   893  	MOV	T0, x+0(FP)
   894  	MOV	T1, y+8(FP)
   895  #endif
   896  	JMP	runtime·goPanicSliceB<ABIInternal>(SB)
   897  TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
   898  #ifdef GOEXPERIMENT_regabiargs
   899  	MOV	T0, X10
   900  	MOV	T1, X11
   901  #else
   902  	MOV	T0, x+0(FP)
   903  	MOV	T1, y+8(FP)
   904  #endif
   905  	JMP	runtime·goPanicSliceBU<ABIInternal>(SB)
   906  TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
   907  #ifdef GOEXPERIMENT_regabiargs
   908  	MOV	T2, X10
   909  	MOV	T3, X11
   910  #else
   911  	MOV	T2, x+0(FP)
   912  	MOV	T3, y+8(FP)
   913  #endif
   914  	JMP	runtime·goPanicSlice3Alen<ABIInternal>(SB)
   915  TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
   916  #ifdef GOEXPERIMENT_regabiargs
   917  	MOV	T2, X10
   918  	MOV	T3, X11
   919  #else
   920  	MOV	T2, x+0(FP)
   921  	MOV	T3, y+8(FP)
   922  #endif
   923  	JMP	runtime·goPanicSlice3AlenU<ABIInternal>(SB)
   924  TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
   925  #ifdef GOEXPERIMENT_regabiargs
   926  	MOV	T2, X10
   927  	MOV	T3, X11
   928  #else
   929  	MOV	T2, x+0(FP)
   930  	MOV	T3, y+8(FP)
   931  #endif
   932  	JMP	runtime·goPanicSlice3Acap<ABIInternal>(SB)
   933  TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
   934  #ifdef GOEXPERIMENT_regabiargs
   935  	MOV	T2, X10
   936  	MOV	T3, X11
   937  #else
   938  	MOV	T2, x+0(FP)
   939  	MOV	T3, y+8(FP)
   940  #endif
   941  	JMP	runtime·goPanicSlice3AcapU<ABIInternal>(SB)
   942  TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
   943  #ifdef GOEXPERIMENT_regabiargs
   944  	MOV	T1, X10
   945  	MOV	T2, X11
   946  #else
   947  	MOV	T1, x+0(FP)
   948  	MOV	T2, y+8(FP)
   949  #endif
   950  	JMP	runtime·goPanicSlice3B<ABIInternal>(SB)
   951  TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
   952  #ifdef GOEXPERIMENT_regabiargs
   953  	MOV	T1, X10
   954  	MOV	T2, X11
   955  #else
   956  	MOV	T1, x+0(FP)
   957  	MOV	T2, y+8(FP)
   958  #endif
   959  	JMP	runtime·goPanicSlice3BU<ABIInternal>(SB)
   960  TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
   961  #ifdef GOEXPERIMENT_regabiargs
   962  	MOV	T0, X10
   963  	MOV	T1, X11
   964  #else
   965  	MOV	T0, x+0(FP)
   966  	MOV	T1, y+8(FP)
   967  #endif
   968  	JMP	runtime·goPanicSlice3C<ABIInternal>(SB)
   969  TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
   970  #ifdef GOEXPERIMENT_regabiargs
   971  	MOV	T0, X10
   972  	MOV	T1, X11
   973  #else
   974  	MOV	T0, x+0(FP)
   975  	MOV	T1, y+8(FP)
   976  #endif
   977  	JMP	runtime·goPanicSlice3CU<ABIInternal>(SB)
   978  TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
   979  #ifdef GOEXPERIMENT_regabiargs
   980  	MOV	T2, X10
   981  	MOV	T3, X11
   982  #else
   983  	MOV	T2, x+0(FP)
   984  	MOV	T3, y+8(FP)
   985  #endif
   986  	JMP	runtime·goPanicSliceConvert<ABIInternal>(SB)
   987  
   988  DATA	runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
   989  GLOBL	runtime·mainPC(SB),RODATA,$8
   990  

View as plain text