Text file
src/runtime/asm_riscv64.s
1 // Copyright 2017 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "funcdata.h"
7 #include "textflag.h"
8
9 // func rt0_go()
10 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
11 // X2 = stack; A0 = argc; A1 = argv
12 ADD $-24, X2
13 MOV A0, 8(X2) // argc
14 MOV A1, 16(X2) // argv
15
16 // create istack out of the given (operating system) stack.
17 // _cgo_init may update stackguard.
18 MOV $runtime·g0(SB), g
19 MOV $(-64*1024), T0
20 ADD T0, X2, T1
21 MOV T1, g_stackguard0(g)
22 MOV T1, g_stackguard1(g)
23 MOV T1, (g_stack+stack_lo)(g)
24 MOV X2, (g_stack+stack_hi)(g)
25
26 // if there is a _cgo_init, call it using the gcc ABI.
27 MOV _cgo_init(SB), T0
28 BEQ T0, ZERO, nocgo
29
30 MOV ZERO, A3 // arg 3: not used
31 MOV ZERO, A2 // arg 2: not used
32 MOV $setg_gcc<>(SB), A1 // arg 1: setg
33 MOV g, A0 // arg 0: G
34 JALR RA, T0
35
36 nocgo:
37 // update stackguard after _cgo_init
38 MOV (g_stack+stack_lo)(g), T0
39 ADD $const__StackGuard, T0
40 MOV T0, g_stackguard0(g)
41 MOV T0, g_stackguard1(g)
42
43 // set the per-goroutine and per-mach "registers"
44 MOV $runtime·m0(SB), T0
45
46 // save m->g0 = g0
47 MOV g, m_g0(T0)
48 // save m0 to g0->m
49 MOV T0, g_m(g)
50
51 CALL runtime·check(SB)
52
53 // args are already prepared
54 CALL runtime·args(SB)
55 CALL runtime·osinit(SB)
56 CALL runtime·schedinit(SB)
57
58 // create a new goroutine to start program
59 MOV $runtime·mainPC(SB), T0 // entry
60 ADD $-16, X2
61 MOV T0, 8(X2)
62 MOV ZERO, 0(X2)
63 CALL runtime·newproc(SB)
64 ADD $16, X2
65
66 // start this M
67 CALL runtime·mstart(SB)
68
69 WORD $0 // crash if reached
70 RET
71
72 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
73 CALL runtime·mstart0(SB)
74 RET // not reached
75
76 // void setg_gcc(G*); set g called from gcc with g in A0
77 TEXT setg_gcc<>(SB),NOSPLIT,$0-0
78 MOV A0, g
79 CALL runtime·save_g(SB)
80 RET
81
82 // func cputicks() int64
83 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
84 // RDTIME to emulate cpu ticks
85 // RDCYCLE reads counter that is per HART(core) based
86 // according to the riscv manual, see issue 46737
87 RDTIME A0
88 MOV A0, ret+0(FP)
89 RET
90
91 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
92 // of the G stack. We need to distinguish the routine that
93 // lives at the bottom of the G stack from the one that lives
94 // at the top of the system stack because the one at the top of
95 // the system stack terminates the stack walk (see topofstack()).
96 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
97 UNDEF
98 JALR RA, ZERO // make sure this function is not leaf
99 RET
100
101 // func systemstack(fn func())
102 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
103 MOV fn+0(FP), CTXT // CTXT = fn
104 MOV g_m(g), T0 // T0 = m
105
106 MOV m_gsignal(T0), T1 // T1 = gsignal
107 BEQ g, T1, noswitch
108
109 MOV m_g0(T0), T1 // T1 = g0
110 BEQ g, T1, noswitch
111
112 MOV m_curg(T0), T2
113 BEQ g, T2, switch
114
115 // Bad: g is not gsignal, not g0, not curg. What is it?
116 // Hide call from linker nosplit analysis.
117 MOV $runtime·badsystemstack(SB), T1
118 JALR RA, T1
119
120 switch:
121 // save our state in g->sched. Pretend to
122 // be systemstack_switch if the G stack is scanned.
123 CALL gosave_systemstack_switch<>(SB)
124
125 // switch to g0
126 MOV T1, g
127 CALL runtime·save_g(SB)
128 MOV (g_sched+gobuf_sp)(g), T0
129 MOV T0, X2
130
131 // call target function
132 MOV 0(CTXT), T1 // code pointer
133 JALR RA, T1
134
135 // switch back to g
136 MOV g_m(g), T0
137 MOV m_curg(T0), g
138 CALL runtime·save_g(SB)
139 MOV (g_sched+gobuf_sp)(g), X2
140 MOV ZERO, (g_sched+gobuf_sp)(g)
141 RET
142
143 noswitch:
144 // already on m stack, just call directly
145 // Using a tail call here cleans up tracebacks since we won't stop
146 // at an intermediate systemstack.
147 MOV 0(CTXT), T1 // code pointer
148 ADD $8, X2
149 JMP (T1)
150
151 TEXT runtime·getcallerpc(SB),NOSPLIT|NOFRAME,$0-8
152 MOV 0(X2), T0 // LR saved by caller
153 MOV T0, ret+0(FP)
154 RET
155
156 /*
157 * support for morestack
158 */
159
160 // Called during function prolog when more stack is needed.
161 // Called with return address (i.e. caller's PC) in X5 (aka T0),
162 // and the LR register contains the caller's LR.
163 //
164 // The traceback routines see morestack on a g0 as being
165 // the top of a stack (for example, morestack calling newstack
166 // calling the scheduler calling newm calling gc), so we must
167 // record an argument size. For that purpose, it has no arguments.
168
169 // func morestack()
170 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
171 // Cannot grow scheduler stack (m->g0).
172 MOV g_m(g), A0
173 MOV m_g0(A0), A1
174 BNE g, A1, 3(PC)
175 CALL runtime·badmorestackg0(SB)
176 CALL runtime·abort(SB)
177
178 // Cannot grow signal stack (m->gsignal).
179 MOV m_gsignal(A0), A1
180 BNE g, A1, 3(PC)
181 CALL runtime·badmorestackgsignal(SB)
182 CALL runtime·abort(SB)
183
184 // Called from f.
185 // Set g->sched to context in f.
186 MOV X2, (g_sched+gobuf_sp)(g)
187 MOV T0, (g_sched+gobuf_pc)(g)
188 MOV RA, (g_sched+gobuf_lr)(g)
189 MOV CTXT, (g_sched+gobuf_ctxt)(g)
190
191 // Called from f.
192 // Set m->morebuf to f's caller.
193 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC
194 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP
195 MOV g, (m_morebuf+gobuf_g)(A0)
196
197 // Call newstack on m->g0's stack.
198 MOV m_g0(A0), g
199 CALL runtime·save_g(SB)
200 MOV (g_sched+gobuf_sp)(g), X2
201 // Create a stack frame on g0 to call newstack.
202 MOV ZERO, -8(X2) // Zero saved LR in frame
203 ADD $-8, X2
204 CALL runtime·newstack(SB)
205
206 // Not reached, but make sure the return PC from the call to newstack
207 // is still in this function, and not the beginning of the next.
208 UNDEF
209
210 // func morestack_noctxt()
211 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
212 // Force SPWRITE. This function doesn't actually write SP,
213 // but it is called with a special calling convention where
214 // the caller doesn't save LR on stack but passes it as a
215 // register, and the unwinder currently doesn't understand.
216 // Make it SPWRITE to stop unwinding. (See issue 54332)
217 MOV X2, X2
218
219 MOV ZERO, CTXT
220 JMP runtime·morestack(SB)
221
222 // AES hashing not implemented for riscv64
223 TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32
224 JMP runtime·memhashFallback<ABIInternal>(SB)
225 TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
226 JMP runtime·strhashFallback<ABIInternal>(SB)
227 TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
228 JMP runtime·memhash32Fallback<ABIInternal>(SB)
229 TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24
230 JMP runtime·memhash64Fallback<ABIInternal>(SB)
231
232 // func return0()
233 TEXT runtime·return0(SB), NOSPLIT, $0
234 MOV $0, A0
235 RET
236
237 // restore state from Gobuf; longjmp
238
239 // func gogo(buf *gobuf)
240 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
241 MOV buf+0(FP), T0
242 MOV gobuf_g(T0), T1
243 MOV 0(T1), ZERO // make sure g != nil
244 JMP gogo<>(SB)
245
246 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
247 MOV T1, g
248 CALL runtime·save_g(SB)
249
250 MOV gobuf_sp(T0), X2
251 MOV gobuf_lr(T0), RA
252 MOV gobuf_ret(T0), A0
253 MOV gobuf_ctxt(T0), CTXT
254 MOV ZERO, gobuf_sp(T0)
255 MOV ZERO, gobuf_ret(T0)
256 MOV ZERO, gobuf_lr(T0)
257 MOV ZERO, gobuf_ctxt(T0)
258 MOV gobuf_pc(T0), T0
259 JALR ZERO, T0
260
261 // func procyield(cycles uint32)
262 TEXT runtime·procyield(SB),NOSPLIT,$0-0
263 RET
264
265 // Switch to m->g0's stack, call fn(g).
266 // Fn must never return. It should gogo(&g->sched)
267 // to keep running g.
268
269 // func mcall(fn func(*g))
270 TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8
271 MOV X10, CTXT
272
273 // Save caller state in g->sched
274 MOV X2, (g_sched+gobuf_sp)(g)
275 MOV RA, (g_sched+gobuf_pc)(g)
276 MOV ZERO, (g_sched+gobuf_lr)(g)
277
278 // Switch to m->g0 & its stack, call fn.
279 MOV g, X10
280 MOV g_m(g), T1
281 MOV m_g0(T1), g
282 CALL runtime·save_g(SB)
283 BNE g, X10, 2(PC)
284 JMP runtime·badmcall(SB)
285 MOV 0(CTXT), T1 // code pointer
286 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp
287 // we don't need special macro for regabi since arg0(X10) = g
288 ADD $-16, X2
289 MOV X10, 8(X2) // setup g
290 MOV ZERO, 0(X2) // clear return address
291 JALR RA, T1
292 JMP runtime·badmcall2(SB)
293
294 // Save state of caller into g->sched,
295 // but using fake PC from systemstack_switch.
296 // Must only be called from functions with no locals ($0)
297 // or else unwinding from systemstack_switch is incorrect.
298 // Smashes X31.
299 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
300 MOV $runtime·systemstack_switch(SB), X31
301 ADD $8, X31 // get past prologue
302 MOV X31, (g_sched+gobuf_pc)(g)
303 MOV X2, (g_sched+gobuf_sp)(g)
304 MOV ZERO, (g_sched+gobuf_lr)(g)
305 MOV ZERO, (g_sched+gobuf_ret)(g)
306 // Assert ctxt is zero. See func save.
307 MOV (g_sched+gobuf_ctxt)(g), X31
308 BEQ ZERO, X31, 2(PC)
309 CALL runtime·abort(SB)
310 RET
311
312 // func asmcgocall(fn, arg unsafe.Pointer) int32
313 // Call fn(arg) on the scheduler stack,
314 // aligned appropriately for the gcc ABI.
315 // See cgocall.go for more details.
316 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
317 MOV fn+0(FP), X5
318 MOV arg+8(FP), X10
319
320 MOV X2, X8 // save original stack pointer
321 MOV g, X9
322
323 // Figure out if we need to switch to m->g0 stack.
324 // We get called to create new OS threads too, and those
325 // come in on the m->g0 stack already. Or we might already
326 // be on the m->gsignal stack.
327 MOV g_m(g), X6
328 MOV m_gsignal(X6), X7
329 BEQ X7, g, g0
330 MOV m_g0(X6), X7
331 BEQ X7, g, g0
332
333 CALL gosave_systemstack_switch<>(SB)
334 MOV X7, g
335 CALL runtime·save_g(SB)
336 MOV (g_sched+gobuf_sp)(g), X2
337
338 // Now on a scheduling stack (a pthread-created stack).
339 g0:
340 // Save room for two of our pointers.
341 ADD $-16, X2
342 MOV X9, 0(X2) // save old g on stack
343 MOV (g_stack+stack_hi)(X9), X9
344 SUB X8, X9, X8
345 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
346
347 JALR RA, (X5)
348
349 // Restore g, stack pointer. X10 is return value.
350 MOV 0(X2), g
351 CALL runtime·save_g(SB)
352 MOV (g_stack+stack_hi)(g), X5
353 MOV 8(X2), X6
354 SUB X6, X5, X6
355 MOV X6, X2
356
357 MOVW X10, ret+16(FP)
358 RET
359
360 // func asminit()
361 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
362 RET
363
364 // reflectcall: call a function with the given argument list
365 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
366 // we don't have variable-sized frames, so we use a small number
367 // of constant-sized-frame functions to encode a few bits of size in the pc.
368 // Caution: ugly multiline assembly macros in your future!
369
370 #define DISPATCH(NAME,MAXSIZE) \
371 MOV $MAXSIZE, T1 \
372 BLTU T1, T0, 3(PC) \
373 MOV $NAME(SB), T2; \
374 JALR ZERO, T2
375 // Note: can't just "BR NAME(SB)" - bad inlining results.
376
377 // func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
378 TEXT reflect·call(SB), NOSPLIT, $0-0
379 JMP ·reflectcall(SB)
380
381 // func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
382 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48
383 MOVWU frameSize+32(FP), T0
384 DISPATCH(runtime·call16, 16)
385 DISPATCH(runtime·call32, 32)
386 DISPATCH(runtime·call64, 64)
387 DISPATCH(runtime·call128, 128)
388 DISPATCH(runtime·call256, 256)
389 DISPATCH(runtime·call512, 512)
390 DISPATCH(runtime·call1024, 1024)
391 DISPATCH(runtime·call2048, 2048)
392 DISPATCH(runtime·call4096, 4096)
393 DISPATCH(runtime·call8192, 8192)
394 DISPATCH(runtime·call16384, 16384)
395 DISPATCH(runtime·call32768, 32768)
396 DISPATCH(runtime·call65536, 65536)
397 DISPATCH(runtime·call131072, 131072)
398 DISPATCH(runtime·call262144, 262144)
399 DISPATCH(runtime·call524288, 524288)
400 DISPATCH(runtime·call1048576, 1048576)
401 DISPATCH(runtime·call2097152, 2097152)
402 DISPATCH(runtime·call4194304, 4194304)
403 DISPATCH(runtime·call8388608, 8388608)
404 DISPATCH(runtime·call16777216, 16777216)
405 DISPATCH(runtime·call33554432, 33554432)
406 DISPATCH(runtime·call67108864, 67108864)
407 DISPATCH(runtime·call134217728, 134217728)
408 DISPATCH(runtime·call268435456, 268435456)
409 DISPATCH(runtime·call536870912, 536870912)
410 DISPATCH(runtime·call1073741824, 1073741824)
411 MOV $runtime·badreflectcall(SB), T2
412 JALR ZERO, T2
413
414 #define CALLFN(NAME,MAXSIZE) \
415 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
416 NO_LOCAL_POINTERS; \
417 /* copy arguments to stack */ \
418 MOV stackArgs+16(FP), A1; \
419 MOVWU stackArgsSize+24(FP), A2; \
420 MOV X2, A3; \
421 ADD $8, A3; \
422 ADD A3, A2; \
423 BEQ A3, A2, 6(PC); \
424 MOVBU (A1), A4; \
425 ADD $1, A1; \
426 MOVB A4, (A3); \
427 ADD $1, A3; \
428 JMP -5(PC); \
429 /* set up argument registers */ \
430 MOV regArgs+40(FP), X25; \
431 CALL ·unspillArgs(SB); \
432 /* call function */ \
433 MOV f+8(FP), CTXT; \
434 MOV (CTXT), X25; \
435 PCDATA $PCDATA_StackMapIndex, $0; \
436 JALR RA, X25; \
437 /* copy return values back */ \
438 MOV regArgs+40(FP), X25; \
439 CALL ·spillArgs(SB); \
440 MOV stackArgsType+0(FP), A5; \
441 MOV stackArgs+16(FP), A1; \
442 MOVWU stackArgsSize+24(FP), A2; \
443 MOVWU stackRetOffset+28(FP), A4; \
444 ADD $8, X2, A3; \
445 ADD A4, A3; \
446 ADD A4, A1; \
447 SUB A4, A2; \
448 CALL callRet<>(SB); \
449 RET
450
451 // callRet copies return values back at the end of call*. This is a
452 // separate function so it can allocate stack space for the arguments
453 // to reflectcallmove. It does not follow the Go ABI; it expects its
454 // arguments in registers.
455 TEXT callRet<>(SB), NOSPLIT, $40-0
456 NO_LOCAL_POINTERS
457 MOV A5, 8(X2)
458 MOV A1, 16(X2)
459 MOV A3, 24(X2)
460 MOV A2, 32(X2)
461 MOV X25, 40(X2)
462 CALL runtime·reflectcallmove(SB)
463 RET
464
465 CALLFN(·call16, 16)
466 CALLFN(·call32, 32)
467 CALLFN(·call64, 64)
468 CALLFN(·call128, 128)
469 CALLFN(·call256, 256)
470 CALLFN(·call512, 512)
471 CALLFN(·call1024, 1024)
472 CALLFN(·call2048, 2048)
473 CALLFN(·call4096, 4096)
474 CALLFN(·call8192, 8192)
475 CALLFN(·call16384, 16384)
476 CALLFN(·call32768, 32768)
477 CALLFN(·call65536, 65536)
478 CALLFN(·call131072, 131072)
479 CALLFN(·call262144, 262144)
480 CALLFN(·call524288, 524288)
481 CALLFN(·call1048576, 1048576)
482 CALLFN(·call2097152, 2097152)
483 CALLFN(·call4194304, 4194304)
484 CALLFN(·call8388608, 8388608)
485 CALLFN(·call16777216, 16777216)
486 CALLFN(·call33554432, 33554432)
487 CALLFN(·call67108864, 67108864)
488 CALLFN(·call134217728, 134217728)
489 CALLFN(·call268435456, 268435456)
490 CALLFN(·call536870912, 536870912)
491 CALLFN(·call1073741824, 1073741824)
492
493 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
494 // Must obey the gcc calling convention.
495 TEXT _cgo_topofstack(SB),NOSPLIT,$8
496 // g (X27) and REG_TMP (X31) might be clobbered by load_g.
497 // X27 is callee-save in the gcc calling convention, so save it.
498 MOV g, savedX27-8(SP)
499
500 CALL runtime·load_g(SB)
501 MOV g_m(g), X5
502 MOV m_curg(X5), X5
503 MOV (g_stack+stack_hi)(X5), X10 // return value in X10
504
505 MOV savedX27-8(SP), g
506 RET
507
508 // func goexit(neverCallThisFunction)
509 // The top-most function running on a goroutine
510 // returns to goexit+PCQuantum.
511 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
512 MOV ZERO, ZERO // NOP
513 JMP runtime·goexit1(SB) // does not return
514 // traceback from goexit1 must hit code range of goexit
515 MOV ZERO, ZERO // NOP
516
517 // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
518 // See cgocall.go for more details.
519 TEXT ·cgocallback(SB),NOSPLIT,$24-24
520 NO_LOCAL_POINTERS
521
522 // Load m and g from thread-local storage.
523 MOVBU runtime·iscgo(SB), X5
524 BEQ ZERO, X5, nocgo
525 CALL runtime·load_g(SB)
526 nocgo:
527
528 // If g is nil, Go did not create the current thread.
529 // Call needm to obtain one for temporary use.
530 // In this case, we're running on the thread stack, so there's
531 // lots of space, but the linker doesn't know. Hide the call from
532 // the linker analysis by using an indirect call.
533 BEQ ZERO, g, needm
534
535 MOV g_m(g), X5
536 MOV X5, savedm-8(SP)
537 JMP havem
538
539 needm:
540 MOV g, savedm-8(SP) // g is zero, so is m.
541 MOV $runtime·needm(SB), X6
542 JALR RA, X6
543
544 // Set m->sched.sp = SP, so that if a panic happens
545 // during the function we are about to execute, it will
546 // have a valid SP to run on the g0 stack.
547 // The next few lines (after the havem label)
548 // will save this SP onto the stack and then write
549 // the same SP back to m->sched.sp. That seems redundant,
550 // but if an unrecovered panic happens, unwindm will
551 // restore the g->sched.sp from the stack location
552 // and then systemstack will try to use it. If we don't set it here,
553 // that restored SP will be uninitialized (typically 0) and
554 // will not be usable.
555 MOV g_m(g), X5
556 MOV m_g0(X5), X6
557 MOV X2, (g_sched+gobuf_sp)(X6)
558
559 havem:
560 // Now there's a valid m, and we're running on its m->g0.
561 // Save current m->g0->sched.sp on stack and then set it to SP.
562 // Save current sp in m->g0->sched.sp in preparation for
563 // switch back to m->curg stack.
564 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP).
565 MOV m_g0(X5), X6
566 MOV (g_sched+gobuf_sp)(X6), X7
567 MOV X7, savedsp-24(SP) // must match frame size
568 MOV X2, (g_sched+gobuf_sp)(X6)
569
570 // Switch to m->curg stack and call runtime.cgocallbackg.
571 // Because we are taking over the execution of m->curg
572 // but *not* resuming what had been running, we need to
573 // save that information (m->curg->sched) so we can restore it.
574 // We can restore m->curg->sched.sp easily, because calling
575 // runtime.cgocallbackg leaves SP unchanged upon return.
576 // To save m->curg->sched.pc, we push it onto the curg stack and
577 // open a frame the same size as cgocallback's g0 frame.
578 // Once we switch to the curg stack, the pushed PC will appear
579 // to be the return PC of cgocallback, so that the traceback
580 // will seamlessly trace back into the earlier calls.
581 MOV m_curg(X5), g
582 CALL runtime·save_g(SB)
583 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6
584 MOV (g_sched+gobuf_pc)(g), X7
585 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size
586 // Gather our arguments into registers.
587 MOV fn+0(FP), X7
588 MOV frame+8(FP), X8
589 MOV ctxt+16(FP), X9
590 MOV $-(24+8)(X6), X2 // switch stack; must match frame size
591 MOV X7, 8(X2)
592 MOV X8, 16(X2)
593 MOV X9, 24(X2)
594 CALL runtime·cgocallbackg(SB)
595
596 // Restore g->sched (== m->curg->sched) from saved values.
597 MOV 0(X2), X7
598 MOV X7, (g_sched+gobuf_pc)(g)
599 MOV $(24+8)(X2), X6 // must match frame size
600 MOV X6, (g_sched+gobuf_sp)(g)
601
602 // Switch back to m->g0's stack and restore m->g0->sched.sp.
603 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
604 // so we do not have to restore it.)
605 MOV g_m(g), X5
606 MOV m_g0(X5), g
607 CALL runtime·save_g(SB)
608 MOV (g_sched+gobuf_sp)(g), X2
609 MOV savedsp-24(SP), X6 // must match frame size
610 MOV X6, (g_sched+gobuf_sp)(g)
611
612 // If the m on entry was nil, we called needm above to borrow an m
613 // for the duration of the call. Since the call is over, return it with dropm.
614 MOV savedm-8(SP), X5
615 BNE ZERO, X5, droppedm
616 MOV $runtime·dropm(SB), X6
617 JALR RA, X6
618 droppedm:
619
620 // Done!
621 RET
622
623 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
624 EBREAK
625 RET
626
627 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
628 EBREAK
629 RET
630
631 // void setg(G*); set g. for use by needm.
632 TEXT runtime·setg(SB), NOSPLIT, $0-8
633 MOV gg+0(FP), g
634 // This only happens if iscgo, so jump straight to save_g
635 CALL runtime·save_g(SB)
636 RET
637
638 TEXT ·checkASM(SB),NOSPLIT,$0-1
639 MOV $1, T0
640 MOV T0, ret+0(FP)
641 RET
642
643 // spillArgs stores return values from registers to a *internal/abi.RegArgs in X25.
644 TEXT ·spillArgs(SB),NOSPLIT,$0-0
645 MOV X10, (0*8)(X25)
646 MOV X11, (1*8)(X25)
647 MOV X12, (2*8)(X25)
648 MOV X13, (3*8)(X25)
649 MOV X14, (4*8)(X25)
650 MOV X15, (5*8)(X25)
651 MOV X16, (6*8)(X25)
652 MOV X17, (7*8)(X25)
653 MOV X8, (8*8)(X25)
654 MOV X9, (9*8)(X25)
655 MOV X18, (10*8)(X25)
656 MOV X19, (11*8)(X25)
657 MOV X20, (12*8)(X25)
658 MOV X21, (13*8)(X25)
659 MOV X22, (14*8)(X25)
660 MOV X23, (15*8)(X25)
661 MOVD F10, (16*8)(X25)
662 MOVD F11, (17*8)(X25)
663 MOVD F12, (18*8)(X25)
664 MOVD F13, (19*8)(X25)
665 MOVD F14, (20*8)(X25)
666 MOVD F15, (21*8)(X25)
667 MOVD F16, (22*8)(X25)
668 MOVD F17, (23*8)(X25)
669 MOVD F8, (24*8)(X25)
670 MOVD F9, (25*8)(X25)
671 MOVD F18, (26*8)(X25)
672 MOVD F19, (27*8)(X25)
673 MOVD F20, (28*8)(X25)
674 MOVD F21, (29*8)(X25)
675 MOVD F22, (30*8)(X25)
676 MOVD F23, (31*8)(X25)
677 RET
678
679 // unspillArgs loads args into registers from a *internal/abi.RegArgs in X25.
680 TEXT ·unspillArgs(SB),NOSPLIT,$0-0
681 MOV (0*8)(X25), X10
682 MOV (1*8)(X25), X11
683 MOV (2*8)(X25), X12
684 MOV (3*8)(X25), X13
685 MOV (4*8)(X25), X14
686 MOV (5*8)(X25), X15
687 MOV (6*8)(X25), X16
688 MOV (7*8)(X25), X17
689 MOV (8*8)(X25), X8
690 MOV (9*8)(X25), X9
691 MOV (10*8)(X25), X18
692 MOV (11*8)(X25), X19
693 MOV (12*8)(X25), X20
694 MOV (13*8)(X25), X21
695 MOV (14*8)(X25), X22
696 MOV (15*8)(X25), X23
697 MOVD (16*8)(X25), F10
698 MOVD (17*8)(X25), F11
699 MOVD (18*8)(X25), F12
700 MOVD (19*8)(X25), F13
701 MOVD (20*8)(X25), F14
702 MOVD (21*8)(X25), F15
703 MOVD (22*8)(X25), F16
704 MOVD (23*8)(X25), F17
705 MOVD (24*8)(X25), F8
706 MOVD (25*8)(X25), F9
707 MOVD (26*8)(X25), F18
708 MOVD (27*8)(X25), F19
709 MOVD (28*8)(X25), F20
710 MOVD (29*8)(X25), F21
711 MOVD (30*8)(X25), F22
712 MOVD (31*8)(X25), F23
713 RET
714
715 // gcWriteBarrier performs a heap pointer write and informs the GC.
716 //
717 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
718 // - T0 is the destination of the write
719 // - T1 is the value being written at T0.
720 // It clobbers R30 (the linker temp register - REG_TMP).
721 // The act of CALLing gcWriteBarrier will clobber RA (LR).
722 // It does not clobber any other general-purpose registers,
723 // but may clobber others (e.g., floating point registers).
724 TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$208
725 // Save the registers clobbered by the fast path.
726 MOV A0, 24*8(X2)
727 MOV A1, 25*8(X2)
728 MOV g_m(g), A0
729 MOV m_p(A0), A0
730 MOV (p_wbBuf+wbBuf_next)(A0), A1
731 // Increment wbBuf.next position.
732 ADD $16, A1
733 MOV A1, (p_wbBuf+wbBuf_next)(A0)
734 MOV (p_wbBuf+wbBuf_end)(A0), A0
735 MOV A0, T6 // T6 is linker temp register (REG_TMP)
736 // Record the write.
737 MOV T1, -16(A1) // Record value
738 MOV (T0), A0 // TODO: This turns bad writes into bad reads.
739 MOV A0, -8(A1) // Record *slot
740 // Is the buffer full?
741 BEQ A1, T6, flush
742 ret:
743 MOV 24*8(X2), A0
744 MOV 25*8(X2), A1
745 // Do the write.
746 MOV T1, (T0)
747 RET
748
749 flush:
750 // Save all general purpose registers since these could be
751 // clobbered by wbBufFlush and were not saved by the caller.
752 MOV T0, 1*8(X2) // Also first argument to wbBufFlush
753 MOV T1, 2*8(X2) // Also second argument to wbBufFlush
754 // X0 is zero register
755 // X1 is LR, saved by prologue
756 // X2 is SP
757 // X3 is GP
758 // X4 is TP
759 // X5 is first arg to wbBufFlush (T0)
760 // X6 is second arg to wbBufFlush (T1)
761 MOV X7, 3*8(X2)
762 MOV X8, 4*8(X2)
763 MOV X9, 5*8(X2)
764 // X10 already saved (A0)
765 // X11 already saved (A1)
766 MOV X12, 6*8(X2)
767 MOV X13, 7*8(X2)
768 MOV X14, 8*8(X2)
769 MOV X15, 9*8(X2)
770 MOV X16, 10*8(X2)
771 MOV X17, 11*8(X2)
772 MOV X18, 12*8(X2)
773 MOV X19, 13*8(X2)
774 MOV X20, 14*8(X2)
775 MOV X21, 15*8(X2)
776 MOV X22, 16*8(X2)
777 MOV X23, 17*8(X2)
778 MOV X24, 18*8(X2)
779 MOV X25, 19*8(X2)
780 MOV X26, 20*8(X2)
781 // X27 is g.
782 MOV X28, 21*8(X2)
783 MOV X29, 22*8(X2)
784 MOV X30, 23*8(X2)
785 // X31 is tmp register.
786
787 // This takes arguments T0 and T1.
788 CALL runtime·wbBufFlush(SB)
789
790 MOV 1*8(X2), T0
791 MOV 2*8(X2), T1
792 MOV 3*8(X2), X7
793 MOV 4*8(X2), X8
794 MOV 5*8(X2), X9
795 MOV 6*8(X2), X12
796 MOV 7*8(X2), X13
797 MOV 8*8(X2), X14
798 MOV 9*8(X2), X15
799 MOV 10*8(X2), X16
800 MOV 11*8(X2), X17
801 MOV 12*8(X2), X18
802 MOV 13*8(X2), X19
803 MOV 14*8(X2), X20
804 MOV 15*8(X2), X21
805 MOV 16*8(X2), X22
806 MOV 17*8(X2), X23
807 MOV 18*8(X2), X24
808 MOV 19*8(X2), X25
809 MOV 20*8(X2), X26
810 MOV 21*8(X2), X28
811 MOV 22*8(X2), X29
812 MOV 23*8(X2), X30
813
814 JMP ret
815
816 // Note: these functions use a special calling convention to save generated code space.
817 // Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those
818 // arguments are allocated in the caller's stack frame.
819 // These stubs write the args into that stack space and then tail call to the
820 // corresponding runtime handler.
821 // The tail call makes these stubs disappear in backtraces.
822 TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
823 MOV T0, X10
824 MOV T1, X11
825 JMP runtime·goPanicIndex<ABIInternal>(SB)
826 TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
827 MOV T0, X10
828 MOV T1, X11
829 JMP runtime·goPanicIndexU<ABIInternal>(SB)
830 TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
831 MOV T1, X10
832 MOV T2, X11
833 JMP runtime·goPanicSliceAlen<ABIInternal>(SB)
834 TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
835 MOV T1, X10
836 MOV T2, X11
837 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB)
838 TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
839 MOV T1, X10
840 MOV T2, X11
841 JMP runtime·goPanicSliceAcap<ABIInternal>(SB)
842 TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
843 MOV T1, X10
844 MOV T2, X11
845 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB)
846 TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
847 MOV T0, X10
848 MOV T1, X11
849 JMP runtime·goPanicSliceB<ABIInternal>(SB)
850 TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
851 MOV T0, X10
852 MOV T1, X11
853 JMP runtime·goPanicSliceBU<ABIInternal>(SB)
854 TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
855 MOV T2, X10
856 MOV T3, X11
857 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB)
858 TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
859 MOV T2, X10
860 MOV T3, X11
861 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB)
862 TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
863 MOV T2, X10
864 MOV T3, X11
865 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB)
866 TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
867 MOV T2, X10
868 MOV T3, X11
869 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB)
870 TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
871 MOV T1, X10
872 MOV T2, X11
873 JMP runtime·goPanicSlice3B<ABIInternal>(SB)
874 TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
875 MOV T1, X10
876 MOV T2, X11
877 JMP runtime·goPanicSlice3BU<ABIInternal>(SB)
878 TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
879 MOV T0, X10
880 MOV T1, X11
881 JMP runtime·goPanicSlice3C<ABIInternal>(SB)
882 TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
883 MOV T0, X10
884 MOV T1, X11
885 JMP runtime·goPanicSlice3CU<ABIInternal>(SB)
886 TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
887 MOV T2, X10
888 MOV T3, X11
889 JMP runtime·goPanicSliceConvert<ABIInternal>(SB)
890
891 DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
892 GLOBL runtime·mainPC(SB),RODATA,$8
893
View as plain text