Text file
src/runtime/asm_s390x.s
1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_s390x_lib is common startup code for s390x systems when
11 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
12 // arrange to invoke this function as a global constructor (for
13 // c-archive) or when the shared library is loaded (for c-shared).
14 // We expect argc and argv to be passed in the usual C ABI registers
15 // R2 and R3.
16 TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Synchronous initialization.
38 MOVD $runtime·libpreinit(SB), R1
39 BL R1
40
41 // Create a new thread to finish Go runtime initialization.
42 MOVD _cgo_sys_thread_create(SB), R1
43 CMP R1, $0
44 BEQ nocgo
45 MOVD $_rt0_s390x_lib_go(SB), R2
46 MOVD $0, R3
47 BL R1
48 BR restore
49
50 nocgo:
51 MOVD $0x800000, R1 // stacksize
52 MOVD R1, 0(R15)
53 MOVD $_rt0_s390x_lib_go(SB), R1
54 MOVD R1, 8(R15) // fn
55 MOVD $runtime·newosproc(SB), R1
56 BL R1
57
58 restore:
59 // Restore F8-F15 from our stack frame.
60 FMOVD 16(R15), F8
61 FMOVD 24(R15), F9
62 FMOVD 32(R15), F10
63 FMOVD 40(R15), F11
64 FMOVD 48(R15), F12
65 FMOVD 56(R15), F13
66 FMOVD 64(R15), F14
67 FMOVD 72(R15), F15
68 MOVD $80(R15), R15
69
70 // Restore R6-R15.
71 LMG 48(R15), R6, R15
72 RET
73
74 // _rt0_s390x_lib_go initializes the Go runtime.
75 // This is started in a separate thread by _rt0_s390x_lib.
76 TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0
77 MOVD _rt0_s390x_lib_argc<>(SB), R2
78 MOVD _rt0_s390x_lib_argv<>(SB), R3
79 MOVD $runtime·rt0_go(SB), R1
80 BR R1
81
82 DATA _rt0_s390x_lib_argc<>(SB)/8, $0
83 GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
84 DATA _rt0_s90x_lib_argv<>(SB)/8, $0
85 GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
86
87 TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
88 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
89 // C TLS base pointer in AR0:AR1
90
91 // initialize essential registers
92 XOR R0, R0
93
94 SUB $24, R15
95 MOVW R2, 8(R15) // argc
96 MOVD R3, 16(R15) // argv
97
98 // create istack out of the given (operating system) stack.
99 // _cgo_init may update stackguard.
100 MOVD $runtime·g0(SB), g
101 MOVD R15, R11
102 SUB $(64*1024), R11
103 MOVD R11, g_stackguard0(g)
104 MOVD R11, g_stackguard1(g)
105 MOVD R11, (g_stack+stack_lo)(g)
106 MOVD R15, (g_stack+stack_hi)(g)
107
108 // if there is a _cgo_init, call it using the gcc ABI.
109 MOVD _cgo_init(SB), R11
110 CMPBEQ R11, $0, nocgo
111 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
112 SLD $32, R4, R4
113 MOVW AR1, R4 // arg 2: TLS base pointer
114 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
115 MOVD g, R2 // arg 0: G
116 // C functions expect 160 bytes of space on caller stack frame
117 // and an 8-byte aligned stack pointer
118 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
119 SUB $160, R15 // reserve 160 bytes
120 MOVD $~7, R6
121 AND R6, R15 // 8-byte align
122 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
123 MOVD R9, R15 // restore stack
124 XOR R0, R0 // zero R0
125
126 nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R2
129 ADD $const_stackGuard, R2
130 MOVD R2, g_stackguard0(g)
131 MOVD R2, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R2
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R2)
138 // save m0 to g0->m
139 MOVD R2, g_m(g)
140
141 BL runtime·check(SB)
142
143 // argc/argv are already prepared on stack
144 BL runtime·args(SB)
145 BL runtime·osinit(SB)
146 BL runtime·schedinit(SB)
147
148 // create a new goroutine to start program
149 MOVD $runtime·mainPC(SB), R2 // entry
150 SUB $16, R15
151 MOVD R2, 8(R15)
152 MOVD $0, 0(R15)
153 BL runtime·newproc(SB)
154 ADD $16, R15
155
156 // start this M
157 BL runtime·mstart(SB)
158
159 MOVD $0, 1(R0)
160 RET
161
162 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
163 GLOBL runtime·mainPC(SB),RODATA,$8
164
165 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
166 BRRK
167 RET
168
169 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
170 RET
171
172 TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
173 CALL runtime·mstart0(SB)
174 RET // not reached
175
176 /*
177 * go-routine
178 */
179
180 // void gogo(Gobuf*)
181 // restore state from Gobuf; longjmp
182 TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8
183 MOVD buf+0(FP), R5
184 MOVD gobuf_g(R5), R6
185 MOVD 0(R6), R7 // make sure g != nil
186 BR gogo<>(SB)
187
188 TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0
189 MOVD R6, g
190 BL runtime·save_g(SB)
191
192 MOVD 0(g), R4
193 MOVD gobuf_sp(R5), R15
194 MOVD gobuf_lr(R5), LR
195 MOVD gobuf_ret(R5), R3
196 MOVD gobuf_ctxt(R5), R12
197 MOVD $0, gobuf_sp(R5)
198 MOVD $0, gobuf_ret(R5)
199 MOVD $0, gobuf_lr(R5)
200 MOVD $0, gobuf_ctxt(R5)
201 CMP R0, R0 // set condition codes for == test, needed by stack split
202 MOVD gobuf_pc(R5), R6
203 BR (R6)
204
205 // void mcall(fn func(*g))
206 // Switch to m->g0's stack, call fn(g).
207 // Fn must never return. It should gogo(&g->sched)
208 // to keep running g.
209 TEXT runtime·mcall(SB), NOSPLIT, $-8-8
210 // Save caller state in g->sched
211 MOVD R15, (g_sched+gobuf_sp)(g)
212 MOVD LR, (g_sched+gobuf_pc)(g)
213 MOVD $0, (g_sched+gobuf_lr)(g)
214
215 // Switch to m->g0 & its stack, call fn.
216 MOVD g, R3
217 MOVD g_m(g), R8
218 MOVD m_g0(R8), g
219 BL runtime·save_g(SB)
220 CMP g, R3
221 BNE 2(PC)
222 BR runtime·badmcall(SB)
223 MOVD fn+0(FP), R12 // context
224 MOVD 0(R12), R4 // code pointer
225 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
226 SUB $16, R15
227 MOVD R3, 8(R15)
228 MOVD $0, 0(R15)
229 BL (R4)
230 BR runtime·badmcall2(SB)
231
232 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
233 // of the G stack. We need to distinguish the routine that
234 // lives at the bottom of the G stack from the one that lives
235 // at the top of the system stack because the one at the top of
236 // the system stack terminates the stack walk (see topofstack()).
237 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
238 UNDEF
239 BL (LR) // make sure this function is not leaf
240 RET
241
242 // func systemstack(fn func())
243 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
244 MOVD fn+0(FP), R3 // R3 = fn
245 MOVD R3, R12 // context
246 MOVD g_m(g), R4 // R4 = m
247
248 MOVD m_gsignal(R4), R5 // R5 = gsignal
249 CMPBEQ g, R5, noswitch
250
251 MOVD m_g0(R4), R5 // R5 = g0
252 CMPBEQ g, R5, noswitch
253
254 MOVD m_curg(R4), R6
255 CMPBEQ g, R6, switch
256
257 // Bad: g is not gsignal, not g0, not curg. What is it?
258 // Hide call from linker nosplit analysis.
259 MOVD $runtime·badsystemstack(SB), R3
260 BL (R3)
261 BL runtime·abort(SB)
262
263 switch:
264 // save our state in g->sched. Pretend to
265 // be systemstack_switch if the G stack is scanned.
266 BL gosave_systemstack_switch<>(SB)
267
268 // switch to g0
269 MOVD R5, g
270 BL runtime·save_g(SB)
271 MOVD (g_sched+gobuf_sp)(g), R15
272
273 // call target function
274 MOVD 0(R12), R3 // code pointer
275 BL (R3)
276
277 // switch back to g
278 MOVD g_m(g), R3
279 MOVD m_curg(R3), g
280 BL runtime·save_g(SB)
281 MOVD (g_sched+gobuf_sp)(g), R15
282 MOVD $0, (g_sched+gobuf_sp)(g)
283 RET
284
285 noswitch:
286 // already on m stack, just call directly
287 // Using a tail call here cleans up tracebacks since we won't stop
288 // at an intermediate systemstack.
289 MOVD 0(R12), R3 // code pointer
290 MOVD 0(R15), LR // restore LR
291 ADD $8, R15
292 BR (R3)
293
294 /*
295 * support for morestack
296 */
297
298 // Called during function prolog when more stack is needed.
299 // Caller has already loaded:
300 // R3: framesize, R4: argsize, R5: LR
301 //
302 // The traceback routines see morestack on a g0 as being
303 // the top of a stack (for example, morestack calling newstack
304 // calling the scheduler calling newm calling gc), so we must
305 // record an argument size. For that purpose, it has no arguments.
306 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
307 // Cannot grow scheduler stack (m->g0).
308 MOVD g_m(g), R7
309 MOVD m_g0(R7), R8
310 CMPBNE g, R8, 3(PC)
311 BL runtime·badmorestackg0(SB)
312 BL runtime·abort(SB)
313
314 // Cannot grow signal stack (m->gsignal).
315 MOVD m_gsignal(R7), R8
316 CMP g, R8
317 BNE 3(PC)
318 BL runtime·badmorestackgsignal(SB)
319 BL runtime·abort(SB)
320
321 // Called from f.
322 // Set g->sched to context in f.
323 MOVD R15, (g_sched+gobuf_sp)(g)
324 MOVD LR, R8
325 MOVD R8, (g_sched+gobuf_pc)(g)
326 MOVD R5, (g_sched+gobuf_lr)(g)
327 MOVD R12, (g_sched+gobuf_ctxt)(g)
328
329 // Called from f.
330 // Set m->morebuf to f's caller.
331 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
332 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
333 MOVD g, (m_morebuf+gobuf_g)(R7)
334
335 // Call newstack on m->g0's stack.
336 MOVD m_g0(R7), g
337 BL runtime·save_g(SB)
338 MOVD (g_sched+gobuf_sp)(g), R15
339 // Create a stack frame on g0 to call newstack.
340 MOVD $0, -8(R15) // Zero saved LR in frame
341 SUB $8, R15
342 BL runtime·newstack(SB)
343
344 // Not reached, but make sure the return PC from the call to newstack
345 // is still in this function, and not the beginning of the next.
346 UNDEF
347
348 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
349 // Force SPWRITE. This function doesn't actually write SP,
350 // but it is called with a special calling convention where
351 // the caller doesn't save LR on stack but passes it as a
352 // register (R5), and the unwinder currently doesn't understand.
353 // Make it SPWRITE to stop unwinding. (See issue 54332)
354 MOVD R15, R15
355
356 MOVD $0, R12
357 BR runtime·morestack(SB)
358
359 // reflectcall: call a function with the given argument list
360 // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
361 // we don't have variable-sized frames, so we use a small number
362 // of constant-sized-frame functions to encode a few bits of size in the pc.
363 // Caution: ugly multiline assembly macros in your future!
364
365 #define DISPATCH(NAME,MAXSIZE) \
366 MOVD $MAXSIZE, R4; \
367 CMP R3, R4; \
368 BGT 3(PC); \
369 MOVD $NAME(SB), R5; \
370 BR (R5)
371 // Note: can't just "BR NAME(SB)" - bad inlining results.
372
373 TEXT ·reflectcall(SB), NOSPLIT, $-8-48
374 MOVWZ frameSize+32(FP), R3
375 DISPATCH(runtime·call16, 16)
376 DISPATCH(runtime·call32, 32)
377 DISPATCH(runtime·call64, 64)
378 DISPATCH(runtime·call128, 128)
379 DISPATCH(runtime·call256, 256)
380 DISPATCH(runtime·call512, 512)
381 DISPATCH(runtime·call1024, 1024)
382 DISPATCH(runtime·call2048, 2048)
383 DISPATCH(runtime·call4096, 4096)
384 DISPATCH(runtime·call8192, 8192)
385 DISPATCH(runtime·call16384, 16384)
386 DISPATCH(runtime·call32768, 32768)
387 DISPATCH(runtime·call65536, 65536)
388 DISPATCH(runtime·call131072, 131072)
389 DISPATCH(runtime·call262144, 262144)
390 DISPATCH(runtime·call524288, 524288)
391 DISPATCH(runtime·call1048576, 1048576)
392 DISPATCH(runtime·call2097152, 2097152)
393 DISPATCH(runtime·call4194304, 4194304)
394 DISPATCH(runtime·call8388608, 8388608)
395 DISPATCH(runtime·call16777216, 16777216)
396 DISPATCH(runtime·call33554432, 33554432)
397 DISPATCH(runtime·call67108864, 67108864)
398 DISPATCH(runtime·call134217728, 134217728)
399 DISPATCH(runtime·call268435456, 268435456)
400 DISPATCH(runtime·call536870912, 536870912)
401 DISPATCH(runtime·call1073741824, 1073741824)
402 MOVD $runtime·badreflectcall(SB), R5
403 BR (R5)
404
405 #define CALLFN(NAME,MAXSIZE) \
406 TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \
407 NO_LOCAL_POINTERS; \
408 /* copy arguments to stack */ \
409 MOVD stackArgs+16(FP), R4; \
410 MOVWZ stackArgsSize+24(FP), R5; \
411 MOVD $stack-MAXSIZE(SP), R6; \
412 loopArgs: /* copy 256 bytes at a time */ \
413 CMP R5, $256; \
414 BLT tailArgs; \
415 SUB $256, R5; \
416 MVC $256, 0(R4), 0(R6); \
417 MOVD $256(R4), R4; \
418 MOVD $256(R6), R6; \
419 BR loopArgs; \
420 tailArgs: /* copy remaining bytes */ \
421 CMP R5, $0; \
422 BEQ callFunction; \
423 SUB $1, R5; \
424 EXRL $callfnMVC<>(SB), R5; \
425 callFunction: \
426 MOVD f+8(FP), R12; \
427 MOVD (R12), R8; \
428 PCDATA $PCDATA_StackMapIndex, $0; \
429 BL (R8); \
430 /* copy return values back */ \
431 MOVD stackArgsType+0(FP), R7; \
432 MOVD stackArgs+16(FP), R6; \
433 MOVWZ stackArgsSize+24(FP), R5; \
434 MOVD $stack-MAXSIZE(SP), R4; \
435 MOVWZ stackRetOffset+28(FP), R1; \
436 ADD R1, R4; \
437 ADD R1, R6; \
438 SUB R1, R5; \
439 BL callRet<>(SB); \
440 RET
441
442 // callRet copies return values back at the end of call*. This is a
443 // separate function so it can allocate stack space for the arguments
444 // to reflectcallmove. It does not follow the Go ABI; it expects its
445 // arguments in registers.
446 TEXT callRet<>(SB), NOSPLIT, $40-0
447 MOVD R7, 8(R15)
448 MOVD R6, 16(R15)
449 MOVD R4, 24(R15)
450 MOVD R5, 32(R15)
451 MOVD $0, 40(R15)
452 BL runtime·reflectcallmove(SB)
453 RET
454
455 CALLFN(·call16, 16)
456 CALLFN(·call32, 32)
457 CALLFN(·call64, 64)
458 CALLFN(·call128, 128)
459 CALLFN(·call256, 256)
460 CALLFN(·call512, 512)
461 CALLFN(·call1024, 1024)
462 CALLFN(·call2048, 2048)
463 CALLFN(·call4096, 4096)
464 CALLFN(·call8192, 8192)
465 CALLFN(·call16384, 16384)
466 CALLFN(·call32768, 32768)
467 CALLFN(·call65536, 65536)
468 CALLFN(·call131072, 131072)
469 CALLFN(·call262144, 262144)
470 CALLFN(·call524288, 524288)
471 CALLFN(·call1048576, 1048576)
472 CALLFN(·call2097152, 2097152)
473 CALLFN(·call4194304, 4194304)
474 CALLFN(·call8388608, 8388608)
475 CALLFN(·call16777216, 16777216)
476 CALLFN(·call33554432, 33554432)
477 CALLFN(·call67108864, 67108864)
478 CALLFN(·call134217728, 134217728)
479 CALLFN(·call268435456, 268435456)
480 CALLFN(·call536870912, 536870912)
481 CALLFN(·call1073741824, 1073741824)
482
483 // Not a function: target for EXRL (execute relative long) instruction.
484 TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
485 MVC $1, 0(R4), 0(R6)
486
487 TEXT runtime·procyield(SB),NOSPLIT,$0-0
488 RET
489
490 // Save state of caller into g->sched,
491 // but using fake PC from systemstack_switch.
492 // Must only be called from functions with no locals ($0)
493 // or else unwinding from systemstack_switch is incorrect.
494 // Smashes R1.
495 TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0
496 MOVD $runtime·systemstack_switch(SB), R1
497 ADD $16, R1 // get past prologue
498 MOVD R1, (g_sched+gobuf_pc)(g)
499 MOVD R15, (g_sched+gobuf_sp)(g)
500 MOVD $0, (g_sched+gobuf_lr)(g)
501 MOVD $0, (g_sched+gobuf_ret)(g)
502 // Assert ctxt is zero. See func save.
503 MOVD (g_sched+gobuf_ctxt)(g), R1
504 CMPBEQ R1, $0, 2(PC)
505 BL runtime·abort(SB)
506 RET
507
508 // func asmcgocall(fn, arg unsafe.Pointer) int32
509 // Call fn(arg) on the scheduler stack,
510 // aligned appropriately for the gcc ABI.
511 // See cgocall.go for more details.
512 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
513 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
514 // C TLS base pointer in AR0:AR1
515 MOVD fn+0(FP), R3
516 MOVD arg+8(FP), R4
517
518 MOVD R15, R2 // save original stack pointer
519 MOVD g, R5
520
521 // Figure out if we need to switch to m->g0 stack.
522 // We get called to create new OS threads too, and those
523 // come in on the m->g0 stack already. Or we might already
524 // be on the m->gsignal stack.
525 MOVD g_m(g), R6
526 MOVD m_gsignal(R6), R7
527 CMPBEQ R7, g, g0
528 MOVD m_g0(R6), R7
529 CMPBEQ R7, g, g0
530 BL gosave_systemstack_switch<>(SB)
531 MOVD R7, g
532 BL runtime·save_g(SB)
533 MOVD (g_sched+gobuf_sp)(g), R15
534
535 // Now on a scheduling stack (a pthread-created stack).
536 g0:
537 // Save room for two of our pointers, plus 160 bytes of callee
538 // save area that lives on the caller stack.
539 SUB $176, R15
540 MOVD $~7, R6
541 AND R6, R15 // 8-byte alignment for gcc ABI
542 MOVD R5, 168(R15) // save old g on stack
543 MOVD (g_stack+stack_hi)(R5), R5
544 SUB R2, R5
545 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
546 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
547 MOVD R4, R2 // arg in R2
548 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
549
550 XOR R0, R0 // set R0 back to 0.
551 // Restore g, stack pointer.
552 MOVD 168(R15), g
553 BL runtime·save_g(SB)
554 MOVD (g_stack+stack_hi)(g), R5
555 MOVD 160(R15), R6
556 SUB R6, R5
557 MOVD R5, R15
558
559 MOVW R2, ret+16(FP)
560 RET
561
562 // cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
563 // See cgocall.go for more details.
564 TEXT ·cgocallback(SB),NOSPLIT,$24-24
565 NO_LOCAL_POINTERS
566
567 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g.
568 // It is used to dropm while thread is exiting.
569 MOVD fn+0(FP), R1
570 CMPBNE R1, $0, loadg
571 // Restore the g from frame.
572 MOVD frame+8(FP), g
573 BR dropm
574
575 loadg:
576 // Load m and g from thread-local storage.
577 MOVB runtime·iscgo(SB), R3
578 CMPBEQ R3, $0, nocgo
579 BL runtime·load_g(SB)
580
581 nocgo:
582 // If g is nil, Go did not create the current thread,
583 // or if this thread never called into Go on pthread platforms.
584 // Call needm to obtain one for temporary use.
585 // In this case, we're running on the thread stack, so there's
586 // lots of space, but the linker doesn't know. Hide the call from
587 // the linker analysis by using an indirect call.
588 CMPBEQ g, $0, needm
589
590 MOVD g_m(g), R8
591 MOVD R8, savedm-8(SP)
592 BR havem
593
594 needm:
595 MOVD g, savedm-8(SP) // g is zero, so is m.
596 MOVD $runtime·needAndBindM(SB), R3
597 BL (R3)
598
599 // Set m->sched.sp = SP, so that if a panic happens
600 // during the function we are about to execute, it will
601 // have a valid SP to run on the g0 stack.
602 // The next few lines (after the havem label)
603 // will save this SP onto the stack and then write
604 // the same SP back to m->sched.sp. That seems redundant,
605 // but if an unrecovered panic happens, unwindm will
606 // restore the g->sched.sp from the stack location
607 // and then systemstack will try to use it. If we don't set it here,
608 // that restored SP will be uninitialized (typically 0) and
609 // will not be usable.
610 MOVD g_m(g), R8
611 MOVD m_g0(R8), R3
612 MOVD R15, (g_sched+gobuf_sp)(R3)
613
614 havem:
615 // Now there's a valid m, and we're running on its m->g0.
616 // Save current m->g0->sched.sp on stack and then set it to SP.
617 // Save current sp in m->g0->sched.sp in preparation for
618 // switch back to m->curg stack.
619 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
620 MOVD m_g0(R8), R3
621 MOVD (g_sched+gobuf_sp)(R3), R4
622 MOVD R4, savedsp-24(SP) // must match frame size
623 MOVD R15, (g_sched+gobuf_sp)(R3)
624
625 // Switch to m->curg stack and call runtime.cgocallbackg.
626 // Because we are taking over the execution of m->curg
627 // but *not* resuming what had been running, we need to
628 // save that information (m->curg->sched) so we can restore it.
629 // We can restore m->curg->sched.sp easily, because calling
630 // runtime.cgocallbackg leaves SP unchanged upon return.
631 // To save m->curg->sched.pc, we push it onto the curg stack and
632 // open a frame the same size as cgocallback's g0 frame.
633 // Once we switch to the curg stack, the pushed PC will appear
634 // to be the return PC of cgocallback, so that the traceback
635 // will seamlessly trace back into the earlier calls.
636 MOVD m_curg(R8), g
637 BL runtime·save_g(SB)
638 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
639 MOVD (g_sched+gobuf_pc)(g), R5
640 MOVD R5, -(24+8)(R4) // "saved LR"; must match frame size
641 // Gather our arguments into registers.
642 MOVD fn+0(FP), R1
643 MOVD frame+8(FP), R2
644 MOVD ctxt+16(FP), R3
645 MOVD $-(24+8)(R4), R15 // switch stack; must match frame size
646 MOVD R1, 8(R15)
647 MOVD R2, 16(R15)
648 MOVD R3, 24(R15)
649 BL runtime·cgocallbackg(SB)
650
651 // Restore g->sched (== m->curg->sched) from saved values.
652 MOVD 0(R15), R5
653 MOVD R5, (g_sched+gobuf_pc)(g)
654 MOVD $(24+8)(R15), R4 // must match frame size
655 MOVD R4, (g_sched+gobuf_sp)(g)
656
657 // Switch back to m->g0's stack and restore m->g0->sched.sp.
658 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
659 // so we do not have to restore it.)
660 MOVD g_m(g), R8
661 MOVD m_g0(R8), g
662 BL runtime·save_g(SB)
663 MOVD (g_sched+gobuf_sp)(g), R15
664 MOVD savedsp-24(SP), R4 // must match frame size
665 MOVD R4, (g_sched+gobuf_sp)(g)
666
667 // If the m on entry was nil, we called needm above to borrow an m,
668 // 1. for the duration of the call on non-pthread platforms,
669 // 2. or the duration of the C thread alive on pthread platforms.
670 // If the m on entry wasn't nil,
671 // 1. the thread might be a Go thread,
672 // 2. or it wasn't the first call from a C thread on pthread platforms,
673 // since then we skip dropm to reuse the m in the first call.
674 MOVD savedm-8(SP), R6
675 CMPBNE R6, $0, droppedm
676
677 // Skip dropm to reuse it in the next call, when a pthread key has been created.
678 MOVD _cgo_pthread_key_created(SB), R6
679 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm.
680 CMPBEQ R6, $0, dropm
681 MOVD (R6), R6
682 CMPBNE R6, $0, droppedm
683
684 dropm:
685 MOVD $runtime·dropm(SB), R3
686 BL (R3)
687 droppedm:
688
689 // Done!
690 RET
691
692 // void setg(G*); set g. for use by needm.
693 TEXT runtime·setg(SB), NOSPLIT, $0-8
694 MOVD gg+0(FP), g
695 // This only happens if iscgo, so jump straight to save_g
696 BL runtime·save_g(SB)
697 RET
698
699 // void setg_gcc(G*); set g in C TLS.
700 // Must obey the gcc calling convention.
701 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
702 // The standard prologue clobbers LR (R14), which is callee-save in
703 // the C ABI, so we have to use NOFRAME and save LR ourselves.
704 MOVD LR, R1
705 // Also save g, R10, and R11 since they're callee-save in C ABI
706 MOVD R10, R3
707 MOVD g, R4
708 MOVD R11, R5
709
710 MOVD R2, g
711 BL runtime·save_g(SB)
712
713 MOVD R5, R11
714 MOVD R4, g
715 MOVD R3, R10
716 MOVD R1, LR
717 RET
718
719 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
720 MOVW (R0), R0
721 UNDEF
722
723 // int64 runtime·cputicks(void)
724 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
725 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
726 // This means that since about 1972 the msb has been set, making the
727 // result of a call to STORE CLOCK (stck) a negative number.
728 // We clear the msb to make it positive.
729 STCK ret+0(FP) // serialises before and after call
730 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
731 SLD $1, R3
732 SRD $1, R3
733 MOVD R3, ret+0(FP)
734 RET
735
736 // AES hashing not implemented for s390x
737 TEXT runtime·memhash(SB),NOSPLIT|NOFRAME,$0-32
738 JMP runtime·memhashFallback(SB)
739 TEXT runtime·strhash(SB),NOSPLIT|NOFRAME,$0-24
740 JMP runtime·strhashFallback(SB)
741 TEXT runtime·memhash32(SB),NOSPLIT|NOFRAME,$0-24
742 JMP runtime·memhash32Fallback(SB)
743 TEXT runtime·memhash64(SB),NOSPLIT|NOFRAME,$0-24
744 JMP runtime·memhash64Fallback(SB)
745
746 TEXT runtime·return0(SB), NOSPLIT, $0
747 MOVW $0, R3
748 RET
749
750 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
751 // Must obey the gcc calling convention.
752 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
753 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
754 MOVD g, R1
755 MOVD R10, R3
756 MOVD LR, R4
757 MOVD R11, R5
758
759 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
760 MOVD g_m(g), R2
761 MOVD m_curg(R2), R2
762 MOVD (g_stack+stack_hi)(R2), R2
763
764 MOVD R1, g
765 MOVD R3, R10
766 MOVD R4, LR
767 MOVD R5, R11
768 RET
769
770 // The top-most function running on a goroutine
771 // returns to goexit+PCQuantum.
772 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
773 BYTE $0x07; BYTE $0x00; // 2-byte nop
774 BL runtime·goexit1(SB) // does not return
775 // traceback from goexit1 must hit code range of goexit
776 BYTE $0x07; BYTE $0x00; // 2-byte nop
777
778 TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
779 // Stores are already ordered on s390x, so this is just a
780 // compile barrier.
781 RET
782
783 // This is called from .init_array and follows the platform, not Go, ABI.
784 // We are overly conservative. We could only save the registers we use.
785 // However, since this function is only called once per loaded module
786 // performance is unimportant.
787 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
788 // Save R6-R15 in the register save area of the calling function.
789 // Don't bother saving F8-F15 as we aren't doing any calls.
790 STMG R6, R15, 48(R15)
791
792 // append the argument (passed in R2, as per the ELF ABI) to the
793 // moduledata linked list.
794 MOVD runtime·lastmoduledatap(SB), R1
795 MOVD R2, moduledata_next(R1)
796 MOVD R2, runtime·lastmoduledatap(SB)
797
798 // Restore R6-R15.
799 LMG 48(R15), R6, R15
800 RET
801
802 TEXT ·checkASM(SB),NOSPLIT,$0-1
803 MOVB $1, ret+0(FP)
804 RET
805
806 // gcWriteBarrier informs the GC about heap pointer writes.
807 //
808 // gcWriteBarrier does NOT follow the Go ABI. It accepts the
809 // number of bytes of buffer needed in R9, and returns a pointer
810 // to the buffer space in R9.
811 // It clobbers R10 (the temp register) and R1 (used by PLT stub).
812 // It does not clobber any other general-purpose registers,
813 // but may clobber others (e.g., floating point registers).
814 TEXT gcWriteBarrier<>(SB),NOSPLIT,$96
815 // Save the registers clobbered by the fast path.
816 MOVD R4, 96(R15)
817 retry:
818 MOVD g_m(g), R1
819 MOVD m_p(R1), R1
820 // Increment wbBuf.next position.
821 MOVD R9, R4
822 ADD (p_wbBuf+wbBuf_next)(R1), R4
823 // Is the buffer full?
824 MOVD (p_wbBuf+wbBuf_end)(R1), R10
825 CMPUBGT R4, R10, flush
826 // Commit to the larger buffer.
827 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
828 // Make return value (the original next position)
829 SUB R9, R4, R9
830 // Restore registers.
831 MOVD 96(R15), R4
832 RET
833
834 flush:
835 // Save all general purpose registers since these could be
836 // clobbered by wbBufFlush and were not saved by the caller.
837 STMG R2, R3, 8(R15)
838 MOVD R0, 24(R15)
839 // R1 already saved.
840 // R4 already saved.
841 STMG R5, R12, 32(R15) // save R5 - R12
842 // R13 is g.
843 // R14 is LR.
844 // R15 is SP.
845
846 CALL runtime·wbBufFlush(SB)
847
848 LMG 8(R15), R2, R3 // restore R2 - R3
849 MOVD 24(R15), R0 // restore R0
850 LMG 32(R15), R5, R12 // restore R5 - R12
851 JMP retry
852
853 TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0
854 MOVD $8, R9
855 JMP gcWriteBarrier<>(SB)
856 TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0
857 MOVD $16, R9
858 JMP gcWriteBarrier<>(SB)
859 TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0
860 MOVD $24, R9
861 JMP gcWriteBarrier<>(SB)
862 TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0
863 MOVD $32, R9
864 JMP gcWriteBarrier<>(SB)
865 TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0
866 MOVD $40, R9
867 JMP gcWriteBarrier<>(SB)
868 TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0
869 MOVD $48, R9
870 JMP gcWriteBarrier<>(SB)
871 TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0
872 MOVD $56, R9
873 JMP gcWriteBarrier<>(SB)
874 TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0
875 MOVD $64, R9
876 JMP gcWriteBarrier<>(SB)
877
878 // Note: these functions use a special calling convention to save generated code space.
879 // Arguments are passed in registers, but the space for those arguments are allocated
880 // in the caller's stack frame. These stubs write the args into that stack space and
881 // then tail call to the corresponding runtime handler.
882 // The tail call makes these stubs disappear in backtraces.
883 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
884 MOVD R0, x+0(FP)
885 MOVD R1, y+8(FP)
886 JMP runtime·goPanicIndex(SB)
887 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
888 MOVD R0, x+0(FP)
889 MOVD R1, y+8(FP)
890 JMP runtime·goPanicIndexU(SB)
891 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
892 MOVD R1, x+0(FP)
893 MOVD R2, y+8(FP)
894 JMP runtime·goPanicSliceAlen(SB)
895 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
896 MOVD R1, x+0(FP)
897 MOVD R2, y+8(FP)
898 JMP runtime·goPanicSliceAlenU(SB)
899 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
900 MOVD R1, x+0(FP)
901 MOVD R2, y+8(FP)
902 JMP runtime·goPanicSliceAcap(SB)
903 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
904 MOVD R1, x+0(FP)
905 MOVD R2, y+8(FP)
906 JMP runtime·goPanicSliceAcapU(SB)
907 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
908 MOVD R0, x+0(FP)
909 MOVD R1, y+8(FP)
910 JMP runtime·goPanicSliceB(SB)
911 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
912 MOVD R0, x+0(FP)
913 MOVD R1, y+8(FP)
914 JMP runtime·goPanicSliceBU(SB)
915 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
916 MOVD R2, x+0(FP)
917 MOVD R3, y+8(FP)
918 JMP runtime·goPanicSlice3Alen(SB)
919 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
920 MOVD R2, x+0(FP)
921 MOVD R3, y+8(FP)
922 JMP runtime·goPanicSlice3AlenU(SB)
923 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
924 MOVD R2, x+0(FP)
925 MOVD R3, y+8(FP)
926 JMP runtime·goPanicSlice3Acap(SB)
927 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
928 MOVD R2, x+0(FP)
929 MOVD R3, y+8(FP)
930 JMP runtime·goPanicSlice3AcapU(SB)
931 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
932 MOVD R1, x+0(FP)
933 MOVD R2, y+8(FP)
934 JMP runtime·goPanicSlice3B(SB)
935 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
936 MOVD R1, x+0(FP)
937 MOVD R2, y+8(FP)
938 JMP runtime·goPanicSlice3BU(SB)
939 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
940 MOVD R0, x+0(FP)
941 MOVD R1, y+8(FP)
942 JMP runtime·goPanicSlice3C(SB)
943 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
944 MOVD R0, x+0(FP)
945 MOVD R1, y+8(FP)
946 JMP runtime·goPanicSlice3CU(SB)
947 TEXT runtime·panicSliceConvert(SB),NOSPLIT,$0-16
948 MOVD R2, x+0(FP)
949 MOVD R3, y+8(FP)
950 JMP runtime·goPanicSliceConvert(SB)
951
View as plain text