Text file src/runtime/asm_amd64.s

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  #include "go_asm.h"
     6  #include "go_tls.h"
     7  #include "funcdata.h"
     8  #include "textflag.h"
     9  #include "cgo/abi_amd64.h"
    10  
    11  // _rt0_amd64 is common startup code for most amd64 systems when using
    12  // internal linking. This is the entry point for the program from the
    13  // kernel for an ordinary -buildmode=exe program. The stack holds the
    14  // number of arguments and the C-style argv.
    15  TEXT _rt0_amd64(SB),NOSPLIT,$-8
    16  	MOVQ	0(SP), DI	// argc
    17  	LEAQ	8(SP), SI	// argv
    18  	JMP	runtime·rt0_go(SB)
    19  
    20  // main is common startup code for most amd64 systems when using
    21  // external linking. The C startup code will call the symbol "main"
    22  // passing argc and argv in the usual C ABI registers DI and SI.
    23  TEXT main(SB),NOSPLIT,$-8
    24  	JMP	runtime·rt0_go(SB)
    25  
    26  // _rt0_amd64_lib is common startup code for most amd64 systems when
    27  // using -buildmode=c-archive or -buildmode=c-shared. The linker will
    28  // arrange to invoke this function as a global constructor (for
    29  // c-archive) or when the shared library is loaded (for c-shared).
    30  // We expect argc and argv to be passed in the usual C ABI registers
    31  // DI and SI.
    32  TEXT _rt0_amd64_lib(SB),NOSPLIT,$0
    33  	// Transition from C ABI to Go ABI.
    34  	PUSH_REGS_HOST_TO_ABI0()
    35  
    36  	MOVQ	DI, _rt0_amd64_lib_argc<>(SB)
    37  	MOVQ	SI, _rt0_amd64_lib_argv<>(SB)
    38  
    39  	// Synchronous initialization.
    40  	CALL	runtime·libpreinit(SB)
    41  
    42  	// Create a new thread to finish Go runtime initialization.
    43  	MOVQ	_cgo_sys_thread_create(SB), AX
    44  	TESTQ	AX, AX
    45  	JZ	nocgo
    46  
    47  	// We're calling back to C.
    48  	// Align stack per ELF ABI requirements.
    49  	MOVQ	SP, BX  // Callee-save in C ABI
    50  	ANDQ	$~15, SP
    51  	MOVQ	$_rt0_amd64_lib_go(SB), DI
    52  	MOVQ	$0, SI
    53  	CALL	AX
    54  	MOVQ	BX, SP
    55  	JMP	restore
    56  
    57  nocgo:
    58  	ADJSP	$16
    59  	MOVQ	$0x800000, 0(SP)		// stacksize
    60  	MOVQ	$_rt0_amd64_lib_go(SB), AX
    61  	MOVQ	AX, 8(SP)			// fn
    62  	CALL	runtime·newosproc0(SB)
    63  	ADJSP	$-16
    64  
    65  restore:
    66  	POP_REGS_HOST_TO_ABI0()
    67  	RET
    68  
    69  // _rt0_amd64_lib_go initializes the Go runtime.
    70  // This is started in a separate thread by _rt0_amd64_lib.
    71  TEXT _rt0_amd64_lib_go(SB),NOSPLIT,$0
    72  	MOVQ	_rt0_amd64_lib_argc<>(SB), DI
    73  	MOVQ	_rt0_amd64_lib_argv<>(SB), SI
    74  	JMP	runtime·rt0_go(SB)
    75  
    76  DATA _rt0_amd64_lib_argc<>(SB)/8, $0
    77  GLOBL _rt0_amd64_lib_argc<>(SB),NOPTR, $8
    78  DATA _rt0_amd64_lib_argv<>(SB)/8, $0
    79  GLOBL _rt0_amd64_lib_argv<>(SB),NOPTR, $8
    80  
    81  #ifdef GOAMD64_v2
    82  DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v2 microarchitecture support.\n"
    83  #endif
    84  
    85  #ifdef GOAMD64_v3
    86  DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v3 microarchitecture support.\n"
    87  #endif
    88  
    89  #ifdef GOAMD64_v4
    90  DATA bad_cpu_msg<>+0x00(SB)/84, $"This program can only be run on AMD64 processors with v4 microarchitecture support.\n"
    91  #endif
    92  
    93  GLOBL bad_cpu_msg<>(SB), RODATA, $84
    94  
    95  // Define a list of AMD64 microarchitecture level features
    96  // https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
    97  
    98                       // SSE3     SSSE3    CMPXCHNG16 SSE4.1    SSE4.2    POPCNT
    99  #define V2_FEATURES_CX (1 << 0 | 1 << 9 | 1 << 13  | 1 << 19 | 1 << 20 | 1 << 23)
   100                           // LAHF/SAHF
   101  #define V2_EXT_FEATURES_CX (1 << 0)
   102                                        // FMA       MOVBE     OSXSAVE   AVX       F16C
   103  #define V3_FEATURES_CX (V2_FEATURES_CX | 1 << 12 | 1 << 22 | 1 << 27 | 1 << 28 | 1 << 29)
   104                                                // ABM (FOR LZNCT)
   105  #define V3_EXT_FEATURES_CX (V2_EXT_FEATURES_CX | 1 << 5)
   106                           // BMI1     AVX2     BMI2
   107  #define V3_EXT_FEATURES_BX (1 << 3 | 1 << 5 | 1 << 8)
   108                         // XMM      YMM
   109  #define V3_OS_SUPPORT_AX (1 << 1 | 1 << 2)
   110  
   111  #define V4_FEATURES_CX V3_FEATURES_CX
   112  
   113  #define V4_EXT_FEATURES_CX V3_EXT_FEATURES_CX
   114                                                // AVX512F   AVX512DQ  AVX512CD  AVX512BW  AVX512VL
   115  #define V4_EXT_FEATURES_BX (V3_EXT_FEATURES_BX | 1 << 16 | 1 << 17 | 1 << 28 | 1 << 30 | 1 << 31)
   116                                            // OPMASK   ZMM
   117  #define V4_OS_SUPPORT_AX (V3_OS_SUPPORT_AX | 1 << 5 | (1 << 6 | 1 << 7))
   118  
   119  #ifdef GOAMD64_v2
   120  #define NEED_MAX_CPUID 0x80000001
   121  #define NEED_FEATURES_CX V2_FEATURES_CX
   122  #define NEED_EXT_FEATURES_CX V2_EXT_FEATURES_CX
   123  #endif
   124  
   125  #ifdef GOAMD64_v3
   126  #define NEED_MAX_CPUID 0x80000001
   127  #define NEED_FEATURES_CX V3_FEATURES_CX
   128  #define NEED_EXT_FEATURES_CX V3_EXT_FEATURES_CX
   129  #define NEED_EXT_FEATURES_BX V3_EXT_FEATURES_BX
   130  #define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX
   131  #endif
   132  
   133  #ifdef GOAMD64_v4
   134  #define NEED_MAX_CPUID 0x80000001
   135  #define NEED_FEATURES_CX V4_FEATURES_CX
   136  #define NEED_EXT_FEATURES_CX V4_EXT_FEATURES_CX
   137  #define NEED_EXT_FEATURES_BX V4_EXT_FEATURES_BX
   138  
   139  // Darwin requires a different approach to check AVX512 support, see CL 285572.
   140  #ifdef GOOS_darwin
   141  #define NEED_OS_SUPPORT_AX V3_OS_SUPPORT_AX
   142  // These values are from:
   143  // https://github.com/apple/darwin-xnu/blob/xnu-4570.1.46/osfmk/i386/cpu_capabilities.h
   144  #define commpage64_base_address         0x00007fffffe00000
   145  #define commpage64_cpu_capabilities64   (commpage64_base_address+0x010)
   146  #define commpage64_version              (commpage64_base_address+0x01E)
   147  #define hasAVX512F                      0x0000004000000000
   148  #define hasAVX512CD                     0x0000008000000000
   149  #define hasAVX512DQ                     0x0000010000000000
   150  #define hasAVX512BW                     0x0000020000000000
   151  #define hasAVX512VL                     0x0000100000000000
   152  #define NEED_DARWIN_SUPPORT             (hasAVX512F | hasAVX512DQ | hasAVX512CD | hasAVX512BW | hasAVX512VL)
   153  #else
   154  #define NEED_OS_SUPPORT_AX V4_OS_SUPPORT_AX
   155  #endif
   156  
   157  #endif
   158  
   159  TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0
   160  	// copy arguments forward on an even stack
   161  	MOVQ	DI, AX		// argc
   162  	MOVQ	SI, BX		// argv
   163  	SUBQ	$(5*8), SP		// 3args 2auto
   164  	ANDQ	$~15, SP
   165  	MOVQ	AX, 24(SP)
   166  	MOVQ	BX, 32(SP)
   167  
   168  	// create istack out of the given (operating system) stack.
   169  	// _cgo_init may update stackguard.
   170  	MOVQ	$runtime·g0(SB), DI
   171  	LEAQ	(-64*1024+104)(SP), BX
   172  	MOVQ	BX, g_stackguard0(DI)
   173  	MOVQ	BX, g_stackguard1(DI)
   174  	MOVQ	BX, (g_stack+stack_lo)(DI)
   175  	MOVQ	SP, (g_stack+stack_hi)(DI)
   176  
   177  	// find out information about the processor we're on
   178  	MOVL	$0, AX
   179  	CPUID
   180  	CMPL	AX, $0
   181  	JE	nocpuinfo
   182  
   183  	CMPL	BX, $0x756E6547  // "Genu"
   184  	JNE	notintel
   185  	CMPL	DX, $0x49656E69  // "ineI"
   186  	JNE	notintel
   187  	CMPL	CX, $0x6C65746E  // "ntel"
   188  	JNE	notintel
   189  	MOVB	$1, runtime·isIntel(SB)
   190  
   191  notintel:
   192  	// Load EAX=1 cpuid flags
   193  	MOVL	$1, AX
   194  	CPUID
   195  	MOVL	AX, runtime·processorVersionInfo(SB)
   196  
   197  nocpuinfo:
   198  	// if there is an _cgo_init, call it.
   199  	MOVQ	_cgo_init(SB), AX
   200  	TESTQ	AX, AX
   201  	JZ	needtls
   202  	// arg 1: g0, already in DI
   203  	MOVQ	$setg_gcc<>(SB), SI // arg 2: setg_gcc
   204  #ifdef GOOS_android
   205  	MOVQ	$runtime·tls_g(SB), DX 	// arg 3: &tls_g
   206  	// arg 4: TLS base, stored in slot 0 (Android's TLS_SLOT_SELF).
   207  	// Compensate for tls_g (+16).
   208  	MOVQ	-16(TLS), CX
   209  #else
   210  	MOVQ	$0, DX	// arg 3, 4: not used when using platform's TLS
   211  	MOVQ	$0, CX
   212  #endif
   213  #ifdef GOOS_windows
   214  	// Adjust for the Win64 calling convention.
   215  	MOVQ	CX, R9 // arg 4
   216  	MOVQ	DX, R8 // arg 3
   217  	MOVQ	SI, DX // arg 2
   218  	MOVQ	DI, CX // arg 1
   219  #endif
   220  	CALL	AX
   221  
   222  	// update stackguard after _cgo_init
   223  	MOVQ	$runtime·g0(SB), CX
   224  	MOVQ	(g_stack+stack_lo)(CX), AX
   225  	ADDQ	$const__StackGuard, AX
   226  	MOVQ	AX, g_stackguard0(CX)
   227  	MOVQ	AX, g_stackguard1(CX)
   228  
   229  #ifndef GOOS_windows
   230  	JMP ok
   231  #endif
   232  needtls:
   233  #ifdef GOOS_plan9
   234  	// skip TLS setup on Plan 9
   235  	JMP ok
   236  #endif
   237  #ifdef GOOS_solaris
   238  	// skip TLS setup on Solaris
   239  	JMP ok
   240  #endif
   241  #ifdef GOOS_illumos
   242  	// skip TLS setup on illumos
   243  	JMP ok
   244  #endif
   245  #ifdef GOOS_darwin
   246  	// skip TLS setup on Darwin
   247  	JMP ok
   248  #endif
   249  #ifdef GOOS_openbsd
   250  	// skip TLS setup on OpenBSD
   251  	JMP ok
   252  #endif
   253  
   254  	LEAQ	runtime·m0+m_tls(SB), DI
   255  	CALL	runtime·settls(SB)
   256  
   257  	// store through it, to make sure it works
   258  	get_tls(BX)
   259  	MOVQ	$0x123, g(BX)
   260  	MOVQ	runtime·m0+m_tls(SB), AX
   261  	CMPQ	AX, $0x123
   262  	JEQ 2(PC)
   263  	CALL	runtime·abort(SB)
   264  ok:
   265  	// set the per-goroutine and per-mach "registers"
   266  	get_tls(BX)
   267  	LEAQ	runtime·g0(SB), CX
   268  	MOVQ	CX, g(BX)
   269  	LEAQ	runtime·m0(SB), AX
   270  
   271  	// save m->g0 = g0
   272  	MOVQ	CX, m_g0(AX)
   273  	// save m0 to g0->m
   274  	MOVQ	AX, g_m(CX)
   275  
   276  	CLD				// convention is D is always left cleared
   277  
   278  	// Check GOAMD64 reqirements
   279  	// We need to do this after setting up TLS, so that
   280  	// we can report an error if there is a failure. See issue 49586.
   281  #ifdef NEED_FEATURES_CX
   282  	MOVL	$0, AX
   283  	CPUID
   284  	CMPL	AX, $0
   285  	JE	bad_cpu
   286  	MOVL	$1, AX
   287  	CPUID
   288  	ANDL	$NEED_FEATURES_CX, CX
   289  	CMPL	CX, $NEED_FEATURES_CX
   290  	JNE	bad_cpu
   291  #endif
   292  
   293  #ifdef NEED_MAX_CPUID
   294  	MOVL	$0x80000000, AX
   295  	CPUID
   296  	CMPL	AX, $NEED_MAX_CPUID
   297  	JL	bad_cpu
   298  #endif
   299  
   300  #ifdef NEED_EXT_FEATURES_BX
   301  	MOVL	$7, AX
   302  	MOVL	$0, CX
   303  	CPUID
   304  	ANDL	$NEED_EXT_FEATURES_BX, BX
   305  	CMPL	BX, $NEED_EXT_FEATURES_BX
   306  	JNE	bad_cpu
   307  #endif
   308  
   309  #ifdef NEED_EXT_FEATURES_CX
   310  	MOVL	$0x80000001, AX
   311  	CPUID
   312  	ANDL	$NEED_EXT_FEATURES_CX, CX
   313  	CMPL	CX, $NEED_EXT_FEATURES_CX
   314  	JNE	bad_cpu
   315  #endif
   316  
   317  #ifdef NEED_OS_SUPPORT_AX
   318  	XORL    CX, CX
   319  	XGETBV
   320  	ANDL	$NEED_OS_SUPPORT_AX, AX
   321  	CMPL	AX, $NEED_OS_SUPPORT_AX
   322  	JNE	bad_cpu
   323  #endif
   324  
   325  #ifdef NEED_DARWIN_SUPPORT
   326  	MOVQ	$commpage64_version, BX
   327  	CMPW	(BX), $13  // cpu_capabilities64 undefined in versions < 13
   328  	JL	bad_cpu
   329  	MOVQ	$commpage64_cpu_capabilities64, BX
   330  	MOVQ	(BX), BX
   331  	MOVQ	$NEED_DARWIN_SUPPORT, CX
   332  	ANDQ	CX, BX
   333  	CMPQ	BX, CX
   334  	JNE	bad_cpu
   335  #endif
   336  
   337  	CALL	runtime·check(SB)
   338  
   339  	MOVL	24(SP), AX		// copy argc
   340  	MOVL	AX, 0(SP)
   341  	MOVQ	32(SP), AX		// copy argv
   342  	MOVQ	AX, 8(SP)
   343  	CALL	runtime·args(SB)
   344  	CALL	runtime·osinit(SB)
   345  	CALL	runtime·schedinit(SB)
   346  
   347  	// create a new goroutine to start program
   348  	MOVQ	$runtime·mainPC(SB), AX		// entry
   349  	PUSHQ	AX
   350  	CALL	runtime·newproc(SB)
   351  	POPQ	AX
   352  
   353  	// start this M
   354  	CALL	runtime·mstart(SB)
   355  
   356  	CALL	runtime·abort(SB)	// mstart should never return
   357  	RET
   358  
   359  bad_cpu: // show that the program requires a certain microarchitecture level.
   360  	MOVQ	$2, 0(SP)
   361  	MOVQ	$bad_cpu_msg<>(SB), AX
   362  	MOVQ	AX, 8(SP)
   363  	MOVQ	$84, 16(SP)
   364  	CALL	runtime·write(SB)
   365  	MOVQ	$1, 0(SP)
   366  	CALL	runtime·exit(SB)
   367  	CALL	runtime·abort(SB)
   368  	RET
   369  
   370  	// Prevent dead-code elimination of debugCallV2, which is
   371  	// intended to be called by debuggers.
   372  	MOVQ	$runtime·debugCallV2<ABIInternal>(SB), AX
   373  	RET
   374  
   375  // mainPC is a function value for runtime.main, to be passed to newproc.
   376  // The reference to runtime.main is made via ABIInternal, since the
   377  // actual function (not the ABI0 wrapper) is needed by newproc.
   378  DATA	runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB)
   379  GLOBL	runtime·mainPC(SB),RODATA,$8
   380  
   381  TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
   382  	BYTE	$0xcc
   383  	RET
   384  
   385  TEXT runtime·asminit(SB),NOSPLIT,$0-0
   386  	// No per-thread init.
   387  	RET
   388  
   389  TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0
   390  	CALL	runtime·mstart0(SB)
   391  	RET // not reached
   392  
   393  /*
   394   *  go-routine
   395   */
   396  
   397  // func gogo(buf *gobuf)
   398  // restore state from Gobuf; longjmp
   399  TEXT runtime·gogo(SB), NOSPLIT, $0-8
   400  	MOVQ	buf+0(FP), BX		// gobuf
   401  	MOVQ	gobuf_g(BX), DX
   402  	MOVQ	0(DX), CX		// make sure g != nil
   403  	JMP	gogo<>(SB)
   404  
   405  TEXT gogo<>(SB), NOSPLIT, $0
   406  	get_tls(CX)
   407  	MOVQ	DX, g(CX)
   408  	MOVQ	DX, R14		// set the g register
   409  	MOVQ	gobuf_sp(BX), SP	// restore SP
   410  	MOVQ	gobuf_ret(BX), AX
   411  	MOVQ	gobuf_ctxt(BX), DX
   412  	MOVQ	gobuf_bp(BX), BP
   413  	MOVQ	$0, gobuf_sp(BX)	// clear to help garbage collector
   414  	MOVQ	$0, gobuf_ret(BX)
   415  	MOVQ	$0, gobuf_ctxt(BX)
   416  	MOVQ	$0, gobuf_bp(BX)
   417  	MOVQ	gobuf_pc(BX), BX
   418  	JMP	BX
   419  
   420  // func mcall(fn func(*g))
   421  // Switch to m->g0's stack, call fn(g).
   422  // Fn must never return. It should gogo(&g->sched)
   423  // to keep running g.
   424  TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT, $0-8
   425  	MOVQ	AX, DX	// DX = fn
   426  
   427  	// save state in g->sched
   428  	MOVQ	0(SP), BX	// caller's PC
   429  	MOVQ	BX, (g_sched+gobuf_pc)(R14)
   430  	LEAQ	fn+0(FP), BX	// caller's SP
   431  	MOVQ	BX, (g_sched+gobuf_sp)(R14)
   432  	MOVQ	BP, (g_sched+gobuf_bp)(R14)
   433  
   434  	// switch to m->g0 & its stack, call fn
   435  	MOVQ	g_m(R14), BX
   436  	MOVQ	m_g0(BX), SI	// SI = g.m.g0
   437  	CMPQ	SI, R14	// if g == m->g0 call badmcall
   438  	JNE	goodm
   439  	JMP	runtime·badmcall(SB)
   440  goodm:
   441  	MOVQ	R14, AX		// AX (and arg 0) = g
   442  	MOVQ	SI, R14		// g = g.m.g0
   443  	get_tls(CX)		// Set G in TLS
   444  	MOVQ	R14, g(CX)
   445  	MOVQ	(g_sched+gobuf_sp)(R14), SP	// sp = g0.sched.sp
   446  	PUSHQ	AX	// open up space for fn's arg spill slot
   447  	MOVQ	0(DX), R12
   448  	CALL	R12		// fn(g)
   449  	POPQ	AX
   450  	JMP	runtime·badmcall2(SB)
   451  	RET
   452  
   453  // systemstack_switch is a dummy routine that systemstack leaves at the bottom
   454  // of the G stack. We need to distinguish the routine that
   455  // lives at the bottom of the G stack from the one that lives
   456  // at the top of the system stack because the one at the top of
   457  // the system stack terminates the stack walk (see topofstack()).
   458  TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
   459  	RET
   460  
   461  // func systemstack(fn func())
   462  TEXT runtime·systemstack(SB), NOSPLIT, $0-8
   463  	MOVQ	fn+0(FP), DI	// DI = fn
   464  	get_tls(CX)
   465  	MOVQ	g(CX), AX	// AX = g
   466  	MOVQ	g_m(AX), BX	// BX = m
   467  
   468  	CMPQ	AX, m_gsignal(BX)
   469  	JEQ	noswitch
   470  
   471  	MOVQ	m_g0(BX), DX	// DX = g0
   472  	CMPQ	AX, DX
   473  	JEQ	noswitch
   474  
   475  	CMPQ	AX, m_curg(BX)
   476  	JNE	bad
   477  
   478  	// switch stacks
   479  	// save our state in g->sched. Pretend to
   480  	// be systemstack_switch if the G stack is scanned.
   481  	CALL	gosave_systemstack_switch<>(SB)
   482  
   483  	// switch to g0
   484  	MOVQ	DX, g(CX)
   485  	MOVQ	DX, R14 // set the g register
   486  	MOVQ	(g_sched+gobuf_sp)(DX), BX
   487  	MOVQ	BX, SP
   488  
   489  	// call target function
   490  	MOVQ	DI, DX
   491  	MOVQ	0(DI), DI
   492  	CALL	DI
   493  
   494  	// switch back to g
   495  	get_tls(CX)
   496  	MOVQ	g(CX), AX
   497  	MOVQ	g_m(AX), BX
   498  	MOVQ	m_curg(BX), AX
   499  	MOVQ	AX, g(CX)
   500  	MOVQ	(g_sched+gobuf_sp)(AX), SP
   501  	MOVQ	$0, (g_sched+gobuf_sp)(AX)
   502  	RET
   503  
   504  noswitch:
   505  	// already on m stack; tail call the function
   506  	// Using a tail call here cleans up tracebacks since we won't stop
   507  	// at an intermediate systemstack.
   508  	MOVQ	DI, DX
   509  	MOVQ	0(DI), DI
   510  	JMP	DI
   511  
   512  bad:
   513  	// Bad: g is not gsignal, not g0, not curg. What is it?
   514  	MOVQ	$runtime·badsystemstack(SB), AX
   515  	CALL	AX
   516  	INT	$3
   517  
   518  
   519  /*
   520   * support for morestack
   521   */
   522  
   523  // Called during function prolog when more stack is needed.
   524  //
   525  // The traceback routines see morestack on a g0 as being
   526  // the top of a stack (for example, morestack calling newstack
   527  // calling the scheduler calling newm calling gc), so we must
   528  // record an argument size. For that purpose, it has no arguments.
   529  TEXT runtime·morestack(SB),NOSPLIT,$0-0
   530  	// Cannot grow scheduler stack (m->g0).
   531  	get_tls(CX)
   532  	MOVQ	g(CX), BX
   533  	MOVQ	g_m(BX), BX
   534  	MOVQ	m_g0(BX), SI
   535  	CMPQ	g(CX), SI
   536  	JNE	3(PC)
   537  	CALL	runtime·badmorestackg0(SB)
   538  	CALL	runtime·abort(SB)
   539  
   540  	// Cannot grow signal stack (m->gsignal).
   541  	MOVQ	m_gsignal(BX), SI
   542  	CMPQ	g(CX), SI
   543  	JNE	3(PC)
   544  	CALL	runtime·badmorestackgsignal(SB)
   545  	CALL	runtime·abort(SB)
   546  
   547  	// Called from f.
   548  	// Set m->morebuf to f's caller.
   549  	NOP	SP	// tell vet SP changed - stop checking offsets
   550  	MOVQ	8(SP), AX	// f's caller's PC
   551  	MOVQ	AX, (m_morebuf+gobuf_pc)(BX)
   552  	LEAQ	16(SP), AX	// f's caller's SP
   553  	MOVQ	AX, (m_morebuf+gobuf_sp)(BX)
   554  	get_tls(CX)
   555  	MOVQ	g(CX), SI
   556  	MOVQ	SI, (m_morebuf+gobuf_g)(BX)
   557  
   558  	// Set g->sched to context in f.
   559  	MOVQ	0(SP), AX // f's PC
   560  	MOVQ	AX, (g_sched+gobuf_pc)(SI)
   561  	LEAQ	8(SP), AX // f's SP
   562  	MOVQ	AX, (g_sched+gobuf_sp)(SI)
   563  	MOVQ	BP, (g_sched+gobuf_bp)(SI)
   564  	MOVQ	DX, (g_sched+gobuf_ctxt)(SI)
   565  
   566  	// Call newstack on m->g0's stack.
   567  	MOVQ	m_g0(BX), BX
   568  	MOVQ	BX, g(CX)
   569  	MOVQ	(g_sched+gobuf_sp)(BX), SP
   570  	CALL	runtime·newstack(SB)
   571  	CALL	runtime·abort(SB)	// crash if newstack returns
   572  	RET
   573  
   574  // morestack but not preserving ctxt.
   575  TEXT runtime·morestack_noctxt(SB),NOSPLIT,$0
   576  	MOVL	$0, DX
   577  	JMP	runtime·morestack(SB)
   578  
   579  // spillArgs stores return values from registers to a *internal/abi.RegArgs in R12.
   580  TEXT ·spillArgs(SB),NOSPLIT,$0-0
   581  	MOVQ AX, 0(R12)
   582  	MOVQ BX, 8(R12)
   583  	MOVQ CX, 16(R12)
   584  	MOVQ DI, 24(R12)
   585  	MOVQ SI, 32(R12)
   586  	MOVQ R8, 40(R12)
   587  	MOVQ R9, 48(R12)
   588  	MOVQ R10, 56(R12)
   589  	MOVQ R11, 64(R12)
   590  	MOVQ X0, 72(R12)
   591  	MOVQ X1, 80(R12)
   592  	MOVQ X2, 88(R12)
   593  	MOVQ X3, 96(R12)
   594  	MOVQ X4, 104(R12)
   595  	MOVQ X5, 112(R12)
   596  	MOVQ X6, 120(R12)
   597  	MOVQ X7, 128(R12)
   598  	MOVQ X8, 136(R12)
   599  	MOVQ X9, 144(R12)
   600  	MOVQ X10, 152(R12)
   601  	MOVQ X11, 160(R12)
   602  	MOVQ X12, 168(R12)
   603  	MOVQ X13, 176(R12)
   604  	MOVQ X14, 184(R12)
   605  	RET
   606  
   607  // unspillArgs loads args into registers from a *internal/abi.RegArgs in R12.
   608  TEXT ·unspillArgs(SB),NOSPLIT,$0-0
   609  	MOVQ 0(R12), AX
   610  	MOVQ 8(R12), BX
   611  	MOVQ 16(R12), CX
   612  	MOVQ 24(R12), DI
   613  	MOVQ 32(R12), SI
   614  	MOVQ 40(R12), R8
   615  	MOVQ 48(R12), R9
   616  	MOVQ 56(R12), R10
   617  	MOVQ 64(R12), R11
   618  	MOVQ 72(R12), X0
   619  	MOVQ 80(R12), X1
   620  	MOVQ 88(R12), X2
   621  	MOVQ 96(R12), X3
   622  	MOVQ 104(R12), X4
   623  	MOVQ 112(R12), X5
   624  	MOVQ 120(R12), X6
   625  	MOVQ 128(R12), X7
   626  	MOVQ 136(R12), X8
   627  	MOVQ 144(R12), X9
   628  	MOVQ 152(R12), X10
   629  	MOVQ 160(R12), X11
   630  	MOVQ 168(R12), X12
   631  	MOVQ 176(R12), X13
   632  	MOVQ 184(R12), X14
   633  	RET
   634  
   635  // reflectcall: call a function with the given argument list
   636  // func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs).
   637  // we don't have variable-sized frames, so we use a small number
   638  // of constant-sized-frame functions to encode a few bits of size in the pc.
   639  // Caution: ugly multiline assembly macros in your future!
   640  
   641  #define DISPATCH(NAME,MAXSIZE)		\
   642  	CMPQ	CX, $MAXSIZE;		\
   643  	JA	3(PC);			\
   644  	MOVQ	$NAME(SB), AX;		\
   645  	JMP	AX
   646  // Note: can't just "JMP NAME(SB)" - bad inlining results.
   647  
   648  TEXT ·reflectcall(SB), NOSPLIT, $0-48
   649  	MOVLQZX frameSize+32(FP), CX
   650  	DISPATCH(runtime·call16, 16)
   651  	DISPATCH(runtime·call32, 32)
   652  	DISPATCH(runtime·call64, 64)
   653  	DISPATCH(runtime·call128, 128)
   654  	DISPATCH(runtime·call256, 256)
   655  	DISPATCH(runtime·call512, 512)
   656  	DISPATCH(runtime·call1024, 1024)
   657  	DISPATCH(runtime·call2048, 2048)
   658  	DISPATCH(runtime·call4096, 4096)
   659  	DISPATCH(runtime·call8192, 8192)
   660  	DISPATCH(runtime·call16384, 16384)
   661  	DISPATCH(runtime·call32768, 32768)
   662  	DISPATCH(runtime·call65536, 65536)
   663  	DISPATCH(runtime·call131072, 131072)
   664  	DISPATCH(runtime·call262144, 262144)
   665  	DISPATCH(runtime·call524288, 524288)
   666  	DISPATCH(runtime·call1048576, 1048576)
   667  	DISPATCH(runtime·call2097152, 2097152)
   668  	DISPATCH(runtime·call4194304, 4194304)
   669  	DISPATCH(runtime·call8388608, 8388608)
   670  	DISPATCH(runtime·call16777216, 16777216)
   671  	DISPATCH(runtime·call33554432, 33554432)
   672  	DISPATCH(runtime·call67108864, 67108864)
   673  	DISPATCH(runtime·call134217728, 134217728)
   674  	DISPATCH(runtime·call268435456, 268435456)
   675  	DISPATCH(runtime·call536870912, 536870912)
   676  	DISPATCH(runtime·call1073741824, 1073741824)
   677  	MOVQ	$runtime·badreflectcall(SB), AX
   678  	JMP	AX
   679  
   680  #define CALLFN(NAME,MAXSIZE)			\
   681  TEXT NAME(SB), WRAPPER, $MAXSIZE-48;		\
   682  	NO_LOCAL_POINTERS;			\
   683  	/* copy arguments to stack */		\
   684  	MOVQ	stackArgs+16(FP), SI;		\
   685  	MOVLQZX stackArgsSize+24(FP), CX;		\
   686  	MOVQ	SP, DI;				\
   687  	REP;MOVSB;				\
   688  	/* set up argument registers */		\
   689  	MOVQ    regArgs+40(FP), R12;		\
   690  	CALL    ·unspillArgs(SB);		\
   691  	/* call function */			\
   692  	MOVQ	f+8(FP), DX;			\
   693  	PCDATA  $PCDATA_StackMapIndex, $0;	\
   694  	MOVQ	(DX), R12;			\
   695  	CALL	R12;				\
   696  	/* copy register return values back */		\
   697  	MOVQ    regArgs+40(FP), R12;		\
   698  	CALL    ·spillArgs(SB);		\
   699  	MOVLQZX	stackArgsSize+24(FP), CX;		\
   700  	MOVLQZX	stackRetOffset+28(FP), BX;		\
   701  	MOVQ	stackArgs+16(FP), DI;		\
   702  	MOVQ	stackArgsType+0(FP), DX;		\
   703  	MOVQ	SP, SI;				\
   704  	ADDQ	BX, DI;				\
   705  	ADDQ	BX, SI;				\
   706  	SUBQ	BX, CX;				\
   707  	CALL	callRet<>(SB);			\
   708  	RET
   709  
   710  // callRet copies return values back at the end of call*. This is a
   711  // separate function so it can allocate stack space for the arguments
   712  // to reflectcallmove. It does not follow the Go ABI; it expects its
   713  // arguments in registers.
   714  TEXT callRet<>(SB), NOSPLIT, $40-0
   715  	NO_LOCAL_POINTERS
   716  	MOVQ	DX, 0(SP)
   717  	MOVQ	DI, 8(SP)
   718  	MOVQ	SI, 16(SP)
   719  	MOVQ	CX, 24(SP)
   720  	MOVQ	R12, 32(SP)
   721  	CALL	runtime·reflectcallmove(SB)
   722  	RET
   723  
   724  CALLFN(·call16, 16)
   725  CALLFN(·call32, 32)
   726  CALLFN(·call64, 64)
   727  CALLFN(·call128, 128)
   728  CALLFN(·call256, 256)
   729  CALLFN(·call512, 512)
   730  CALLFN(·call1024, 1024)
   731  CALLFN(·call2048, 2048)
   732  CALLFN(·call4096, 4096)
   733  CALLFN(·call8192, 8192)
   734  CALLFN(·call16384, 16384)
   735  CALLFN(·call32768, 32768)
   736  CALLFN(·call65536, 65536)
   737  CALLFN(·call131072, 131072)
   738  CALLFN(·call262144, 262144)
   739  CALLFN(·call524288, 524288)
   740  CALLFN(·call1048576, 1048576)
   741  CALLFN(·call2097152, 2097152)
   742  CALLFN(·call4194304, 4194304)
   743  CALLFN(·call8388608, 8388608)
   744  CALLFN(·call16777216, 16777216)
   745  CALLFN(·call33554432, 33554432)
   746  CALLFN(·call67108864, 67108864)
   747  CALLFN(·call134217728, 134217728)
   748  CALLFN(·call268435456, 268435456)
   749  CALLFN(·call536870912, 536870912)
   750  CALLFN(·call1073741824, 1073741824)
   751  
   752  TEXT runtime·procyield(SB),NOSPLIT,$0-0
   753  	MOVL	cycles+0(FP), AX
   754  again:
   755  	PAUSE
   756  	SUBL	$1, AX
   757  	JNZ	again
   758  	RET
   759  
   760  
   761  TEXT ·publicationBarrier(SB),NOSPLIT,$0-0
   762  	// Stores are already ordered on x86, so this is just a
   763  	// compile barrier.
   764  	RET
   765  
   766  // Save state of caller into g->sched,
   767  // but using fake PC from systemstack_switch.
   768  // Must only be called from functions with no locals ($0)
   769  // or else unwinding from systemstack_switch is incorrect.
   770  // Smashes R9.
   771  TEXT gosave_systemstack_switch<>(SB),NOSPLIT,$0
   772  	MOVQ	$runtime·systemstack_switch(SB), R9
   773  	MOVQ	R9, (g_sched+gobuf_pc)(R14)
   774  	LEAQ	8(SP), R9
   775  	MOVQ	R9, (g_sched+gobuf_sp)(R14)
   776  	MOVQ	$0, (g_sched+gobuf_ret)(R14)
   777  	MOVQ	BP, (g_sched+gobuf_bp)(R14)
   778  	// Assert ctxt is zero. See func save.
   779  	MOVQ	(g_sched+gobuf_ctxt)(R14), R9
   780  	TESTQ	R9, R9
   781  	JZ	2(PC)
   782  	CALL	runtime·abort(SB)
   783  	RET
   784  
   785  // func asmcgocall_no_g(fn, arg unsafe.Pointer)
   786  // Call fn(arg) aligned appropriately for the gcc ABI.
   787  // Called on a system stack, and there may be no g yet (during needm).
   788  TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16
   789  	MOVQ	fn+0(FP), AX
   790  	MOVQ	arg+8(FP), BX
   791  	MOVQ	SP, DX
   792  	SUBQ	$32, SP
   793  	ANDQ	$~15, SP	// alignment
   794  	MOVQ	DX, 8(SP)
   795  	MOVQ	BX, DI		// DI = first argument in AMD64 ABI
   796  	MOVQ	BX, CX		// CX = first argument in Win64
   797  	CALL	AX
   798  	MOVQ	8(SP), DX
   799  	MOVQ	DX, SP
   800  	RET
   801  
   802  // func asmcgocall(fn, arg unsafe.Pointer) int32
   803  // Call fn(arg) on the scheduler stack,
   804  // aligned appropriately for the gcc ABI.
   805  // See cgocall.go for more details.
   806  TEXT ·asmcgocall(SB),NOSPLIT,$0-20
   807  	MOVQ	fn+0(FP), AX
   808  	MOVQ	arg+8(FP), BX
   809  
   810  	MOVQ	SP, DX
   811  
   812  	// Figure out if we need to switch to m->g0 stack.
   813  	// We get called to create new OS threads too, and those
   814  	// come in on the m->g0 stack already. Or we might already
   815  	// be on the m->gsignal stack.
   816  	get_tls(CX)
   817  	MOVQ	g(CX), DI
   818  	CMPQ	DI, $0
   819  	JEQ	nosave
   820  	MOVQ	g_m(DI), R8
   821  	MOVQ	m_gsignal(R8), SI
   822  	CMPQ	DI, SI
   823  	JEQ	nosave
   824  	MOVQ	m_g0(R8), SI
   825  	CMPQ	DI, SI
   826  	JEQ	nosave
   827  
   828  	// Switch to system stack.
   829  	CALL	gosave_systemstack_switch<>(SB)
   830  	MOVQ	SI, g(CX)
   831  	MOVQ	(g_sched+gobuf_sp)(SI), SP
   832  
   833  	// Now on a scheduling stack (a pthread-created stack).
   834  	// Make sure we have enough room for 4 stack-backed fast-call
   835  	// registers as per windows amd64 calling convention.
   836  	SUBQ	$64, SP
   837  	ANDQ	$~15, SP	// alignment for gcc ABI
   838  	MOVQ	DI, 48(SP)	// save g
   839  	MOVQ	(g_stack+stack_hi)(DI), DI
   840  	SUBQ	DX, DI
   841  	MOVQ	DI, 40(SP)	// save depth in stack (can't just save SP, as stack might be copied during a callback)
   842  	MOVQ	BX, DI		// DI = first argument in AMD64 ABI
   843  	MOVQ	BX, CX		// CX = first argument in Win64
   844  	CALL	AX
   845  
   846  	// Restore registers, g, stack pointer.
   847  	get_tls(CX)
   848  	MOVQ	48(SP), DI
   849  	MOVQ	(g_stack+stack_hi)(DI), SI
   850  	SUBQ	40(SP), SI
   851  	MOVQ	DI, g(CX)
   852  	MOVQ	SI, SP
   853  
   854  	MOVL	AX, ret+16(FP)
   855  	RET
   856  
   857  nosave:
   858  	// Running on a system stack, perhaps even without a g.
   859  	// Having no g can happen during thread creation or thread teardown
   860  	// (see needm/dropm on Solaris, for example).
   861  	// This code is like the above sequence but without saving/restoring g
   862  	// and without worrying about the stack moving out from under us
   863  	// (because we're on a system stack, not a goroutine stack).
   864  	// The above code could be used directly if already on a system stack,
   865  	// but then the only path through this code would be a rare case on Solaris.
   866  	// Using this code for all "already on system stack" calls exercises it more,
   867  	// which should help keep it correct.
   868  	SUBQ	$64, SP
   869  	ANDQ	$~15, SP
   870  	MOVQ	$0, 48(SP)		// where above code stores g, in case someone looks during debugging
   871  	MOVQ	DX, 40(SP)	// save original stack pointer
   872  	MOVQ	BX, DI		// DI = first argument in AMD64 ABI
   873  	MOVQ	BX, CX		// CX = first argument in Win64
   874  	CALL	AX
   875  	MOVQ	40(SP), SI	// restore original stack pointer
   876  	MOVQ	SI, SP
   877  	MOVL	AX, ret+16(FP)
   878  	RET
   879  
   880  #ifdef GOOS_windows
   881  // Dummy TLS that's used on Windows so that we don't crash trying
   882  // to restore the G register in needm. needm and its callees are
   883  // very careful never to actually use the G, the TLS just can't be
   884  // unset since we're in Go code.
   885  GLOBL zeroTLS<>(SB),RODATA,$const_tlsSize
   886  #endif
   887  
   888  // func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr)
   889  // See cgocall.go for more details.
   890  TEXT ·cgocallback(SB),NOSPLIT,$24-24
   891  	NO_LOCAL_POINTERS
   892  
   893  	// If g is nil, Go did not create the current thread.
   894  	// Call needm to obtain one m for temporary use.
   895  	// In this case, we're running on the thread stack, so there's
   896  	// lots of space, but the linker doesn't know. Hide the call from
   897  	// the linker analysis by using an indirect call through AX.
   898  	get_tls(CX)
   899  #ifdef GOOS_windows
   900  	MOVL	$0, BX
   901  	CMPQ	CX, $0
   902  	JEQ	2(PC)
   903  #endif
   904  	MOVQ	g(CX), BX
   905  	CMPQ	BX, $0
   906  	JEQ	needm
   907  	MOVQ	g_m(BX), BX
   908  	MOVQ	BX, savedm-8(SP)	// saved copy of oldm
   909  	JMP	havem
   910  needm:
   911  #ifdef GOOS_windows
   912  	// Set up a dummy TLS value. needm is careful not to use it,
   913  	// but it needs to be there to prevent autogenerated code from
   914  	// crashing when it loads from it.
   915  	// We don't need to clear it or anything later because needm
   916  	// will set up TLS properly.
   917  	MOVQ	$zeroTLS<>(SB), DI
   918  	CALL	runtime·settls(SB)
   919  #endif
   920  	// On some platforms (Windows) we cannot call needm through
   921  	// an ABI wrapper because there's no TLS set up, and the ABI
   922  	// wrapper will try to restore the G register (R14) from TLS.
   923  	// Clear X15 because Go expects it and we're not calling
   924  	// through a wrapper, but otherwise avoid setting the G
   925  	// register in the wrapper and call needm directly. It
   926  	// takes no arguments and doesn't return any values so
   927  	// there's no need to handle that. Clear R14 so that there's
   928  	// a bad value in there, in case needm tries to use it.
   929  	XORPS	X15, X15
   930  	XORQ    R14, R14
   931  	MOVQ	$runtime·needm<ABIInternal>(SB), AX
   932  	CALL	AX
   933  	MOVQ	$0, savedm-8(SP) // dropm on return
   934  	get_tls(CX)
   935  	MOVQ	g(CX), BX
   936  	MOVQ	g_m(BX), BX
   937  
   938  	// Set m->sched.sp = SP, so that if a panic happens
   939  	// during the function we are about to execute, it will
   940  	// have a valid SP to run on the g0 stack.
   941  	// The next few lines (after the havem label)
   942  	// will save this SP onto the stack and then write
   943  	// the same SP back to m->sched.sp. That seems redundant,
   944  	// but if an unrecovered panic happens, unwindm will
   945  	// restore the g->sched.sp from the stack location
   946  	// and then systemstack will try to use it. If we don't set it here,
   947  	// that restored SP will be uninitialized (typically 0) and
   948  	// will not be usable.
   949  	MOVQ	m_g0(BX), SI
   950  	MOVQ	SP, (g_sched+gobuf_sp)(SI)
   951  
   952  havem:
   953  	// Now there's a valid m, and we're running on its m->g0.
   954  	// Save current m->g0->sched.sp on stack and then set it to SP.
   955  	// Save current sp in m->g0->sched.sp in preparation for
   956  	// switch back to m->curg stack.
   957  	// NOTE: unwindm knows that the saved g->sched.sp is at 0(SP).
   958  	MOVQ	m_g0(BX), SI
   959  	MOVQ	(g_sched+gobuf_sp)(SI), AX
   960  	MOVQ	AX, 0(SP)
   961  	MOVQ	SP, (g_sched+gobuf_sp)(SI)
   962  
   963  	// Switch to m->curg stack and call runtime.cgocallbackg.
   964  	// Because we are taking over the execution of m->curg
   965  	// but *not* resuming what had been running, we need to
   966  	// save that information (m->curg->sched) so we can restore it.
   967  	// We can restore m->curg->sched.sp easily, because calling
   968  	// runtime.cgocallbackg leaves SP unchanged upon return.
   969  	// To save m->curg->sched.pc, we push it onto the curg stack and
   970  	// open a frame the same size as cgocallback's g0 frame.
   971  	// Once we switch to the curg stack, the pushed PC will appear
   972  	// to be the return PC of cgocallback, so that the traceback
   973  	// will seamlessly trace back into the earlier calls.
   974  	MOVQ	m_curg(BX), SI
   975  	MOVQ	SI, g(CX)
   976  	MOVQ	(g_sched+gobuf_sp)(SI), DI  // prepare stack as DI
   977  	MOVQ	(g_sched+gobuf_pc)(SI), BX
   978  	MOVQ	BX, -8(DI)  // "push" return PC on the g stack
   979  	// Gather our arguments into registers.
   980  	MOVQ	fn+0(FP), BX
   981  	MOVQ	frame+8(FP), CX
   982  	MOVQ	ctxt+16(FP), DX
   983  	// Compute the size of the frame, including return PC and, if
   984  	// GOEXPERIMENT=framepointer, the saved base pointer
   985  	LEAQ	fn+0(FP), AX
   986  	SUBQ	SP, AX   // AX is our actual frame size
   987  	SUBQ	AX, DI   // Allocate the same frame size on the g stack
   988  	MOVQ	DI, SP
   989  
   990  	MOVQ	BX, 0(SP)
   991  	MOVQ	CX, 8(SP)
   992  	MOVQ	DX, 16(SP)
   993  	MOVQ	$runtime·cgocallbackg(SB), AX
   994  	CALL	AX	// indirect call to bypass nosplit check. We're on a different stack now.
   995  
   996  	// Compute the size of the frame again. FP and SP have
   997  	// completely different values here than they did above,
   998  	// but only their difference matters.
   999  	LEAQ	fn+0(FP), AX
  1000  	SUBQ	SP, AX
  1001  
  1002  	// Restore g->sched (== m->curg->sched) from saved values.
  1003  	get_tls(CX)
  1004  	MOVQ	g(CX), SI
  1005  	MOVQ	SP, DI
  1006  	ADDQ	AX, DI
  1007  	MOVQ	-8(DI), BX
  1008  	MOVQ	BX, (g_sched+gobuf_pc)(SI)
  1009  	MOVQ	DI, (g_sched+gobuf_sp)(SI)
  1010  
  1011  	// Switch back to m->g0's stack and restore m->g0->sched.sp.
  1012  	// (Unlike m->curg, the g0 goroutine never uses sched.pc,
  1013  	// so we do not have to restore it.)
  1014  	MOVQ	g(CX), BX
  1015  	MOVQ	g_m(BX), BX
  1016  	MOVQ	m_g0(BX), SI
  1017  	MOVQ	SI, g(CX)
  1018  	MOVQ	(g_sched+gobuf_sp)(SI), SP
  1019  	MOVQ	0(SP), AX
  1020  	MOVQ	AX, (g_sched+gobuf_sp)(SI)
  1021  
  1022  	// If the m on entry was nil, we called needm above to borrow an m
  1023  	// for the duration of the call. Since the call is over, return it with dropm.
  1024  	MOVQ	savedm-8(SP), BX
  1025  	CMPQ	BX, $0
  1026  	JNE	done
  1027  	MOVQ	$runtime·dropm(SB), AX
  1028  	CALL	AX
  1029  #ifdef GOOS_windows
  1030  	// We need to clear the TLS pointer in case the next
  1031  	// thread that comes into Go tries to reuse that space
  1032  	// but uses the same M.
  1033  	XORQ	DI, DI
  1034  	CALL	runtime·settls(SB)
  1035  #endif
  1036  done:
  1037  
  1038  	// Done!
  1039  	RET
  1040  
  1041  // func setg(gg *g)
  1042  // set g. for use by needm.
  1043  TEXT runtime·setg(SB), NOSPLIT, $0-8
  1044  	MOVQ	gg+0(FP), BX
  1045  	get_tls(CX)
  1046  	MOVQ	BX, g(CX)
  1047  	RET
  1048  
  1049  // void setg_gcc(G*); set g called from gcc.
  1050  TEXT setg_gcc<>(SB),NOSPLIT,$0
  1051  	get_tls(AX)
  1052  	MOVQ	DI, g(AX)
  1053  	MOVQ	DI, R14 // set the g register
  1054  	RET
  1055  
  1056  TEXT runtime·abort(SB),NOSPLIT,$0-0
  1057  	INT	$3
  1058  loop:
  1059  	JMP	loop
  1060  
  1061  // check that SP is in range [g->stack.lo, g->stack.hi)
  1062  TEXT runtime·stackcheck(SB), NOSPLIT, $0-0
  1063  	get_tls(CX)
  1064  	MOVQ	g(CX), AX
  1065  	CMPQ	(g_stack+stack_hi)(AX), SP
  1066  	JHI	2(PC)
  1067  	CALL	runtime·abort(SB)
  1068  	CMPQ	SP, (g_stack+stack_lo)(AX)
  1069  	JHI	2(PC)
  1070  	CALL	runtime·abort(SB)
  1071  	RET
  1072  
  1073  // func cputicks() int64
  1074  TEXT runtime·cputicks(SB),NOSPLIT,$0-0
  1075  	CMPB	internal∕cpu·X86+const_offsetX86HasRDTSCP(SB), $1
  1076  	JNE	fences
  1077  	// Instruction stream serializing RDTSCP is supported.
  1078  	// RDTSCP is supported by Intel Nehalem (2008) and
  1079  	// AMD K8 Rev. F (2006) and newer.
  1080  	RDTSCP
  1081  done:
  1082  	SHLQ	$32, DX
  1083  	ADDQ	DX, AX
  1084  	MOVQ	AX, ret+0(FP)
  1085  	RET
  1086  fences:
  1087  	// MFENCE is instruction stream serializing and flushes the
  1088  	// store buffers on AMD. The serialization semantics of LFENCE on AMD
  1089  	// are dependent on MSR C001_1029 and CPU generation.
  1090  	// LFENCE on Intel does wait for all previous instructions to have executed.
  1091  	// Intel recommends MFENCE;LFENCE in its manuals before RDTSC to have all
  1092  	// previous instructions executed and all previous loads and stores to globally visible.
  1093  	// Using MFENCE;LFENCE here aligns the serializing properties without
  1094  	// runtime detection of CPU manufacturer.
  1095  	MFENCE
  1096  	LFENCE
  1097  	RDTSC
  1098  	JMP done
  1099  
  1100  // func memhash(p unsafe.Pointer, h, s uintptr) uintptr
  1101  // hash function using AES hardware instructions
  1102  TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT,$0-32
  1103  	// AX = ptr to data
  1104  	// BX = seed
  1105  	// CX = size
  1106  	CMPB	runtime·useAeshash(SB), $0
  1107  	JEQ	noaes
  1108  	JMP	aeshashbody<>(SB)
  1109  noaes:
  1110  	JMP	runtime·memhashFallback<ABIInternal>(SB)
  1111  
  1112  // func strhash(p unsafe.Pointer, h uintptr) uintptr
  1113  TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT,$0-24
  1114  	// AX = ptr to string struct
  1115  	// BX = seed
  1116  	CMPB	runtime·useAeshash(SB), $0
  1117  	JEQ	noaes
  1118  	MOVQ	8(AX), CX	// length of string
  1119  	MOVQ	(AX), AX	// string data
  1120  	JMP	aeshashbody<>(SB)
  1121  noaes:
  1122  	JMP	runtime·strhashFallback<ABIInternal>(SB)
  1123  
  1124  // AX: data
  1125  // BX: hash seed
  1126  // CX: length
  1127  // At return: AX = return value
  1128  TEXT aeshashbody<>(SB),NOSPLIT,$0-0
  1129  	// Fill an SSE register with our seeds.
  1130  	MOVQ	BX, X0				// 64 bits of per-table hash seed
  1131  	PINSRW	$4, CX, X0			// 16 bits of length
  1132  	PSHUFHW $0, X0, X0			// repeat length 4 times total
  1133  	MOVO	X0, X1				// save unscrambled seed
  1134  	PXOR	runtime·aeskeysched(SB), X0	// xor in per-process seed
  1135  	AESENC	X0, X0				// scramble seed
  1136  
  1137  	CMPQ	CX, $16
  1138  	JB	aes0to15
  1139  	JE	aes16
  1140  	CMPQ	CX, $32
  1141  	JBE	aes17to32
  1142  	CMPQ	CX, $64
  1143  	JBE	aes33to64
  1144  	CMPQ	CX, $128
  1145  	JBE	aes65to128
  1146  	JMP	aes129plus
  1147  
  1148  aes0to15:
  1149  	TESTQ	CX, CX
  1150  	JE	aes0
  1151  
  1152  	ADDQ	$16, AX
  1153  	TESTW	$0xff0, AX
  1154  	JE	endofpage
  1155  
  1156  	// 16 bytes loaded at this address won't cross
  1157  	// a page boundary, so we can load it directly.
  1158  	MOVOU	-16(AX), X1
  1159  	ADDQ	CX, CX
  1160  	MOVQ	$masks<>(SB), AX
  1161  	PAND	(AX)(CX*8), X1
  1162  final1:
  1163  	PXOR	X0, X1	// xor data with seed
  1164  	AESENC	X1, X1	// scramble combo 3 times
  1165  	AESENC	X1, X1
  1166  	AESENC	X1, X1
  1167  	MOVQ	X1, AX	// return X1
  1168  	RET
  1169  
  1170  endofpage:
  1171  	// address ends in 1111xxxx. Might be up against
  1172  	// a page boundary, so load ending at last byte.
  1173  	// Then shift bytes down using pshufb.
  1174  	MOVOU	-32(AX)(CX*1), X1
  1175  	ADDQ	CX, CX
  1176  	MOVQ	$shifts<>(SB), AX
  1177  	PSHUFB	(AX)(CX*8), X1
  1178  	JMP	final1
  1179  
  1180  aes0:
  1181  	// Return scrambled input seed
  1182  	AESENC	X0, X0
  1183  	MOVQ	X0, AX	// return X0
  1184  	RET
  1185  
  1186  aes16:
  1187  	MOVOU	(AX), X1
  1188  	JMP	final1
  1189  
  1190  aes17to32:
  1191  	// make second starting seed
  1192  	PXOR	runtime·aeskeysched+16(SB), X1
  1193  	AESENC	X1, X1
  1194  
  1195  	// load data to be hashed
  1196  	MOVOU	(AX), X2
  1197  	MOVOU	-16(AX)(CX*1), X3
  1198  
  1199  	// xor with seed
  1200  	PXOR	X0, X2
  1201  	PXOR	X1, X3
  1202  
  1203  	// scramble 3 times
  1204  	AESENC	X2, X2
  1205  	AESENC	X3, X3
  1206  	AESENC	X2, X2
  1207  	AESENC	X3, X3
  1208  	AESENC	X2, X2
  1209  	AESENC	X3, X3
  1210  
  1211  	// combine results
  1212  	PXOR	X3, X2
  1213  	MOVQ	X2, AX	// return X2
  1214  	RET
  1215  
  1216  aes33to64:
  1217  	// make 3 more starting seeds
  1218  	MOVO	X1, X2
  1219  	MOVO	X1, X3
  1220  	PXOR	runtime·aeskeysched+16(SB), X1
  1221  	PXOR	runtime·aeskeysched+32(SB), X2
  1222  	PXOR	runtime·aeskeysched+48(SB), X3
  1223  	AESENC	X1, X1
  1224  	AESENC	X2, X2
  1225  	AESENC	X3, X3
  1226  
  1227  	MOVOU	(AX), X4
  1228  	MOVOU	16(AX), X5
  1229  	MOVOU	-32(AX)(CX*1), X6
  1230  	MOVOU	-16(AX)(CX*1), X7
  1231  
  1232  	PXOR	X0, X4
  1233  	PXOR	X1, X5
  1234  	PXOR	X2, X6
  1235  	PXOR	X3, X7
  1236  
  1237  	AESENC	X4, X4
  1238  	AESENC	X5, X5
  1239  	AESENC	X6, X6
  1240  	AESENC	X7, X7
  1241  
  1242  	AESENC	X4, X4
  1243  	AESENC	X5, X5
  1244  	AESENC	X6, X6
  1245  	AESENC	X7, X7
  1246  
  1247  	AESENC	X4, X4
  1248  	AESENC	X5, X5
  1249  	AESENC	X6, X6
  1250  	AESENC	X7, X7
  1251  
  1252  	PXOR	X6, X4
  1253  	PXOR	X7, X5
  1254  	PXOR	X5, X4
  1255  	MOVQ	X4, AX	// return X4
  1256  	RET
  1257  
  1258  aes65to128:
  1259  	// make 7 more starting seeds
  1260  	MOVO	X1, X2
  1261  	MOVO	X1, X3
  1262  	MOVO	X1, X4
  1263  	MOVO	X1, X5
  1264  	MOVO	X1, X6
  1265  	MOVO	X1, X7
  1266  	PXOR	runtime·aeskeysched+16(SB), X1
  1267  	PXOR	runtime·aeskeysched+32(SB), X2
  1268  	PXOR	runtime·aeskeysched+48(SB), X3
  1269  	PXOR	runtime·aeskeysched+64(SB), X4
  1270  	PXOR	runtime·aeskeysched+80(SB), X5
  1271  	PXOR	runtime·aeskeysched+96(SB), X6
  1272  	PXOR	runtime·aeskeysched+112(SB), X7
  1273  	AESENC	X1, X1
  1274  	AESENC	X2, X2
  1275  	AESENC	X3, X3
  1276  	AESENC	X4, X4
  1277  	AESENC	X5, X5
  1278  	AESENC	X6, X6
  1279  	AESENC	X7, X7
  1280  
  1281  	// load data
  1282  	MOVOU	(AX), X8
  1283  	MOVOU	16(AX), X9
  1284  	MOVOU	32(AX), X10
  1285  	MOVOU	48(AX), X11
  1286  	MOVOU	-64(AX)(CX*1), X12
  1287  	MOVOU	-48(AX)(CX*1), X13
  1288  	MOVOU	-32(AX)(CX*1), X14
  1289  	MOVOU	-16(AX)(CX*1), X15
  1290  
  1291  	// xor with seed
  1292  	PXOR	X0, X8
  1293  	PXOR	X1, X9
  1294  	PXOR	X2, X10
  1295  	PXOR	X3, X11
  1296  	PXOR	X4, X12
  1297  	PXOR	X5, X13
  1298  	PXOR	X6, X14
  1299  	PXOR	X7, X15
  1300  
  1301  	// scramble 3 times
  1302  	AESENC	X8, X8
  1303  	AESENC	X9, X9
  1304  	AESENC	X10, X10
  1305  	AESENC	X11, X11
  1306  	AESENC	X12, X12
  1307  	AESENC	X13, X13
  1308  	AESENC	X14, X14
  1309  	AESENC	X15, X15
  1310  
  1311  	AESENC	X8, X8
  1312  	AESENC	X9, X9
  1313  	AESENC	X10, X10
  1314  	AESENC	X11, X11
  1315  	AESENC	X12, X12
  1316  	AESENC	X13, X13
  1317  	AESENC	X14, X14
  1318  	AESENC	X15, X15
  1319  
  1320  	AESENC	X8, X8
  1321  	AESENC	X9, X9
  1322  	AESENC	X10, X10
  1323  	AESENC	X11, X11
  1324  	AESENC	X12, X12
  1325  	AESENC	X13, X13
  1326  	AESENC	X14, X14
  1327  	AESENC	X15, X15
  1328  
  1329  	// combine results
  1330  	PXOR	X12, X8
  1331  	PXOR	X13, X9
  1332  	PXOR	X14, X10
  1333  	PXOR	X15, X11
  1334  	PXOR	X10, X8
  1335  	PXOR	X11, X9
  1336  	PXOR	X9, X8
  1337  	// X15 must be zero on return
  1338  	PXOR	X15, X15
  1339  	MOVQ	X8, AX	// return X8
  1340  	RET
  1341  
  1342  aes129plus:
  1343  	// make 7 more starting seeds
  1344  	MOVO	X1, X2
  1345  	MOVO	X1, X3
  1346  	MOVO	X1, X4
  1347  	MOVO	X1, X5
  1348  	MOVO	X1, X6
  1349  	MOVO	X1, X7
  1350  	PXOR	runtime·aeskeysched+16(SB), X1
  1351  	PXOR	runtime·aeskeysched+32(SB), X2
  1352  	PXOR	runtime·aeskeysched+48(SB), X3
  1353  	PXOR	runtime·aeskeysched+64(SB), X4
  1354  	PXOR	runtime·aeskeysched+80(SB), X5
  1355  	PXOR	runtime·aeskeysched+96(SB), X6
  1356  	PXOR	runtime·aeskeysched+112(SB), X7
  1357  	AESENC	X1, X1
  1358  	AESENC	X2, X2
  1359  	AESENC	X3, X3
  1360  	AESENC	X4, X4
  1361  	AESENC	X5, X5
  1362  	AESENC	X6, X6
  1363  	AESENC	X7, X7
  1364  
  1365  	// start with last (possibly overlapping) block
  1366  	MOVOU	-128(AX)(CX*1), X8
  1367  	MOVOU	-112(AX)(CX*1), X9
  1368  	MOVOU	-96(AX)(CX*1), X10
  1369  	MOVOU	-80(AX)(CX*1), X11
  1370  	MOVOU	-64(AX)(CX*1), X12
  1371  	MOVOU	-48(AX)(CX*1), X13
  1372  	MOVOU	-32(AX)(CX*1), X14
  1373  	MOVOU	-16(AX)(CX*1), X15
  1374  
  1375  	// xor in seed
  1376  	PXOR	X0, X8
  1377  	PXOR	X1, X9
  1378  	PXOR	X2, X10
  1379  	PXOR	X3, X11
  1380  	PXOR	X4, X12
  1381  	PXOR	X5, X13
  1382  	PXOR	X6, X14
  1383  	PXOR	X7, X15
  1384  
  1385  	// compute number of remaining 128-byte blocks
  1386  	DECQ	CX
  1387  	SHRQ	$7, CX
  1388  
  1389  aesloop:
  1390  	// scramble state
  1391  	AESENC	X8, X8
  1392  	AESENC	X9, X9
  1393  	AESENC	X10, X10
  1394  	AESENC	X11, X11
  1395  	AESENC	X12, X12
  1396  	AESENC	X13, X13
  1397  	AESENC	X14, X14
  1398  	AESENC	X15, X15
  1399  
  1400  	// scramble state, xor in a block
  1401  	MOVOU	(AX), X0
  1402  	MOVOU	16(AX), X1
  1403  	MOVOU	32(AX), X2
  1404  	MOVOU	48(AX), X3
  1405  	AESENC	X0, X8
  1406  	AESENC	X1, X9
  1407  	AESENC	X2, X10
  1408  	AESENC	X3, X11
  1409  	MOVOU	64(AX), X4
  1410  	MOVOU	80(AX), X5
  1411  	MOVOU	96(AX), X6
  1412  	MOVOU	112(AX), X7
  1413  	AESENC	X4, X12
  1414  	AESENC	X5, X13
  1415  	AESENC	X6, X14
  1416  	AESENC	X7, X15
  1417  
  1418  	ADDQ	$128, AX
  1419  	DECQ	CX
  1420  	JNE	aesloop
  1421  
  1422  	// 3 more scrambles to finish
  1423  	AESENC	X8, X8
  1424  	AESENC	X9, X9
  1425  	AESENC	X10, X10
  1426  	AESENC	X11, X11
  1427  	AESENC	X12, X12
  1428  	AESENC	X13, X13
  1429  	AESENC	X14, X14
  1430  	AESENC	X15, X15
  1431  	AESENC	X8, X8
  1432  	AESENC	X9, X9
  1433  	AESENC	X10, X10
  1434  	AESENC	X11, X11
  1435  	AESENC	X12, X12
  1436  	AESENC	X13, X13
  1437  	AESENC	X14, X14
  1438  	AESENC	X15, X15
  1439  	AESENC	X8, X8
  1440  	AESENC	X9, X9
  1441  	AESENC	X10, X10
  1442  	AESENC	X11, X11
  1443  	AESENC	X12, X12
  1444  	AESENC	X13, X13
  1445  	AESENC	X14, X14
  1446  	AESENC	X15, X15
  1447  
  1448  	PXOR	X12, X8
  1449  	PXOR	X13, X9
  1450  	PXOR	X14, X10
  1451  	PXOR	X15, X11
  1452  	PXOR	X10, X8
  1453  	PXOR	X11, X9
  1454  	PXOR	X9, X8
  1455  	// X15 must be zero on return
  1456  	PXOR	X15, X15
  1457  	MOVQ	X8, AX	// return X8
  1458  	RET
  1459  
  1460  // func memhash32(p unsafe.Pointer, h uintptr) uintptr
  1461  // ABIInternal for performance.
  1462  TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT,$0-24
  1463  	// AX = ptr to data
  1464  	// BX = seed
  1465  	CMPB	runtime·useAeshash(SB), $0
  1466  	JEQ	noaes
  1467  	MOVQ	BX, X0	// X0 = seed
  1468  	PINSRD	$2, (AX), X0	// data
  1469  	AESENC	runtime·aeskeysched+0(SB), X0
  1470  	AESENC	runtime·aeskeysched+16(SB), X0
  1471  	AESENC	runtime·aeskeysched+32(SB), X0
  1472  	MOVQ	X0, AX	// return X0
  1473  	RET
  1474  noaes:
  1475  	JMP	runtime·memhash32Fallback<ABIInternal>(SB)
  1476  
  1477  // func memhash64(p unsafe.Pointer, h uintptr) uintptr
  1478  // ABIInternal for performance.
  1479  TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT,$0-24
  1480  	// AX = ptr to data
  1481  	// BX = seed
  1482  	CMPB	runtime·useAeshash(SB), $0
  1483  	JEQ	noaes
  1484  	MOVQ	BX, X0	// X0 = seed
  1485  	PINSRQ	$1, (AX), X0	// data
  1486  	AESENC	runtime·aeskeysched+0(SB), X0
  1487  	AESENC	runtime·aeskeysched+16(SB), X0
  1488  	AESENC	runtime·aeskeysched+32(SB), X0
  1489  	MOVQ	X0, AX	// return X0
  1490  	RET
  1491  noaes:
  1492  	JMP	runtime·memhash64Fallback<ABIInternal>(SB)
  1493  
  1494  // simple mask to get rid of data in the high part of the register.
  1495  DATA masks<>+0x00(SB)/8, $0x0000000000000000
  1496  DATA masks<>+0x08(SB)/8, $0x0000000000000000
  1497  DATA masks<>+0x10(SB)/8, $0x00000000000000ff
  1498  DATA masks<>+0x18(SB)/8, $0x0000000000000000
  1499  DATA masks<>+0x20(SB)/8, $0x000000000000ffff
  1500  DATA masks<>+0x28(SB)/8, $0x0000000000000000
  1501  DATA masks<>+0x30(SB)/8, $0x0000000000ffffff
  1502  DATA masks<>+0x38(SB)/8, $0x0000000000000000
  1503  DATA masks<>+0x40(SB)/8, $0x00000000ffffffff
  1504  DATA masks<>+0x48(SB)/8, $0x0000000000000000
  1505  DATA masks<>+0x50(SB)/8, $0x000000ffffffffff
  1506  DATA masks<>+0x58(SB)/8, $0x0000000000000000
  1507  DATA masks<>+0x60(SB)/8, $0x0000ffffffffffff
  1508  DATA masks<>+0x68(SB)/8, $0x0000000000000000
  1509  DATA masks<>+0x70(SB)/8, $0x00ffffffffffffff
  1510  DATA masks<>+0x78(SB)/8, $0x0000000000000000
  1511  DATA masks<>+0x80(SB)/8, $0xffffffffffffffff
  1512  DATA masks<>+0x88(SB)/8, $0x0000000000000000
  1513  DATA masks<>+0x90(SB)/8, $0xffffffffffffffff
  1514  DATA masks<>+0x98(SB)/8, $0x00000000000000ff
  1515  DATA masks<>+0xa0(SB)/8, $0xffffffffffffffff
  1516  DATA masks<>+0xa8(SB)/8, $0x000000000000ffff
  1517  DATA masks<>+0xb0(SB)/8, $0xffffffffffffffff
  1518  DATA masks<>+0xb8(SB)/8, $0x0000000000ffffff
  1519  DATA masks<>+0xc0(SB)/8, $0xffffffffffffffff
  1520  DATA masks<>+0xc8(SB)/8, $0x00000000ffffffff
  1521  DATA masks<>+0xd0(SB)/8, $0xffffffffffffffff
  1522  DATA masks<>+0xd8(SB)/8, $0x000000ffffffffff
  1523  DATA masks<>+0xe0(SB)/8, $0xffffffffffffffff
  1524  DATA masks<>+0xe8(SB)/8, $0x0000ffffffffffff
  1525  DATA masks<>+0xf0(SB)/8, $0xffffffffffffffff
  1526  DATA masks<>+0xf8(SB)/8, $0x00ffffffffffffff
  1527  GLOBL masks<>(SB),RODATA,$256
  1528  
  1529  // func checkASM() bool
  1530  TEXT ·checkASM(SB),NOSPLIT,$0-1
  1531  	// check that masks<>(SB) and shifts<>(SB) are aligned to 16-byte
  1532  	MOVQ	$masks<>(SB), AX
  1533  	MOVQ	$shifts<>(SB), BX
  1534  	ORQ	BX, AX
  1535  	TESTQ	$15, AX
  1536  	SETEQ	ret+0(FP)
  1537  	RET
  1538  
  1539  // these are arguments to pshufb. They move data down from
  1540  // the high bytes of the register to the low bytes of the register.
  1541  // index is how many bytes to move.
  1542  DATA shifts<>+0x00(SB)/8, $0x0000000000000000
  1543  DATA shifts<>+0x08(SB)/8, $0x0000000000000000
  1544  DATA shifts<>+0x10(SB)/8, $0xffffffffffffff0f
  1545  DATA shifts<>+0x18(SB)/8, $0xffffffffffffffff
  1546  DATA shifts<>+0x20(SB)/8, $0xffffffffffff0f0e
  1547  DATA shifts<>+0x28(SB)/8, $0xffffffffffffffff
  1548  DATA shifts<>+0x30(SB)/8, $0xffffffffff0f0e0d
  1549  DATA shifts<>+0x38(SB)/8, $0xffffffffffffffff
  1550  DATA shifts<>+0x40(SB)/8, $0xffffffff0f0e0d0c
  1551  DATA shifts<>+0x48(SB)/8, $0xffffffffffffffff
  1552  DATA shifts<>+0x50(SB)/8, $0xffffff0f0e0d0c0b
  1553  DATA shifts<>+0x58(SB)/8, $0xffffffffffffffff
  1554  DATA shifts<>+0x60(SB)/8, $0xffff0f0e0d0c0b0a
  1555  DATA shifts<>+0x68(SB)/8, $0xffffffffffffffff
  1556  DATA shifts<>+0x70(SB)/8, $0xff0f0e0d0c0b0a09
  1557  DATA shifts<>+0x78(SB)/8, $0xffffffffffffffff
  1558  DATA shifts<>+0x80(SB)/8, $0x0f0e0d0c0b0a0908
  1559  DATA shifts<>+0x88(SB)/8, $0xffffffffffffffff
  1560  DATA shifts<>+0x90(SB)/8, $0x0e0d0c0b0a090807
  1561  DATA shifts<>+0x98(SB)/8, $0xffffffffffffff0f
  1562  DATA shifts<>+0xa0(SB)/8, $0x0d0c0b0a09080706
  1563  DATA shifts<>+0xa8(SB)/8, $0xffffffffffff0f0e
  1564  DATA shifts<>+0xb0(SB)/8, $0x0c0b0a0908070605
  1565  DATA shifts<>+0xb8(SB)/8, $0xffffffffff0f0e0d
  1566  DATA shifts<>+0xc0(SB)/8, $0x0b0a090807060504
  1567  DATA shifts<>+0xc8(SB)/8, $0xffffffff0f0e0d0c
  1568  DATA shifts<>+0xd0(SB)/8, $0x0a09080706050403
  1569  DATA shifts<>+0xd8(SB)/8, $0xffffff0f0e0d0c0b
  1570  DATA shifts<>+0xe0(SB)/8, $0x0908070605040302
  1571  DATA shifts<>+0xe8(SB)/8, $0xffff0f0e0d0c0b0a
  1572  DATA shifts<>+0xf0(SB)/8, $0x0807060504030201
  1573  DATA shifts<>+0xf8(SB)/8, $0xff0f0e0d0c0b0a09
  1574  GLOBL shifts<>(SB),RODATA,$256
  1575  
  1576  TEXT runtime·return0(SB), NOSPLIT, $0
  1577  	MOVL	$0, AX
  1578  	RET
  1579  
  1580  
  1581  // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
  1582  // Must obey the gcc calling convention.
  1583  TEXT _cgo_topofstack(SB),NOSPLIT,$0
  1584  	get_tls(CX)
  1585  	MOVQ	g(CX), AX
  1586  	MOVQ	g_m(AX), AX
  1587  	MOVQ	m_curg(AX), AX
  1588  	MOVQ	(g_stack+stack_hi)(AX), AX
  1589  	RET
  1590  
  1591  // The top-most function running on a goroutine
  1592  // returns to goexit+PCQuantum.
  1593  TEXT runtime·goexit(SB),NOSPLIT|TOPFRAME,$0-0
  1594  	BYTE	$0x90	// NOP
  1595  	CALL	runtime·goexit1(SB)	// does not return
  1596  	// traceback from goexit1 must hit code range of goexit
  1597  	BYTE	$0x90	// NOP
  1598  
  1599  // This is called from .init_array and follows the platform, not Go, ABI.
  1600  TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
  1601  	PUSHQ	R15 // The access to global variables below implicitly uses R15, which is callee-save
  1602  	MOVQ	runtime·lastmoduledatap(SB), AX
  1603  	MOVQ	DI, moduledata_next(AX)
  1604  	MOVQ	DI, runtime·lastmoduledatap(SB)
  1605  	POPQ	R15
  1606  	RET
  1607  
  1608  // Initialize special registers then jump to sigpanic.
  1609  // This function is injected from the signal handler for panicking
  1610  // signals. It is quite painful to set X15 in the signal context,
  1611  // so we do it here.
  1612  TEXT ·sigpanic0(SB),NOSPLIT,$0-0
  1613  	get_tls(R14)
  1614  	MOVQ	g(R14), R14
  1615  #ifndef GOOS_plan9
  1616  	XORPS	X15, X15
  1617  #endif
  1618  	JMP	·sigpanic<ABIInternal>(SB)
  1619  
  1620  // gcWriteBarrier performs a heap pointer write and informs the GC.
  1621  //
  1622  // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
  1623  // - DI is the destination of the write
  1624  // - AX is the value being written at DI
  1625  // It clobbers FLAGS. It does not clobber any general-purpose registers,
  1626  // but may clobber others (e.g., SSE registers).
  1627  // Defined as ABIInternal since it does not use the stack-based Go ABI.
  1628  TEXT runtime·gcWriteBarrier<ABIInternal>(SB),NOSPLIT,$112
  1629  	// Save the registers clobbered by the fast path. This is slightly
  1630  	// faster than having the caller spill these.
  1631  	MOVQ	R12, 96(SP)
  1632  	MOVQ	R13, 104(SP)
  1633  	// TODO: Consider passing g.m.p in as an argument so they can be shared
  1634  	// across a sequence of write barriers.
  1635  	MOVQ	g_m(R14), R13
  1636  	MOVQ	m_p(R13), R13
  1637  	MOVQ	(p_wbBuf+wbBuf_next)(R13), R12
  1638  	// Increment wbBuf.next position.
  1639  	LEAQ	16(R12), R12
  1640  	MOVQ	R12, (p_wbBuf+wbBuf_next)(R13)
  1641  	CMPQ	R12, (p_wbBuf+wbBuf_end)(R13)
  1642  	// Record the write.
  1643  	MOVQ	AX, -16(R12)	// Record value
  1644  	// Note: This turns bad pointer writes into bad
  1645  	// pointer reads, which could be confusing. We could avoid
  1646  	// reading from obviously bad pointers, which would
  1647  	// take care of the vast majority of these. We could
  1648  	// patch this up in the signal handler, or use XCHG to
  1649  	// combine the read and the write.
  1650  	MOVQ	(DI), R13
  1651  	MOVQ	R13, -8(R12)	// Record *slot
  1652  	// Is the buffer full? (flags set in CMPQ above)
  1653  	JEQ	flush
  1654  ret:
  1655  	MOVQ	96(SP), R12
  1656  	MOVQ	104(SP), R13
  1657  	// Do the write.
  1658  	MOVQ	AX, (DI)
  1659  	RET
  1660  
  1661  flush:
  1662  	// Save all general purpose registers since these could be
  1663  	// clobbered by wbBufFlush and were not saved by the caller.
  1664  	// It is possible for wbBufFlush to clobber other registers
  1665  	// (e.g., SSE registers), but the compiler takes care of saving
  1666  	// those in the caller if necessary. This strikes a balance
  1667  	// with registers that are likely to be used.
  1668  	//
  1669  	// We don't have type information for these, but all code under
  1670  	// here is NOSPLIT, so nothing will observe these.
  1671  	//
  1672  	// TODO: We could strike a different balance; e.g., saving X0
  1673  	// and not saving GP registers that are less likely to be used.
  1674  	MOVQ	DI, 0(SP)	// Also first argument to wbBufFlush
  1675  	MOVQ	AX, 8(SP)	// Also second argument to wbBufFlush
  1676  	MOVQ	BX, 16(SP)
  1677  	MOVQ	CX, 24(SP)
  1678  	MOVQ	DX, 32(SP)
  1679  	// DI already saved
  1680  	MOVQ	SI, 40(SP)
  1681  	MOVQ	BP, 48(SP)
  1682  	MOVQ	R8, 56(SP)
  1683  	MOVQ	R9, 64(SP)
  1684  	MOVQ	R10, 72(SP)
  1685  	MOVQ	R11, 80(SP)
  1686  	// R12 already saved
  1687  	// R13 already saved
  1688  	// R14 is g
  1689  	MOVQ	R15, 88(SP)
  1690  
  1691  	// This takes arguments DI and AX
  1692  	CALL	runtime·wbBufFlush(SB)
  1693  
  1694  	MOVQ	0(SP), DI
  1695  	MOVQ	8(SP), AX
  1696  	MOVQ	16(SP), BX
  1697  	MOVQ	24(SP), CX
  1698  	MOVQ	32(SP), DX
  1699  	MOVQ	40(SP), SI
  1700  	MOVQ	48(SP), BP
  1701  	MOVQ	56(SP), R8
  1702  	MOVQ	64(SP), R9
  1703  	MOVQ	72(SP), R10
  1704  	MOVQ	80(SP), R11
  1705  	MOVQ	88(SP), R15
  1706  	JMP	ret
  1707  
  1708  // gcWriteBarrierCX is gcWriteBarrier, but with args in DI and CX.
  1709  // Defined as ABIInternal since it does not use the stable Go ABI.
  1710  TEXT runtime·gcWriteBarrierCX<ABIInternal>(SB),NOSPLIT,$0
  1711  	XCHGQ CX, AX
  1712  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1713  	XCHGQ CX, AX
  1714  	RET
  1715  
  1716  // gcWriteBarrierDX is gcWriteBarrier, but with args in DI and DX.
  1717  // Defined as ABIInternal since it does not use the stable Go ABI.
  1718  TEXT runtime·gcWriteBarrierDX<ABIInternal>(SB),NOSPLIT,$0
  1719  	XCHGQ DX, AX
  1720  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1721  	XCHGQ DX, AX
  1722  	RET
  1723  
  1724  // gcWriteBarrierBX is gcWriteBarrier, but with args in DI and BX.
  1725  // Defined as ABIInternal since it does not use the stable Go ABI.
  1726  TEXT runtime·gcWriteBarrierBX<ABIInternal>(SB),NOSPLIT,$0
  1727  	XCHGQ BX, AX
  1728  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1729  	XCHGQ BX, AX
  1730  	RET
  1731  
  1732  // gcWriteBarrierBP is gcWriteBarrier, but with args in DI and BP.
  1733  // Defined as ABIInternal since it does not use the stable Go ABI.
  1734  TEXT runtime·gcWriteBarrierBP<ABIInternal>(SB),NOSPLIT,$0
  1735  	XCHGQ BP, AX
  1736  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1737  	XCHGQ BP, AX
  1738  	RET
  1739  
  1740  // gcWriteBarrierSI is gcWriteBarrier, but with args in DI and SI.
  1741  // Defined as ABIInternal since it does not use the stable Go ABI.
  1742  TEXT runtime·gcWriteBarrierSI<ABIInternal>(SB),NOSPLIT,$0
  1743  	XCHGQ SI, AX
  1744  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1745  	XCHGQ SI, AX
  1746  	RET
  1747  
  1748  // gcWriteBarrierR8 is gcWriteBarrier, but with args in DI and R8.
  1749  // Defined as ABIInternal since it does not use the stable Go ABI.
  1750  TEXT runtime·gcWriteBarrierR8<ABIInternal>(SB),NOSPLIT,$0
  1751  	XCHGQ R8, AX
  1752  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1753  	XCHGQ R8, AX
  1754  	RET
  1755  
  1756  // gcWriteBarrierR9 is gcWriteBarrier, but with args in DI and R9.
  1757  // Defined as ABIInternal since it does not use the stable Go ABI.
  1758  TEXT runtime·gcWriteBarrierR9<ABIInternal>(SB),NOSPLIT,$0
  1759  	XCHGQ R9, AX
  1760  	CALL runtime·gcWriteBarrier<ABIInternal>(SB)
  1761  	XCHGQ R9, AX
  1762  	RET
  1763  
  1764  DATA	debugCallFrameTooLarge<>+0x00(SB)/20, $"call frame too large"
  1765  GLOBL	debugCallFrameTooLarge<>(SB), RODATA, $20	// Size duplicated below
  1766  
  1767  // debugCallV2 is the entry point for debugger-injected function
  1768  // calls on running goroutines. It informs the runtime that a
  1769  // debug call has been injected and creates a call frame for the
  1770  // debugger to fill in.
  1771  //
  1772  // To inject a function call, a debugger should:
  1773  // 1. Check that the goroutine is in state _Grunning and that
  1774  //    there are at least 256 bytes free on the stack.
  1775  // 2. Push the current PC on the stack (updating SP).
  1776  // 3. Write the desired argument frame size at SP-16 (using the SP
  1777  //    after step 2).
  1778  // 4. Save all machine registers (including flags and XMM registers)
  1779  //    so they can be restored later by the debugger.
  1780  // 5. Set the PC to debugCallV2 and resume execution.
  1781  //
  1782  // If the goroutine is in state _Grunnable, then it's not generally
  1783  // safe to inject a call because it may return out via other runtime
  1784  // operations. Instead, the debugger should unwind the stack to find
  1785  // the return to non-runtime code, add a temporary breakpoint there,
  1786  // and inject the call once that breakpoint is hit.
  1787  //
  1788  // If the goroutine is in any other state, it's not safe to inject a call.
  1789  //
  1790  // This function communicates back to the debugger by setting R12 and
  1791  // invoking INT3 to raise a breakpoint signal. See the comments in the
  1792  // implementation for the protocol the debugger is expected to
  1793  // follow. InjectDebugCall in the runtime tests demonstrates this protocol.
  1794  //
  1795  // The debugger must ensure that any pointers passed to the function
  1796  // obey escape analysis requirements. Specifically, it must not pass
  1797  // a stack pointer to an escaping argument. debugCallV2 cannot check
  1798  // this invariant.
  1799  //
  1800  // This is ABIInternal because Go code injects its PC directly into new
  1801  // goroutine stacks.
  1802  TEXT runtime·debugCallV2<ABIInternal>(SB),NOSPLIT,$152-0
  1803  	// Save all registers that may contain pointers so they can be
  1804  	// conservatively scanned.
  1805  	//
  1806  	// We can't do anything that might clobber any of these
  1807  	// registers before this.
  1808  	MOVQ	R15, r15-(14*8+8)(SP)
  1809  	MOVQ	R14, r14-(13*8+8)(SP)
  1810  	MOVQ	R13, r13-(12*8+8)(SP)
  1811  	MOVQ	R12, r12-(11*8+8)(SP)
  1812  	MOVQ	R11, r11-(10*8+8)(SP)
  1813  	MOVQ	R10, r10-(9*8+8)(SP)
  1814  	MOVQ	R9, r9-(8*8+8)(SP)
  1815  	MOVQ	R8, r8-(7*8+8)(SP)
  1816  	MOVQ	DI, di-(6*8+8)(SP)
  1817  	MOVQ	SI, si-(5*8+8)(SP)
  1818  	MOVQ	BP, bp-(4*8+8)(SP)
  1819  	MOVQ	BX, bx-(3*8+8)(SP)
  1820  	MOVQ	DX, dx-(2*8+8)(SP)
  1821  	// Save the frame size before we clobber it. Either of the last
  1822  	// saves could clobber this depending on whether there's a saved BP.
  1823  	MOVQ	frameSize-24(FP), DX	// aka -16(RSP) before prologue
  1824  	MOVQ	CX, cx-(1*8+8)(SP)
  1825  	MOVQ	AX, ax-(0*8+8)(SP)
  1826  
  1827  	// Save the argument frame size.
  1828  	MOVQ	DX, frameSize-128(SP)
  1829  
  1830  	// Perform a safe-point check.
  1831  	MOVQ	retpc-8(FP), AX	// Caller's PC
  1832  	MOVQ	AX, 0(SP)
  1833  	CALL	runtime·debugCallCheck(SB)
  1834  	MOVQ	8(SP), AX
  1835  	TESTQ	AX, AX
  1836  	JZ	good
  1837  	// The safety check failed. Put the reason string at the top
  1838  	// of the stack.
  1839  	MOVQ	AX, 0(SP)
  1840  	MOVQ	16(SP), AX
  1841  	MOVQ	AX, 8(SP)
  1842  	// Set R12 to 8 and invoke INT3. The debugger should get the
  1843  	// reason a call can't be injected from the top of the stack
  1844  	// and resume execution.
  1845  	MOVQ	$8, R12
  1846  	BYTE	$0xcc
  1847  	JMP	restore
  1848  
  1849  good:
  1850  	// Registers are saved and it's safe to make a call.
  1851  	// Open up a call frame, moving the stack if necessary.
  1852  	//
  1853  	// Once the frame is allocated, this will set R12 to 0 and
  1854  	// invoke INT3. The debugger should write the argument
  1855  	// frame for the call at SP, set up argument registers, push
  1856  	// the trapping PC on the stack, set the PC to the function to
  1857  	// call, set RDX to point to the closure (if a closure call),
  1858  	// and resume execution.
  1859  	//
  1860  	// If the function returns, this will set R12 to 1 and invoke
  1861  	// INT3. The debugger can then inspect any return value saved
  1862  	// on the stack at SP and in registers and resume execution again.
  1863  	//
  1864  	// If the function panics, this will set R12 to 2 and invoke INT3.
  1865  	// The interface{} value of the panic will be at SP. The debugger
  1866  	// can inspect the panic value and resume execution again.
  1867  #define DEBUG_CALL_DISPATCH(NAME,MAXSIZE)	\
  1868  	CMPQ	AX, $MAXSIZE;			\
  1869  	JA	5(PC);				\
  1870  	MOVQ	$NAME(SB), AX;			\
  1871  	MOVQ	AX, 0(SP);			\
  1872  	CALL	runtime·debugCallWrap(SB);	\
  1873  	JMP	restore
  1874  
  1875  	MOVQ	frameSize-128(SP), AX
  1876  	DEBUG_CALL_DISPATCH(debugCall32<>, 32)
  1877  	DEBUG_CALL_DISPATCH(debugCall64<>, 64)
  1878  	DEBUG_CALL_DISPATCH(debugCall128<>, 128)
  1879  	DEBUG_CALL_DISPATCH(debugCall256<>, 256)
  1880  	DEBUG_CALL_DISPATCH(debugCall512<>, 512)
  1881  	DEBUG_CALL_DISPATCH(debugCall1024<>, 1024)
  1882  	DEBUG_CALL_DISPATCH(debugCall2048<>, 2048)
  1883  	DEBUG_CALL_DISPATCH(debugCall4096<>, 4096)
  1884  	DEBUG_CALL_DISPATCH(debugCall8192<>, 8192)
  1885  	DEBUG_CALL_DISPATCH(debugCall16384<>, 16384)
  1886  	DEBUG_CALL_DISPATCH(debugCall32768<>, 32768)
  1887  	DEBUG_CALL_DISPATCH(debugCall65536<>, 65536)
  1888  	// The frame size is too large. Report the error.
  1889  	MOVQ	$debugCallFrameTooLarge<>(SB), AX
  1890  	MOVQ	AX, 0(SP)
  1891  	MOVQ	$20, 8(SP) // length of debugCallFrameTooLarge string
  1892  	MOVQ	$8, R12
  1893  	BYTE	$0xcc
  1894  	JMP	restore
  1895  
  1896  restore:
  1897  	// Calls and failures resume here.
  1898  	//
  1899  	// Set R12 to 16 and invoke INT3. The debugger should restore
  1900  	// all registers except RIP and RSP and resume execution.
  1901  	MOVQ	$16, R12
  1902  	BYTE	$0xcc
  1903  	// We must not modify flags after this point.
  1904  
  1905  	// Restore pointer-containing registers, which may have been
  1906  	// modified from the debugger's copy by stack copying.
  1907  	MOVQ	ax-(0*8+8)(SP), AX
  1908  	MOVQ	cx-(1*8+8)(SP), CX
  1909  	MOVQ	dx-(2*8+8)(SP), DX
  1910  	MOVQ	bx-(3*8+8)(SP), BX
  1911  	MOVQ	bp-(4*8+8)(SP), BP
  1912  	MOVQ	si-(5*8+8)(SP), SI
  1913  	MOVQ	di-(6*8+8)(SP), DI
  1914  	MOVQ	r8-(7*8+8)(SP), R8
  1915  	MOVQ	r9-(8*8+8)(SP), R9
  1916  	MOVQ	r10-(9*8+8)(SP), R10
  1917  	MOVQ	r11-(10*8+8)(SP), R11
  1918  	MOVQ	r12-(11*8+8)(SP), R12
  1919  	MOVQ	r13-(12*8+8)(SP), R13
  1920  	MOVQ	r14-(13*8+8)(SP), R14
  1921  	MOVQ	r15-(14*8+8)(SP), R15
  1922  
  1923  	RET
  1924  
  1925  // runtime.debugCallCheck assumes that functions defined with the
  1926  // DEBUG_CALL_FN macro are safe points to inject calls.
  1927  #define DEBUG_CALL_FN(NAME,MAXSIZE)		\
  1928  TEXT NAME(SB),WRAPPER,$MAXSIZE-0;		\
  1929  	NO_LOCAL_POINTERS;			\
  1930  	MOVQ	$0, R12;				\
  1931  	BYTE	$0xcc;				\
  1932  	MOVQ	$1, R12;				\
  1933  	BYTE	$0xcc;				\
  1934  	RET
  1935  DEBUG_CALL_FN(debugCall32<>, 32)
  1936  DEBUG_CALL_FN(debugCall64<>, 64)
  1937  DEBUG_CALL_FN(debugCall128<>, 128)
  1938  DEBUG_CALL_FN(debugCall256<>, 256)
  1939  DEBUG_CALL_FN(debugCall512<>, 512)
  1940  DEBUG_CALL_FN(debugCall1024<>, 1024)
  1941  DEBUG_CALL_FN(debugCall2048<>, 2048)
  1942  DEBUG_CALL_FN(debugCall4096<>, 4096)
  1943  DEBUG_CALL_FN(debugCall8192<>, 8192)
  1944  DEBUG_CALL_FN(debugCall16384<>, 16384)
  1945  DEBUG_CALL_FN(debugCall32768<>, 32768)
  1946  DEBUG_CALL_FN(debugCall65536<>, 65536)
  1947  
  1948  // func debugCallPanicked(val interface{})
  1949  TEXT runtime·debugCallPanicked(SB),NOSPLIT,$16-16
  1950  	// Copy the panic value to the top of stack.
  1951  	MOVQ	val_type+0(FP), AX
  1952  	MOVQ	AX, 0(SP)
  1953  	MOVQ	val_data+8(FP), AX
  1954  	MOVQ	AX, 8(SP)
  1955  	MOVQ	$2, R12
  1956  	BYTE	$0xcc
  1957  	RET
  1958  
  1959  // Note: these functions use a special calling convention to save generated code space.
  1960  // Arguments are passed in registers, but the space for those arguments are allocated
  1961  // in the caller's stack frame. These stubs write the args into that stack space and
  1962  // then tail call to the corresponding runtime handler.
  1963  // The tail call makes these stubs disappear in backtraces.
  1964  // Defined as ABIInternal since they do not use the stack-based Go ABI.
  1965  TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16
  1966  	MOVQ	CX, BX
  1967  	JMP	runtime·goPanicIndex<ABIInternal>(SB)
  1968  TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16
  1969  	MOVQ	CX, BX
  1970  	JMP	runtime·goPanicIndexU<ABIInternal>(SB)
  1971  TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16
  1972  	MOVQ	CX, AX
  1973  	MOVQ	DX, BX
  1974  	JMP	runtime·goPanicSliceAlen<ABIInternal>(SB)
  1975  TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16
  1976  	MOVQ	CX, AX
  1977  	MOVQ	DX, BX
  1978  	JMP	runtime·goPanicSliceAlenU<ABIInternal>(SB)
  1979  TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16
  1980  	MOVQ	CX, AX
  1981  	MOVQ	DX, BX
  1982  	JMP	runtime·goPanicSliceAcap<ABIInternal>(SB)
  1983  TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16
  1984  	MOVQ	CX, AX
  1985  	MOVQ	DX, BX
  1986  	JMP	runtime·goPanicSliceAcapU<ABIInternal>(SB)
  1987  TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16
  1988  	MOVQ	CX, BX
  1989  	JMP	runtime·goPanicSliceB<ABIInternal>(SB)
  1990  TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16
  1991  	MOVQ	CX, BX
  1992  	JMP	runtime·goPanicSliceBU<ABIInternal>(SB)
  1993  TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16
  1994  	MOVQ	DX, AX
  1995  	JMP	runtime·goPanicSlice3Alen<ABIInternal>(SB)
  1996  TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16
  1997  	MOVQ	DX, AX
  1998  	JMP	runtime·goPanicSlice3AlenU<ABIInternal>(SB)
  1999  TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16
  2000  	MOVQ	DX, AX
  2001  	JMP	runtime·goPanicSlice3Acap<ABIInternal>(SB)
  2002  TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16
  2003  	MOVQ	DX, AX
  2004  	JMP	runtime·goPanicSlice3AcapU<ABIInternal>(SB)
  2005  TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16
  2006  	MOVQ	CX, AX
  2007  	MOVQ	DX, BX
  2008  	JMP	runtime·goPanicSlice3B<ABIInternal>(SB)
  2009  TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16
  2010  	MOVQ	CX, AX
  2011  	MOVQ	DX, BX
  2012  	JMP	runtime·goPanicSlice3BU<ABIInternal>(SB)
  2013  TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16
  2014  	MOVQ	CX, BX
  2015  	JMP	runtime·goPanicSlice3C<ABIInternal>(SB)
  2016  TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16
  2017  	MOVQ	CX, BX
  2018  	JMP	runtime·goPanicSlice3CU<ABIInternal>(SB)
  2019  TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16
  2020  	MOVQ	DX, AX
  2021  	JMP	runtime·goPanicSliceConvert<ABIInternal>(SB)
  2022  
  2023  #ifdef GOOS_android
  2024  // Use the free TLS_SLOT_APP slot #2 on Android Q.
  2025  // Earlier androids are set up in gcc_android.c.
  2026  DATA runtime·tls_g+0(SB)/8, $16
  2027  GLOBL runtime·tls_g+0(SB), NOPTR, $8
  2028  #endif
  2029  
  2030  // The compiler and assembler's -spectre=ret mode rewrites
  2031  // all indirect CALL AX / JMP AX instructions to be
  2032  // CALL retpolineAX / JMP retpolineAX.
  2033  // See https://support.google.com/faqs/answer/7625886.
  2034  #define RETPOLINE(reg) \
  2035  	/*   CALL setup */     BYTE $0xE8; BYTE $(2+2); BYTE $0; BYTE $0; BYTE $0;	\
  2036  	/* nospec: */									\
  2037  	/*   PAUSE */           BYTE $0xF3; BYTE $0x90;					\
  2038  	/*   JMP nospec */      BYTE $0xEB; BYTE $-(2+2);				\
  2039  	/* setup: */									\
  2040  	/*   MOVQ AX, 0(SP) */  BYTE $0x48|((reg&8)>>1); BYTE $0x89;			\
  2041  	                        BYTE $0x04|((reg&7)<<3); BYTE $0x24;			\
  2042  	/*   RET */             BYTE $0xC3
  2043  
  2044  TEXT runtime·retpolineAX(SB),NOSPLIT,$0; RETPOLINE(0)
  2045  TEXT runtime·retpolineCX(SB),NOSPLIT,$0; RETPOLINE(1)
  2046  TEXT runtime·retpolineDX(SB),NOSPLIT,$0; RETPOLINE(2)
  2047  TEXT runtime·retpolineBX(SB),NOSPLIT,$0; RETPOLINE(3)
  2048  /* SP is 4, can't happen / magic encodings */
  2049  TEXT runtime·retpolineBP(SB),NOSPLIT,$0; RETPOLINE(5)
  2050  TEXT runtime·retpolineSI(SB),NOSPLIT,$0; RETPOLINE(6)
  2051  TEXT runtime·retpolineDI(SB),NOSPLIT,$0; RETPOLINE(7)
  2052  TEXT runtime·retpolineR8(SB),NOSPLIT,$0; RETPOLINE(8)
  2053  TEXT runtime·retpolineR9(SB),NOSPLIT,$0; RETPOLINE(9)
  2054  TEXT runtime·retpolineR10(SB),NOSPLIT,$0; RETPOLINE(10)
  2055  TEXT runtime·retpolineR11(SB),NOSPLIT,$0; RETPOLINE(11)
  2056  TEXT runtime·retpolineR12(SB),NOSPLIT,$0; RETPOLINE(12)
  2057  TEXT runtime·retpolineR13(SB),NOSPLIT,$0; RETPOLINE(13)
  2058  TEXT runtime·retpolineR14(SB),NOSPLIT,$0; RETPOLINE(14)
  2059  TEXT runtime·retpolineR15(SB),NOSPLIT,$0; RETPOLINE(15)
  2060  

View as plain text