Source file src/runtime/stubs.go

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/goarch"
    10  	"runtime/internal/math"
    11  	"unsafe"
    12  )
    13  
    14  // Should be a built-in for unsafe.Pointer?
    15  //
    16  //go:nosplit
    17  func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
    18  	return unsafe.Pointer(uintptr(p) + x)
    19  }
    20  
    21  // getg returns the pointer to the current g.
    22  // The compiler rewrites calls to this function into instructions
    23  // that fetch the g directly (from TLS or from the dedicated register).
    24  func getg() *g
    25  
    26  // mcall switches from the g to the g0 stack and invokes fn(g),
    27  // where g is the goroutine that made the call.
    28  // mcall saves g's current PC/SP in g->sched so that it can be restored later.
    29  // It is up to fn to arrange for that later execution, typically by recording
    30  // g in a data structure, causing something to call ready(g) later.
    31  // mcall returns to the original goroutine g later, when g has been rescheduled.
    32  // fn must not return at all; typically it ends by calling schedule, to let the m
    33  // run other goroutines.
    34  //
    35  // mcall can only be called from g stacks (not g0, not gsignal).
    36  //
    37  // This must NOT be go:noescape: if fn is a stack-allocated closure,
    38  // fn puts g on a run queue, and g executes before fn returns, the
    39  // closure will be invalidated while it is still executing.
    40  func mcall(fn func(*g))
    41  
    42  // systemstack runs fn on a system stack.
    43  // If systemstack is called from the per-OS-thread (g0) stack, or
    44  // if systemstack is called from the signal handling (gsignal) stack,
    45  // systemstack calls fn directly and returns.
    46  // Otherwise, systemstack is being called from the limited stack
    47  // of an ordinary goroutine. In this case, systemstack switches
    48  // to the per-OS-thread stack, calls fn, and switches back.
    49  // It is common to use a func literal as the argument, in order
    50  // to share inputs and outputs with the code around the call
    51  // to system stack:
    52  //
    53  //	... set up y ...
    54  //	systemstack(func() {
    55  //		x = bigcall(y)
    56  //	})
    57  //	... use x ...
    58  //
    59  //go:noescape
    60  func systemstack(fn func())
    61  
    62  var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine"
    63  
    64  //go:nosplit
    65  //go:nowritebarrierrec
    66  func badsystemstack() {
    67  	sp := stringStructOf(&badsystemstackMsg)
    68  	write(2, sp.str, int32(sp.len))
    69  }
    70  
    71  // memclrNoHeapPointers clears n bytes starting at ptr.
    72  //
    73  // Usually you should use typedmemclr. memclrNoHeapPointers should be
    74  // used only when the caller knows that *ptr contains no heap pointers
    75  // because either:
    76  //
    77  // *ptr is initialized memory and its type is pointer-free, or
    78  //
    79  // *ptr is uninitialized memory (e.g., memory that's being reused
    80  // for a new allocation) and hence contains only "junk".
    81  //
    82  // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n
    83  // is a multiple of the pointer size, then any pointer-aligned,
    84  // pointer-sized portion is cleared atomically. Despite the function
    85  // name, this is necessary because this function is the underlying
    86  // implementation of typedmemclr and memclrHasPointers. See the doc of
    87  // memmove for more details.
    88  //
    89  // The (CPU-specific) implementations of this function are in memclr_*.s.
    90  //
    91  //go:noescape
    92  func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
    93  
    94  //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
    95  func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
    96  	memclrNoHeapPointers(ptr, n)
    97  }
    98  
    99  // memmove copies n bytes from "from" to "to".
   100  //
   101  // memmove ensures that any pointer in "from" is written to "to" with
   102  // an indivisible write, so that racy reads cannot observe a
   103  // half-written pointer. This is necessary to prevent the garbage
   104  // collector from observing invalid pointers, and differs from memmove
   105  // in unmanaged languages. However, memmove is only required to do
   106  // this if "from" and "to" may contain pointers, which can only be the
   107  // case if "from", "to", and "n" are all be word-aligned.
   108  //
   109  // Implementations are in memmove_*.s.
   110  //
   111  //go:noescape
   112  func memmove(to, from unsafe.Pointer, n uintptr)
   113  
   114  // Outside assembly calls memmove. Make sure it has ABI wrappers.
   115  //
   116  //go:linkname memmove
   117  
   118  //go:linkname reflect_memmove reflect.memmove
   119  func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
   120  	memmove(to, from, n)
   121  }
   122  
   123  // exported value for testing
   124  const hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
   125  
   126  //go:nosplit
   127  func fastrand() uint32 {
   128  	mp := getg().m
   129  	// Implement wyrand: https://github.com/wangyi-fudan/wyhash
   130  	// Only the platform that math.Mul64 can be lowered
   131  	// by the compiler should be in this list.
   132  	if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64|
   133  		goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le|
   134  		goarch.IsS390x|goarch.IsRiscv64 == 1 {
   135  		mp.fastrand += 0xa0761d6478bd642f
   136  		hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db)
   137  		return uint32(hi ^ lo)
   138  	}
   139  
   140  	// Implement xorshift64+: 2 32-bit xorshift sequences added together.
   141  	// Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
   142  	// Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
   143  	// This generator passes the SmallCrush suite, part of TestU01 framework:
   144  	// http://simul.iro.umontreal.ca/testu01/tu01.html
   145  	t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand))
   146  	s1, s0 := t[0], t[1]
   147  	s1 ^= s1 << 17
   148  	s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
   149  	t[0], t[1] = s0, s1
   150  	return s0 + s1
   151  }
   152  
   153  //go:nosplit
   154  func fastrandn(n uint32) uint32 {
   155  	// This is similar to fastrand() % n, but faster.
   156  	// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
   157  	return uint32(uint64(fastrand()) * uint64(n) >> 32)
   158  }
   159  
   160  func fastrand64() uint64 {
   161  	mp := getg().m
   162  	// Implement wyrand: https://github.com/wangyi-fudan/wyhash
   163  	// Only the platform that math.Mul64 can be lowered
   164  	// by the compiler should be in this list.
   165  	if goarch.IsAmd64|goarch.IsArm64|goarch.IsPpc64|
   166  		goarch.IsPpc64le|goarch.IsMips64|goarch.IsMips64le|
   167  		goarch.IsS390x|goarch.IsRiscv64 == 1 {
   168  		mp.fastrand += 0xa0761d6478bd642f
   169  		hi, lo := math.Mul64(mp.fastrand, mp.fastrand^0xe7037ed1a0b428db)
   170  		return hi ^ lo
   171  	}
   172  
   173  	// Implement xorshift64+: 2 32-bit xorshift sequences added together.
   174  	// Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
   175  	// This generator passes the SmallCrush suite, part of TestU01 framework:
   176  	// http://simul.iro.umontreal.ca/testu01/tu01.html
   177  	t := (*[2]uint32)(unsafe.Pointer(&mp.fastrand))
   178  	s1, s0 := t[0], t[1]
   179  	s1 ^= s1 << 17
   180  	s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
   181  	r := uint64(s0 + s1)
   182  
   183  	s0, s1 = s1, s0
   184  	s1 ^= s1 << 17
   185  	s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
   186  	r += uint64(s0+s1) << 32
   187  
   188  	t[0], t[1] = s0, s1
   189  	return r
   190  }
   191  
   192  func fastrandu() uint {
   193  	if goarch.PtrSize == 4 {
   194  		return uint(fastrand())
   195  	}
   196  	return uint(fastrand64())
   197  }
   198  
   199  //go:linkname sync_fastrandn sync.fastrandn
   200  func sync_fastrandn(n uint32) uint32 { return fastrandn(n) }
   201  
   202  //go:linkname net_fastrandu net.fastrandu
   203  func net_fastrandu() uint { return fastrandu() }
   204  
   205  //go:linkname os_fastrand os.fastrand
   206  func os_fastrand() uint32 { return fastrand() }
   207  
   208  // in internal/bytealg/equal_*.s
   209  //
   210  //go:noescape
   211  func memequal(a, b unsafe.Pointer, size uintptr) bool
   212  
   213  // noescape hides a pointer from escape analysis.  noescape is
   214  // the identity function but escape analysis doesn't think the
   215  // output depends on the input.  noescape is inlined and currently
   216  // compiles down to zero instructions.
   217  // USE CAREFULLY!
   218  //
   219  //go:nosplit
   220  func noescape(p unsafe.Pointer) unsafe.Pointer {
   221  	x := uintptr(p)
   222  	return unsafe.Pointer(x ^ 0)
   223  }
   224  
   225  // Not all cgocallback frames are actually cgocallback,
   226  // so not all have these arguments. Mark them uintptr so that the GC
   227  // does not misinterpret memory when the arguments are not present.
   228  // cgocallback is not called from Go, only from crosscall2.
   229  // This in turn calls cgocallbackg, which is where we'll find
   230  // pointer-declared arguments.
   231  func cgocallback(fn, frame, ctxt uintptr)
   232  
   233  func gogo(buf *gobuf)
   234  
   235  func asminit()
   236  func setg(gg *g)
   237  func breakpoint()
   238  
   239  // reflectcall calls fn with arguments described by stackArgs, stackArgsSize,
   240  // frameSize, and regArgs.
   241  //
   242  // Arguments passed on the stack and space for return values passed on the stack
   243  // must be laid out at the space pointed to by stackArgs (with total length
   244  // stackArgsSize) according to the ABI.
   245  //
   246  // stackRetOffset must be some value <= stackArgsSize that indicates the
   247  // offset within stackArgs where the return value space begins.
   248  //
   249  // frameSize is the total size of the argument frame at stackArgs and must
   250  // therefore be >= stackArgsSize. It must include additional space for spilling
   251  // register arguments for stack growth and preemption.
   252  //
   253  // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize,
   254  // since frameSize will be redundant with stackArgsSize.
   255  //
   256  // Arguments passed in registers must be laid out in regArgs according to the ABI.
   257  // regArgs will hold any return values passed in registers after the call.
   258  //
   259  // reflectcall copies stack arguments from stackArgs to the goroutine stack, and
   260  // then copies back stackArgsSize-stackRetOffset bytes back to the return space
   261  // in stackArgs once fn has completed. It also "unspills" argument registers from
   262  // regArgs before calling fn, and spills them back into regArgs immediately
   263  // following the call to fn. If there are results being returned on the stack,
   264  // the caller should pass the argument frame type as stackArgsType so that
   265  // reflectcall can execute appropriate write barriers during the copy.
   266  //
   267  // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which
   268  // registers on the return path will contain Go pointers. It will then store
   269  // these pointers in regArgs.Ptrs such that they are visible to the GC.
   270  //
   271  // Package reflect passes a frame type. In package runtime, there is only
   272  // one call that copies results back, in callbackWrap in syscall_windows.go, and it
   273  // does NOT pass a frame type, meaning there are no write barriers invoked. See that
   274  // call site for justification.
   275  //
   276  // Package reflect accesses this symbol through a linkname.
   277  //
   278  // Arguments passed through to reflectcall do not escape. The type is used
   279  // only in a very limited callee of reflectcall, the stackArgs are copied, and
   280  // regArgs is only used in the reflectcall frame.
   281  //
   282  //go:noescape
   283  func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   284  
   285  func procyield(cycles uint32)
   286  
   287  type neverCallThisFunction struct{}
   288  
   289  // goexit is the return stub at the top of every goroutine call stack.
   290  // Each goroutine stack is constructed as if goexit called the
   291  // goroutine's entry point function, so that when the entry point
   292  // function returns, it will return to goexit, which will call goexit1
   293  // to perform the actual exit.
   294  //
   295  // This function must never be called directly. Call goexit1 instead.
   296  // gentraceback assumes that goexit terminates the stack. A direct
   297  // call on the stack will cause gentraceback to stop walking the stack
   298  // prematurely and if there is leftover state it may panic.
   299  func goexit(neverCallThisFunction)
   300  
   301  // publicationBarrier performs a store/store barrier (a "publication"
   302  // or "export" barrier). Some form of synchronization is required
   303  // between initializing an object and making that object accessible to
   304  // another processor. Without synchronization, the initialization
   305  // writes and the "publication" write may be reordered, allowing the
   306  // other processor to follow the pointer and observe an uninitialized
   307  // object. In general, higher-level synchronization should be used,
   308  // such as locking or an atomic pointer write. publicationBarrier is
   309  // for when those aren't an option, such as in the implementation of
   310  // the memory manager.
   311  //
   312  // There's no corresponding barrier for the read side because the read
   313  // side naturally has a data dependency order. All architectures that
   314  // Go supports or seems likely to ever support automatically enforce
   315  // data dependency ordering.
   316  func publicationBarrier()
   317  
   318  // getcallerpc returns the program counter (PC) of its caller's caller.
   319  // getcallersp returns the stack pointer (SP) of its caller's caller.
   320  // The implementation may be a compiler intrinsic; there is not
   321  // necessarily code implementing this on every platform.
   322  //
   323  // For example:
   324  //
   325  //	func f(arg1, arg2, arg3 int) {
   326  //		pc := getcallerpc()
   327  //		sp := getcallersp()
   328  //	}
   329  //
   330  // These two lines find the PC and SP immediately following
   331  // the call to f (where f will return).
   332  //
   333  // The call to getcallerpc and getcallersp must be done in the
   334  // frame being asked about.
   335  //
   336  // The result of getcallersp is correct at the time of the return,
   337  // but it may be invalidated by any subsequent call to a function
   338  // that might relocate the stack in order to grow or shrink it.
   339  // A general rule is that the result of getcallersp should be used
   340  // immediately and can only be passed to nosplit functions.
   341  
   342  //go:noescape
   343  func getcallerpc() uintptr
   344  
   345  //go:noescape
   346  func getcallersp() uintptr // implemented as an intrinsic on all platforms
   347  
   348  // getclosureptr returns the pointer to the current closure.
   349  // getclosureptr can only be used in an assignment statement
   350  // at the entry of a function. Moreover, go:nosplit directive
   351  // must be specified at the declaration of caller function,
   352  // so that the function prolog does not clobber the closure register.
   353  // for example:
   354  //
   355  //	//go:nosplit
   356  //	func f(arg1, arg2, arg3 int) {
   357  //		dx := getclosureptr()
   358  //	}
   359  //
   360  // The compiler rewrites calls to this function into instructions that fetch the
   361  // pointer from a well-known register (DX on x86 architecture, etc.) directly.
   362  func getclosureptr() uintptr
   363  
   364  //go:noescape
   365  func asmcgocall(fn, arg unsafe.Pointer) int32
   366  
   367  func morestack()
   368  func morestack_noctxt()
   369  func rt0_go()
   370  
   371  // return0 is a stub used to return 0 from deferproc.
   372  // It is called at the very end of deferproc to signal
   373  // the calling Go function that it should not jump
   374  // to deferreturn.
   375  // in asm_*.s
   376  func return0()
   377  
   378  // in asm_*.s
   379  // not called directly; definitions here supply type information for traceback.
   380  // These must have the same signature (arg pointer map) as reflectcall.
   381  func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   382  func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   383  func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   384  func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   385  func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   386  func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   387  func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   388  func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   389  func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   390  func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   391  func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   392  func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   393  func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   394  func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   395  func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   396  func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   397  func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   398  func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   399  func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   400  func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   401  func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   402  func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   403  func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   404  func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   405  func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   406  func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   407  func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   408  
   409  func systemstack_switch()
   410  
   411  // alignUp rounds n up to a multiple of a. a must be a power of 2.
   412  func alignUp(n, a uintptr) uintptr {
   413  	return (n + a - 1) &^ (a - 1)
   414  }
   415  
   416  // alignDown rounds n down to a multiple of a. a must be a power of 2.
   417  func alignDown(n, a uintptr) uintptr {
   418  	return n &^ (a - 1)
   419  }
   420  
   421  // divRoundUp returns ceil(n / a).
   422  func divRoundUp(n, a uintptr) uintptr {
   423  	// a is generally a power of two. This will get inlined and
   424  	// the compiler will optimize the division.
   425  	return (n + a - 1) / a
   426  }
   427  
   428  // checkASM reports whether assembly runtime checks have passed.
   429  func checkASM() bool
   430  
   431  func memequal_varlen(a, b unsafe.Pointer) bool
   432  
   433  // bool2int returns 0 if x is false or 1 if x is true.
   434  func bool2int(x bool) int {
   435  	// Avoid branches. In the SSA compiler, this compiles to
   436  	// exactly what you would want it to.
   437  	return int(uint8(*(*uint8)(unsafe.Pointer(&x))))
   438  }
   439  
   440  // abort crashes the runtime in situations where even throw might not
   441  // work. In general it should do something a debugger will recognize
   442  // (e.g., an INT3 on x86). A crash in abort is recognized by the
   443  // signal handler, which will attempt to tear down the runtime
   444  // immediately.
   445  func abort()
   446  
   447  // Called from compiled code; declared for vet; do NOT call from Go.
   448  func gcWriteBarrier()
   449  func duffzero()
   450  func duffcopy()
   451  
   452  // Called from linker-generated .initarray; declared for go vet; do NOT call from Go.
   453  func addmoduledata()
   454  
   455  // Injected by the signal handler for panicking signals.
   456  // Initializes any registers that have fixed meaning at calls but
   457  // are scratch in bodies and calls sigpanic.
   458  // On many platforms it just jumps to sigpanic.
   459  func sigpanic0()
   460  
   461  // intArgRegs is used by the various register assignment
   462  // algorithm implementations in the runtime. These include:.
   463  // - Finalizers (mfinal.go)
   464  // - Windows callbacks (syscall_windows.go)
   465  //
   466  // Both are stripped-down versions of the algorithm since they
   467  // only have to deal with a subset of cases (finalizers only
   468  // take a pointer or interface argument, Go Windows callbacks
   469  // don't support floating point).
   470  //
   471  // It should be modified with care and are generally only
   472  // modified when testing this package.
   473  //
   474  // It should never be set higher than its internal/abi
   475  // constant counterparts, because the system relies on a
   476  // structure that is at least large enough to hold the
   477  // registers the system supports.
   478  //
   479  // Protected by finlock.
   480  var intArgRegs = abi.IntArgRegs
   481  

View as plain text