Source file src/runtime/stubs.go

     1  // Copyright 2014 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/abi"
     9  	"internal/goexperiment"
    10  	"unsafe"
    11  )
    12  
    13  // Should be a built-in for unsafe.Pointer?
    14  //go:nosplit
    15  func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
    16  	return unsafe.Pointer(uintptr(p) + x)
    17  }
    18  
    19  // getg returns the pointer to the current g.
    20  // The compiler rewrites calls to this function into instructions
    21  // that fetch the g directly (from TLS or from the dedicated register).
    22  func getg() *g
    23  
    24  // mcall switches from the g to the g0 stack and invokes fn(g),
    25  // where g is the goroutine that made the call.
    26  // mcall saves g's current PC/SP in g->sched so that it can be restored later.
    27  // It is up to fn to arrange for that later execution, typically by recording
    28  // g in a data structure, causing something to call ready(g) later.
    29  // mcall returns to the original goroutine g later, when g has been rescheduled.
    30  // fn must not return at all; typically it ends by calling schedule, to let the m
    31  // run other goroutines.
    32  //
    33  // mcall can only be called from g stacks (not g0, not gsignal).
    34  //
    35  // This must NOT be go:noescape: if fn is a stack-allocated closure,
    36  // fn puts g on a run queue, and g executes before fn returns, the
    37  // closure will be invalidated while it is still executing.
    38  func mcall(fn func(*g))
    39  
    40  // systemstack runs fn on a system stack.
    41  // If systemstack is called from the per-OS-thread (g0) stack, or
    42  // if systemstack is called from the signal handling (gsignal) stack,
    43  // systemstack calls fn directly and returns.
    44  // Otherwise, systemstack is being called from the limited stack
    45  // of an ordinary goroutine. In this case, systemstack switches
    46  // to the per-OS-thread stack, calls fn, and switches back.
    47  // It is common to use a func literal as the argument, in order
    48  // to share inputs and outputs with the code around the call
    49  // to system stack:
    50  //
    51  //	... set up y ...
    52  //	systemstack(func() {
    53  //		x = bigcall(y)
    54  //	})
    55  //	... use x ...
    56  //
    57  //go:noescape
    58  func systemstack(fn func())
    59  
    60  var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine"
    61  
    62  //go:nosplit
    63  //go:nowritebarrierrec
    64  func badsystemstack() {
    65  	sp := stringStructOf(&badsystemstackMsg)
    66  	write(2, sp.str, int32(sp.len))
    67  }
    68  
    69  // memclrNoHeapPointers clears n bytes starting at ptr.
    70  //
    71  // Usually you should use typedmemclr. memclrNoHeapPointers should be
    72  // used only when the caller knows that *ptr contains no heap pointers
    73  // because either:
    74  //
    75  // *ptr is initialized memory and its type is pointer-free, or
    76  //
    77  // *ptr is uninitialized memory (e.g., memory that's being reused
    78  // for a new allocation) and hence contains only "junk".
    79  //
    80  // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n
    81  // is a multiple of the pointer size, then any pointer-aligned,
    82  // pointer-sized portion is cleared atomically. Despite the function
    83  // name, this is necessary because this function is the underlying
    84  // implementation of typedmemclr and memclrHasPointers. See the doc of
    85  // memmove for more details.
    86  //
    87  // The (CPU-specific) implementations of this function are in memclr_*.s.
    88  //
    89  //go:noescape
    90  func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr)
    91  
    92  //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers
    93  func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) {
    94  	memclrNoHeapPointers(ptr, n)
    95  }
    96  
    97  // memmove copies n bytes from "from" to "to".
    98  //
    99  // memmove ensures that any pointer in "from" is written to "to" with
   100  // an indivisible write, so that racy reads cannot observe a
   101  // half-written pointer. This is necessary to prevent the garbage
   102  // collector from observing invalid pointers, and differs from memmove
   103  // in unmanaged languages. However, memmove is only required to do
   104  // this if "from" and "to" may contain pointers, which can only be the
   105  // case if "from", "to", and "n" are all be word-aligned.
   106  //
   107  // Implementations are in memmove_*.s.
   108  //
   109  //go:noescape
   110  func memmove(to, from unsafe.Pointer, n uintptr)
   111  
   112  // Outside assembly calls memmove. Make sure it has ABI wrappers.
   113  //go:linkname memmove
   114  
   115  //go:linkname reflect_memmove reflect.memmove
   116  func reflect_memmove(to, from unsafe.Pointer, n uintptr) {
   117  	memmove(to, from, n)
   118  }
   119  
   120  // exported value for testing
   121  var hashLoad = float32(loadFactorNum) / float32(loadFactorDen)
   122  
   123  //go:nosplit
   124  func fastrand() uint32 {
   125  	mp := getg().m
   126  	// Implement xorshift64+: 2 32-bit xorshift sequences added together.
   127  	// Shift triplet [17,7,16] was calculated as indicated in Marsaglia's
   128  	// Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf
   129  	// This generator passes the SmallCrush suite, part of TestU01 framework:
   130  	// http://simul.iro.umontreal.ca/testu01/tu01.html
   131  	s1, s0 := mp.fastrand[0], mp.fastrand[1]
   132  	s1 ^= s1 << 17
   133  	s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16
   134  	mp.fastrand[0], mp.fastrand[1] = s0, s1
   135  	return s0 + s1
   136  }
   137  
   138  //go:nosplit
   139  func fastrandn(n uint32) uint32 {
   140  	// This is similar to fastrand() % n, but faster.
   141  	// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
   142  	return uint32(uint64(fastrand()) * uint64(n) >> 32)
   143  }
   144  
   145  //go:linkname sync_fastrand sync.fastrand
   146  func sync_fastrand() uint32 { return fastrand() }
   147  
   148  //go:linkname net_fastrand net.fastrand
   149  func net_fastrand() uint32 { return fastrand() }
   150  
   151  //go:linkname os_fastrand os.fastrand
   152  func os_fastrand() uint32 { return fastrand() }
   153  
   154  // in internal/bytealg/equal_*.s
   155  //go:noescape
   156  func memequal(a, b unsafe.Pointer, size uintptr) bool
   157  
   158  // noescape hides a pointer from escape analysis.  noescape is
   159  // the identity function but escape analysis doesn't think the
   160  // output depends on the input.  noescape is inlined and currently
   161  // compiles down to zero instructions.
   162  // USE CAREFULLY!
   163  //go:nosplit
   164  func noescape(p unsafe.Pointer) unsafe.Pointer {
   165  	x := uintptr(p)
   166  	return unsafe.Pointer(x ^ 0)
   167  }
   168  
   169  // Not all cgocallback frames are actually cgocallback,
   170  // so not all have these arguments. Mark them uintptr so that the GC
   171  // does not misinterpret memory when the arguments are not present.
   172  // cgocallback is not called from Go, only from crosscall2.
   173  // This in turn calls cgocallbackg, which is where we'll find
   174  // pointer-declared arguments.
   175  func cgocallback(fn, frame, ctxt uintptr)
   176  
   177  func gogo(buf *gobuf)
   178  
   179  //go:noescape
   180  func jmpdefer(fv *funcval, argp uintptr)
   181  func asminit()
   182  func setg(gg *g)
   183  func breakpoint()
   184  
   185  // reflectcall calls fn with arguments described by stackArgs, stackArgsSize,
   186  // frameSize, and regArgs.
   187  //
   188  // Arguments passed on the stack and space for return values passed on the stack
   189  // must be laid out at the space pointed to by stackArgs (with total length
   190  // stackArgsSize) according to the ABI.
   191  //
   192  // stackRetOffset must be some value <= stackArgsSize that indicates the
   193  // offset within stackArgs where the return value space begins.
   194  //
   195  // frameSize is the total size of the argument frame at stackArgs and must
   196  // therefore be >= stackArgsSize. It must include additional space for spilling
   197  // register arguments for stack growth and preemption.
   198  //
   199  // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize,
   200  // since frameSize will be redundant with stackArgsSize.
   201  //
   202  // Arguments passed in registers must be laid out in regArgs according to the ABI.
   203  // regArgs will hold any return values passed in registers after the call.
   204  //
   205  // reflectcall copies stack arguments from stackArgs to the goroutine stack, and
   206  // then copies back stackArgsSize-stackRetOffset bytes back to the return space
   207  // in stackArgs once fn has completed. It also "unspills" argument registers from
   208  // regArgs before calling fn, and spills them back into regArgs immediately
   209  // following the call to fn. If there are results being returned on the stack,
   210  // the caller should pass the argument frame type as stackArgsType so that
   211  // reflectcall can execute appropriate write barriers during the copy.
   212  //
   213  // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which
   214  // registers on the return path will contain Go pointers. It will then store
   215  // these pointers in regArgs.Ptrs such that they are visible to the GC.
   216  //
   217  // Package reflect passes a frame type. In package runtime, there is only
   218  // one call that copies results back, in callbackWrap in syscall_windows.go, and it
   219  // does NOT pass a frame type, meaning there are no write barriers invoked. See that
   220  // call site for justification.
   221  //
   222  // Package reflect accesses this symbol through a linkname.
   223  //
   224  // Arguments passed through to reflectcall do not escape. The type is used
   225  // only in a very limited callee of reflectcall, the stackArgs are copied, and
   226  // regArgs is only used in the reflectcall frame.
   227  //go:noescape
   228  func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   229  
   230  func procyield(cycles uint32)
   231  
   232  type neverCallThisFunction struct{}
   233  
   234  // goexit is the return stub at the top of every goroutine call stack.
   235  // Each goroutine stack is constructed as if goexit called the
   236  // goroutine's entry point function, so that when the entry point
   237  // function returns, it will return to goexit, which will call goexit1
   238  // to perform the actual exit.
   239  //
   240  // This function must never be called directly. Call goexit1 instead.
   241  // gentraceback assumes that goexit terminates the stack. A direct
   242  // call on the stack will cause gentraceback to stop walking the stack
   243  // prematurely and if there is leftover state it may panic.
   244  func goexit(neverCallThisFunction)
   245  
   246  // publicationBarrier performs a store/store barrier (a "publication"
   247  // or "export" barrier). Some form of synchronization is required
   248  // between initializing an object and making that object accessible to
   249  // another processor. Without synchronization, the initialization
   250  // writes and the "publication" write may be reordered, allowing the
   251  // other processor to follow the pointer and observe an uninitialized
   252  // object. In general, higher-level synchronization should be used,
   253  // such as locking or an atomic pointer write. publicationBarrier is
   254  // for when those aren't an option, such as in the implementation of
   255  // the memory manager.
   256  //
   257  // There's no corresponding barrier for the read side because the read
   258  // side naturally has a data dependency order. All architectures that
   259  // Go supports or seems likely to ever support automatically enforce
   260  // data dependency ordering.
   261  func publicationBarrier()
   262  
   263  // getcallerpc returns the program counter (PC) of its caller's caller.
   264  // getcallersp returns the stack pointer (SP) of its caller's caller.
   265  // The implementation may be a compiler intrinsic; there is not
   266  // necessarily code implementing this on every platform.
   267  //
   268  // For example:
   269  //
   270  //	func f(arg1, arg2, arg3 int) {
   271  //		pc := getcallerpc()
   272  //		sp := getcallersp()
   273  //	}
   274  //
   275  // These two lines find the PC and SP immediately following
   276  // the call to f (where f will return).
   277  //
   278  // The call to getcallerpc and getcallersp must be done in the
   279  // frame being asked about.
   280  //
   281  // The result of getcallersp is correct at the time of the return,
   282  // but it may be invalidated by any subsequent call to a function
   283  // that might relocate the stack in order to grow or shrink it.
   284  // A general rule is that the result of getcallersp should be used
   285  // immediately and can only be passed to nosplit functions.
   286  
   287  //go:noescape
   288  func getcallerpc() uintptr
   289  
   290  //go:noescape
   291  func getcallersp() uintptr // implemented as an intrinsic on all platforms
   292  
   293  // getclosureptr returns the pointer to the current closure.
   294  // getclosureptr can only be used in an assignment statement
   295  // at the entry of a function. Moreover, go:nosplit directive
   296  // must be specified at the declaration of caller function,
   297  // so that the function prolog does not clobber the closure register.
   298  // for example:
   299  //
   300  //	//go:nosplit
   301  //	func f(arg1, arg2, arg3 int) {
   302  //		dx := getclosureptr()
   303  //	}
   304  //
   305  // The compiler rewrites calls to this function into instructions that fetch the
   306  // pointer from a well-known register (DX on x86 architecture, etc.) directly.
   307  func getclosureptr() uintptr
   308  
   309  //go:noescape
   310  func asmcgocall(fn, arg unsafe.Pointer) int32
   311  
   312  func morestack()
   313  func morestack_noctxt()
   314  func rt0_go()
   315  
   316  // return0 is a stub used to return 0 from deferproc.
   317  // It is called at the very end of deferproc to signal
   318  // the calling Go function that it should not jump
   319  // to deferreturn.
   320  // in asm_*.s
   321  func return0()
   322  
   323  // in asm_*.s
   324  // not called directly; definitions here supply type information for traceback.
   325  // These must have the same signature (arg pointer map) as reflectcall.
   326  func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   327  func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   328  func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   329  func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   330  func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   331  func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   332  func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   333  func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   334  func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   335  func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   336  func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   337  func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   338  func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   339  func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   340  func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   341  func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   342  func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   343  func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   344  func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   345  func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   346  func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   347  func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   348  func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   349  func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   350  func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   351  func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   352  func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs)
   353  
   354  func systemstack_switch()
   355  
   356  // alignUp rounds n up to a multiple of a. a must be a power of 2.
   357  func alignUp(n, a uintptr) uintptr {
   358  	return (n + a - 1) &^ (a - 1)
   359  }
   360  
   361  // alignDown rounds n down to a multiple of a. a must be a power of 2.
   362  func alignDown(n, a uintptr) uintptr {
   363  	return n &^ (a - 1)
   364  }
   365  
   366  // divRoundUp returns ceil(n / a).
   367  func divRoundUp(n, a uintptr) uintptr {
   368  	// a is generally a power of two. This will get inlined and
   369  	// the compiler will optimize the division.
   370  	return (n + a - 1) / a
   371  }
   372  
   373  // checkASM reports whether assembly runtime checks have passed.
   374  func checkASM() bool
   375  
   376  func memequal_varlen(a, b unsafe.Pointer) bool
   377  
   378  // bool2int returns 0 if x is false or 1 if x is true.
   379  func bool2int(x bool) int {
   380  	// Avoid branches. In the SSA compiler, this compiles to
   381  	// exactly what you would want it to.
   382  	return int(uint8(*(*uint8)(unsafe.Pointer(&x))))
   383  }
   384  
   385  // abort crashes the runtime in situations where even throw might not
   386  // work. In general it should do something a debugger will recognize
   387  // (e.g., an INT3 on x86). A crash in abort is recognized by the
   388  // signal handler, which will attempt to tear down the runtime
   389  // immediately.
   390  func abort()
   391  
   392  // Called from compiled code; declared for vet; do NOT call from Go.
   393  func gcWriteBarrier()
   394  func duffzero()
   395  func duffcopy()
   396  
   397  // Called from linker-generated .initarray; declared for go vet; do NOT call from Go.
   398  func addmoduledata()
   399  
   400  // Injected by the signal handler for panicking signals.
   401  // Initializes any registers that have fixed meaning at calls but
   402  // are scratch in bodies and calls sigpanic.
   403  // On many platforms it just jumps to sigpanic.
   404  func sigpanic0()
   405  
   406  // intArgRegs is used by the various register assignment
   407  // algorithm implementations in the runtime. These include:.
   408  // - Finalizers (mfinal.go)
   409  // - Windows callbacks (syscall_windows.go)
   410  //
   411  // Both are stripped-down versions of the algorithm since they
   412  // only have to deal with a subset of cases (finalizers only
   413  // take a pointer or interface argument, Go Windows callbacks
   414  // don't support floating point).
   415  //
   416  // It should be modified with care and are generally only
   417  // modified when testing this package.
   418  //
   419  // It should never be set higher than its internal/abi
   420  // constant counterparts, because the system relies on a
   421  // structure that is at least large enough to hold the
   422  // registers the system supports.
   423  //
   424  // Currently it's set to zero because using the actual
   425  // constant will break every part of the toolchain that
   426  // uses finalizers or Windows callbacks to call functions
   427  // The value that is currently commented out there should be
   428  // the actual value once we're ready to use the register ABI
   429  // everywhere.
   430  //
   431  // Protected by finlock.
   432  var intArgRegs = abi.IntArgRegs * goexperiment.RegabiArgsInt
   433  

View as plain text