Source file src/runtime/malloc_stubs.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // This file contains stub functions that are not meant to be called directly,
     6  // but that will be assembled together using the inlining logic in runtime/_mkmalloc
     7  // to produce a full mallocgc function that's specialized for a span class
     8  // or specific size in the case of the tiny allocator.
     9  //
    10  // To generate the specialized mallocgc functions, do 'go run .' inside runtime/_mkmalloc.
    11  //
    12  // To assemble a mallocgc function, the mallocStub function is cloned, and the call to
    13  // inlinedMalloc is replaced with the inlined body of smallStub or tinyStub,
    14  // depending on the parameters being specialized.
    15  //
    16  // The size_ (for the tiny case) and elemsize_, sizeclass_, noscanint_, and isNoScan_ (for all
    17  // three cases) identifiers are replaced with the value of the parameter in the specialized case.
    18  // The nextFreeFastStub, nextFreeFastTiny, heapSetTypeNoHeaderStub, and writeHeapBitsSmallStub
    19  // functions are also inlined by _mkmalloc.
    20  
    21  package runtime
    22  
    23  import (
    24  	"internal/goarch"
    25  	"internal/goexperiment"
    26  	"internal/runtime/sys"
    27  	"unsafe"
    28  )
    29  
    30  // These identifiers will all be replaced by the inliner. So their values don't
    31  // really matter: they just need to be set so that the stub functions, which
    32  // will never be used on their own, can compile. elemsize_ can't be  set to
    33  // zero because we divide by it in nextFreeFastTiny, and the compiler would
    34  // complain about a division by zero. Its replaced value will always be greater
    35  // than zero.
    36  const elemsize_ = 8
    37  const sizeclass_ = 0
    38  const noscanint_ = 0
    39  const isNoScan_ = false
    40  const size_ = 0
    41  const isTiny_ = false
    42  const isSlowPath_ = false
    43  
    44  func malloc0(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
    45  	if doubleCheckMalloc {
    46  		if gcphase == _GCmarktermination {
    47  			throw("mallocgc called with gcphase == _GCmarktermination")
    48  		}
    49  	}
    50  
    51  	// Short-circuit zero-sized allocation requests.
    52  	return unsafe.Pointer(&zerobase)
    53  }
    54  
    55  func mallocPanic(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
    56  	panic("not defined for sizeclass")
    57  }
    58  
    59  func mallocgcSlowPathStub(size uintptr, typ *_type, needzero bool, spc spanClass, elemsize uintptr) unsafe.Pointer {
    60  	return mallocStub(size, typ, needzero)
    61  }
    62  
    63  // WARNING: mallocStub does not do any work for sanitizers so callers need
    64  // to steer out of this codepath early if sanitizers are enabled.
    65  func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
    66  	if doubleCheckMalloc {
    67  		if gcphase == _GCmarktermination {
    68  			throw("mallocgc called with gcphase == _GCmarktermination")
    69  		}
    70  	}
    71  
    72  	var mp *m
    73  	if !isSlowPath_ {
    74  		// Fast path.
    75  
    76  		// The fast path assumes that GC marking is not running. We
    77  		// must acquirem to ensure the GC does not start after we
    78  		// check.
    79  		mp = acquirem()
    80  
    81  		// Do we need to fall back to slow path?
    82  		forceSlowPath := debug.malloc || gcBlackenEnabled != 0 || (goexperiment.RuntimeSecret && getg().secret > 0)
    83  
    84  		if forceSlowPath {
    85  			releasem(mp) // Slow path will reacquire.
    86  			if isTiny_ {
    87  				return mallocgcTinySlowPath(size, typ, needzero)
    88  			} else {
    89  				const spc = spanClass(sizeclass_<<1) | spanClass(noscanint_)
    90  				const elemsize = uintptr(elemsize_)
    91  				return mallocgcSlowPathStub(size, typ, needzero, spc, elemsize)
    92  			}
    93  		}
    94  
    95  		// It's possible for any malloc to trigger sweeping, which may
    96  		// in turn queue finalizers. Record this dynamic lock edge.
    97  		// N.B. Compiled away if lockrank experiment is not enabled.
    98  		lockRankMayQueueFinalizer()
    99  	} else {
   100  		// Slow path.
   101  		if isTiny_ {
   102  			// secret code, need to avoid the tiny allocator since
   103  			// it might keep co-located values alive longer and
   104  			// prevent timely zero-ing.
   105  			//
   106  			// Call directly into the NoScan allocator.
   107  			// See go.dev/issue/76356
   108  			gp := getg()
   109  			if goexperiment.RuntimeSecret && gp.secret > 0 {
   110  				return mallocgcSmallNoScanSC2(size, typ, needzero)
   111  			}
   112  		}
   113  
   114  		// It's possible for any malloc to trigger sweeping, which may
   115  		// in turn queue finalizers. Record this dynamic lock edge.
   116  		// N.B. Compiled away if lockrank experiment is not enabled.
   117  		lockRankMayQueueFinalizer()
   118  
   119  		// Pre-malloc debug hooks.
   120  		if debug.malloc {
   121  			if x := preMallocgcDebug(size, typ); x != nil {
   122  				return x
   123  			}
   124  		}
   125  
   126  		// Assist the GC if needed. (On the reuse path, we currently
   127  		// compensate for this; changes here might require changes
   128  		// there.)
   129  		if gcBlackenEnabled != 0 {
   130  			deductAssistCredit(size)
   131  		}
   132  
   133  		mp = acquirem()
   134  	}
   135  
   136  	// Actually do the allocation.
   137  	return inlinedMalloc(mp, size, typ, needzero)
   138  }
   139  
   140  func postMallocgc(x unsafe.Pointer, typ *_type, size uintptr, elemsize uintptr) {
   141  	if isSlowPath_ && !isTiny_ {
   142  		gp := getg()
   143  		if goexperiment.RuntimeSecret && gp.secret > 0 {
   144  			// Mark any object allocated while in secret mode as secret.
   145  			// This ensures we zero it immediately when freeing it.
   146  			addSecret(x, size)
   147  		}
   148  	}
   149  
   150  	// Adjust our GC assist debt to account for internal fragmentation.
   151  	if isSlowPath_ && gcBlackenEnabled != 0 && elemsize != 0 {
   152  		if assistG := getg().m.curg; assistG != nil {
   153  			assistG.gcAssistBytes -= int64(elemsize - size)
   154  		}
   155  	}
   156  
   157  	// Post-malloc debug hooks.
   158  	if isSlowPath_ && debug.malloc {
   159  		postMallocgcDebug(x, elemsize, typ)
   160  	}
   161  }
   162  
   163  // deductAssistCredit reduces the current G's GC assist credit
   164  // by size bytes, and assists the GC if necessary.
   165  //
   166  // Caller must be preemptible.
   167  //
   168  // Defined here so it can be inlined by mkmalloc.
   169  func deductAssistCredit(size uintptr) {
   170  	assistG := getg()
   171  	if assistG.m.curg != nil {
   172  		assistG = assistG.m.curg
   173  	}
   174  	assistG.gcAssistBytes -= int64(size)
   175  	if assistG.gcAssistBytes < 0 {
   176  		gcAssistAlloc(assistG)
   177  	}
   178  }
   179  
   180  // inlinedMalloc will never be called. It is defined just so that the compiler can compile
   181  // the mallocStub function, which will also never be called, but instead used as a template
   182  // to generate a size-specialized malloc function. The call to inlinedMalloc in mallocStub
   183  // will be replaced with the inlined body of smallStub or tinyStub when generating the
   184  // size-specialized malloc function. See the comment at the top of this file for more
   185  // information.
   186  //
   187  // The caller must acquirem prior to calling inlinedMalloc, which will releasem
   188  // before returning.
   189  func inlinedMalloc(mp *m, size uintptr, typ *_type, needzero bool) unsafe.Pointer {
   190  	return unsafe.Pointer(uintptr(0))
   191  }
   192  
   193  func doubleCheckSmallScanNoHeader(size uintptr, typ *_type, mp *m) {
   194  	if mp.mallocing != 0 {
   195  		throw("malloc deadlock")
   196  	}
   197  	if mp.gsignal == getg() {
   198  		throw("malloc during signal")
   199  	}
   200  	if typ == nil || !typ.Pointers() {
   201  		throw("noscan allocated in scan-only path")
   202  	}
   203  	if !heapBitsInSpan(size) {
   204  		throw("heap bits in not in span for non-header-only path")
   205  	}
   206  }
   207  
   208  // The caller must acquirem prior to calling smallStub, which will releasem
   209  // before returning.
   210  func smallStub(mp *m, size uintptr, typ *_type, needzero bool) unsafe.Pointer {
   211  	const sizeclass = sizeclass_
   212  	const elemsize = elemsize_
   213  
   214  	// Set mp.mallocing to keep from being preempted by GC.
   215  	if doubleCheckMalloc {
   216  		if isNoScan_ {
   217  			doubleCheckSmallNoScan(typ, mp)
   218  		}
   219  		if !isNoScan_ {
   220  			doubleCheckSmallScanNoHeader(size, typ, mp)
   221  		}
   222  	}
   223  	mp.mallocing = 1
   224  
   225  	checkGCTrigger := false
   226  	c := getMCache(mp)
   227  	const spc = spanClass(sizeclass<<1) | spanClass(noscanint_)
   228  	span := c.alloc[spc]
   229  
   230  	var v gclinkptr
   231  	var x unsafe.Pointer
   232  	if isNoScan_ {
   233  		// First, check for a reusable object.
   234  		if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
   235  			// We have a reusable object, use it.
   236  			x = mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
   237  			mp.mallocing = 0
   238  			releasem(mp)
   239  			if isSlowPath_ {
   240  				// postMallocgc only does anything in the slow path.
   241  				goto post
   242  			} else {
   243  				return x
   244  			}
   245  		}
   246  	}
   247  	// This is in a block so that the goto above doesn't jump past the
   248  	// definition of nextFreeFastResult that's introduced when nextFreeFastStub
   249  	// is inlined.
   250  	{
   251  		v = nextFreeFastStub(span, elemsize)
   252  		if v == 0 {
   253  			v, span, checkGCTrigger = c.nextFree(spc)
   254  		}
   255  		x = unsafe.Pointer(v)
   256  	}
   257  	if isNoScan_ {
   258  		if needzero && span.needzero != 0 {
   259  			memclrNoHeapPointers(x, elemsize)
   260  		}
   261  	}
   262  	if !isNoScan_ {
   263  		if span.needzero != 0 {
   264  			memclrNoHeapPointers(x, elemsize)
   265  		}
   266  		if goarch.PtrSize == 8 && elemsize == 8 {
   267  			// initHeapBits already set the pointer bits for the 8-byte sizeclass
   268  			// on 64-bit platforms.
   269  			c.scanAlloc += 8
   270  		} else {
   271  			dataSize := size // make the inliner happy
   272  			x := uintptr(x)
   273  			scanSize := heapSetTypeNoHeaderStub(x, dataSize, typ, span)
   274  			c.scanAlloc += scanSize
   275  		}
   276  	}
   277  
   278  	// Ensure that the stores above that initialize x to
   279  	// type-safe memory and set the heap bits occur before
   280  	// the caller can make x observable to the garbage
   281  	// collector. Otherwise, on weakly ordered machines,
   282  	// the garbage collector could follow a pointer to x,
   283  	// but see uninitialized memory or stale heap bits.
   284  	publicationBarrier()
   285  
   286  	if isSlowPath_ && writeBarrier.enabled {
   287  		// Allocate black during GC.
   288  		// All slots hold nil so no scanning is needed.
   289  		// This may be racing with GC so do it atomically if there can be
   290  		// a race marking the bit.
   291  		gcmarknewobject(span, uintptr(x))
   292  	} else {
   293  		// Track the last free index before the mark phase. This field
   294  		// is only used by the garbage collector. During the mark phase
   295  		// this is used by the conservative scanner to filter out objects
   296  		// that are both free and recently-allocated. It's safe to do that
   297  		// because we allocate-black if the GC is enabled. The conservative
   298  		// scanner produces pointers out of thin air, so without additional
   299  		// synchronization it might otherwise observe a partially-initialized
   300  		// object, which could crash the program.
   301  		span.freeIndexForScan = span.freeindex
   302  	}
   303  
   304  	// Note cache c only valid while m acquired; see #47302
   305  	//
   306  	// N.B. Use the full size because that matches how the GC
   307  	// will update the mem profile on the "free" side.
   308  	//
   309  	// TODO(mknyszek): We should really count the header as part
   310  	// of gc_sys or something. The code below just pretends it is
   311  	// internal fragmentation and matches the GC's accounting by
   312  	// using the whole allocation slot.
   313  	c.nextSample -= int64(elemsize)
   314  	if c.nextSample < 0 || MemProfileRate != c.memProfRate {
   315  		profilealloc(mp, x, elemsize)
   316  	}
   317  	mp.mallocing = 0
   318  	releasem(mp)
   319  
   320  	if checkGCTrigger {
   321  		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
   322  			gcStart(t)
   323  		}
   324  	}
   325  
   326  post:
   327  	if isSlowPath_ {
   328  		postMallocgc(x, typ, size, elemsize)
   329  	}
   330  
   331  	return x
   332  }
   333  
   334  func doubleCheckSmallNoScan(typ *_type, mp *m) {
   335  	if mp.mallocing != 0 {
   336  		throw("malloc deadlock")
   337  	}
   338  	if mp.gsignal == getg() {
   339  		throw("malloc during signal")
   340  	}
   341  	if typ != nil && typ.Pointers() {
   342  		throw("expected noscan type for noscan alloc")
   343  	}
   344  }
   345  
   346  func doubleCheckTiny(size uintptr, typ *_type, mp *m) {
   347  	if mp.mallocing != 0 {
   348  		throw("malloc deadlock")
   349  	}
   350  	if mp.gsignal == getg() {
   351  		throw("malloc during signal")
   352  	}
   353  	if typ != nil && typ.Pointers() {
   354  		throw("expected noscan for tiny alloc")
   355  	}
   356  }
   357  
   358  // The caller must acquirem prior to calling tinyStub, which will releasem
   359  // before returning.
   360  func tinyStub(mp *m, size uintptr, typ *_type, needzero bool) unsafe.Pointer {
   361  	const elemsize = elemsize_
   362  
   363  	// Set mp.mallocing to keep from being preempted by GC.
   364  	if doubleCheckMalloc {
   365  		doubleCheckTiny(size, typ, mp)
   366  	}
   367  	mp.mallocing = 1
   368  
   369  	// Tiny allocator.
   370  	//
   371  	// Tiny allocator combines several tiny allocation requests
   372  	// into a single memory block. The resulting memory block
   373  	// is freed when all subobjects are unreachable. The subobjects
   374  	// must be noscan (don't have pointers), this ensures that
   375  	// the amount of potentially wasted memory is bounded.
   376  	//
   377  	// Size of the memory block used for combining (maxTinySize) is tunable.
   378  	// Current setting is 16 bytes, which relates to 2x worst case memory
   379  	// wastage (when all but one subobjects are unreachable).
   380  	// 8 bytes would result in no wastage at all, but provides less
   381  	// opportunities for combining.
   382  	// 32 bytes provides more opportunities for combining,
   383  	// but can lead to 4x worst case wastage.
   384  	// The best case winning is 8x regardless of block size.
   385  	//
   386  	// Objects obtained from tiny allocator must not be freed explicitly.
   387  	// So when an object will be freed explicitly, we ensure that
   388  	// its size >= maxTinySize.
   389  	//
   390  	// SetFinalizer has a special case for objects potentially coming
   391  	// from tiny allocator, it such case it allows to set finalizers
   392  	// for an inner byte of a memory block.
   393  	//
   394  	// The main targets of tiny allocator are small strings and
   395  	// standalone escaping variables. On a json benchmark
   396  	// the allocator reduces number of allocations by ~12% and
   397  	// reduces heap size by ~20%.
   398  	c := getMCache(mp)
   399  	off := c.tinyoffset
   400  	// Align tiny pointer for required (conservative) alignment.
   401  	if size&7 == 0 {
   402  		off = alignUp(off, 8)
   403  	} else if goarch.PtrSize == 4 && size == 12 {
   404  		// Conservatively align 12-byte objects to 8 bytes on 32-bit
   405  		// systems so that objects whose first field is a 64-bit
   406  		// value is aligned to 8 bytes and does not cause a fault on
   407  		// atomic access. See issue 37262.
   408  		// TODO(mknyszek): Remove this workaround if/when issue 36606
   409  		// is resolved.
   410  		off = alignUp(off, 8)
   411  	} else if size&3 == 0 {
   412  		off = alignUp(off, 4)
   413  	} else if size&1 == 0 {
   414  		off = alignUp(off, 2)
   415  	}
   416  	if off+size <= maxTinySize && c.tiny != 0 {
   417  		// The object fits into existing tiny block.
   418  		x := unsafe.Pointer(c.tiny + off)
   419  		c.tinyoffset = off + size
   420  		c.tinyAllocs++
   421  		mp.mallocing = 0
   422  		releasem(mp)
   423  		const elemsize = 0
   424  		postMallocgc(x, typ, size, elemsize)
   425  		return x
   426  	}
   427  	// Allocate a new maxTinySize block.
   428  	checkGCTrigger := false
   429  	span := c.alloc[tinySpanClass]
   430  	v := nextFreeFastTiny(span)
   431  	if v == 0 {
   432  		v, span, checkGCTrigger = c.nextFree(tinySpanClass)
   433  	}
   434  	x := unsafe.Pointer(v)
   435  	(*[2]uint64)(x)[0] = 0 // Always zero
   436  	(*[2]uint64)(x)[1] = 0
   437  	// See if we need to replace the existing tiny block with the new one
   438  	// based on amount of remaining free space.
   439  	if !raceenabled && (size < c.tinyoffset || c.tiny == 0) {
   440  		// Note: disabled when race detector is on, see comment near end of this function.
   441  		c.tiny = uintptr(x)
   442  		c.tinyoffset = size
   443  	}
   444  
   445  	// Ensure that the stores above that initialize x to
   446  	// type-safe memory and set the heap bits occur before
   447  	// the caller can make x observable to the garbage
   448  	// collector. Otherwise, on weakly ordered machines,
   449  	// the garbage collector could follow a pointer to x,
   450  	// but see uninitialized memory or stale heap bits.
   451  	publicationBarrier()
   452  
   453  	if isSlowPath_ && writeBarrier.enabled {
   454  		// Allocate black during GC.
   455  		// All slots hold nil so no scanning is needed.
   456  		// This may be racing with GC so do it atomically if there can be
   457  		// a race marking the bit.
   458  		gcmarknewobject(span, uintptr(x))
   459  	} else {
   460  		// Track the last free index before the mark phase. This field
   461  		// is only used by the garbage collector. During the mark phase
   462  		// this is used by the conservative scanner to filter out objects
   463  		// that are both free and recently-allocated. It's safe to do that
   464  		// because we allocate-black if the GC is enabled. The conservative
   465  		// scanner produces pointers out of thin air, so without additional
   466  		// synchronization it might otherwise observe a partially-initialized
   467  		// object, which could crash the program.
   468  		span.freeIndexForScan = span.freeindex
   469  	}
   470  
   471  	// Note cache c only valid while m acquired; see #47302
   472  	//
   473  	// N.B. Use the full size because that matches how the GC
   474  	// will update the mem profile on the "free" side.
   475  	//
   476  	// TODO(mknyszek): We should really count the header as part
   477  	// of gc_sys or something. The code below just pretends it is
   478  	// internal fragmentation and matches the GC's accounting by
   479  	// using the whole allocation slot.
   480  	c.nextSample -= int64(elemsize)
   481  	if c.nextSample < 0 || MemProfileRate != c.memProfRate {
   482  		profilealloc(mp, x, elemsize)
   483  	}
   484  	mp.mallocing = 0
   485  	releasem(mp)
   486  
   487  	if checkGCTrigger {
   488  		if t := (gcTrigger{kind: gcTriggerHeap}); t.test() {
   489  			gcStart(t)
   490  		}
   491  	}
   492  	postMallocgc(x, typ, size, elemsize)
   493  
   494  	return x
   495  }
   496  
   497  // TODO(matloob): Should we let the go compiler inline this instead of using mkmalloc?
   498  // We won't be able to use elemsize_ but that's probably ok.
   499  func nextFreeFastTiny(span *mspan) gclinkptr {
   500  	const nbytes = 8192
   501  	const nelems = uint16((nbytes - unsafe.Sizeof(spanInlineMarkBits{})) / elemsize_)
   502  	var nextFreeFastResult gclinkptr
   503  	if span.allocCache != 0 {
   504  		theBit := sys.TrailingZeros64(span.allocCache) // Is there a free object in the allocCache?
   505  		result := span.freeindex + uint16(theBit)
   506  		if result < nelems {
   507  			freeidx := result + 1
   508  			if !(freeidx%64 == 0 && freeidx != nelems) {
   509  				span.allocCache >>= uint(theBit + 1)
   510  				span.freeindex = freeidx
   511  				span.allocCount++
   512  				nextFreeFastResult = gclinkptr(uintptr(result)*elemsize_ + span.base())
   513  			}
   514  		}
   515  	}
   516  	return nextFreeFastResult
   517  }
   518  
   519  func nextFreeFastStub(span *mspan, elemsize uintptr) gclinkptr {
   520  	var nextFreeFastResult gclinkptr
   521  	if span.allocCache != 0 {
   522  		theBit := sys.TrailingZeros64(span.allocCache) // Is there a free object in the allocCache?
   523  		result := span.freeindex + uint16(theBit)
   524  		if result < span.nelems {
   525  			freeidx := result + 1
   526  			if !(freeidx%64 == 0 && freeidx != span.nelems) {
   527  				span.allocCache >>= uint(theBit + 1)
   528  				span.freeindex = freeidx
   529  				span.allocCount++
   530  				nextFreeFastResult = gclinkptr(uintptr(result)*elemsize + span.base())
   531  			}
   532  		}
   533  	}
   534  	return nextFreeFastResult
   535  }
   536  
   537  func heapSetTypeNoHeaderStub(x, dataSize uintptr, typ *_type, span *mspan) uintptr {
   538  	if doubleCheckHeapSetType && (!heapBitsInSpan(dataSize) || !heapBitsInSpan(elemsize_)) {
   539  		throw("tried to write heap bits, but no heap bits in span")
   540  	}
   541  	scanSize := writeHeapBitsSmallStub(span, x, dataSize, typ)
   542  	if doubleCheckHeapSetType {
   543  		doubleCheckHeapType(x, dataSize, typ, nil, span)
   544  	}
   545  	return scanSize
   546  }
   547  
   548  // writeHeapBitsSmallStub writes the heap bits for small objects whose ptr/scalar data is
   549  // stored as a bitmap at the end of the span.
   550  //
   551  // Assumes dataSize is <= ptrBits*goarch.PtrSize. x must be a pointer into the span.
   552  // heapBitsInSpan(dataSize) must be true. dataSize must be >= typ.Size_.
   553  //
   554  //go:nosplit
   555  func writeHeapBitsSmallStub(span *mspan, x, dataSize uintptr, typ *_type) uintptr {
   556  	// The objects here are always really small, so a single load is sufficient.
   557  	src0 := readUintptr(getGCMask(typ))
   558  
   559  	const elemsize = elemsize_
   560  
   561  	// Create repetitions of the bitmap if we have a small slice backing store.
   562  	var scanSize uintptr
   563  	src := src0
   564  	if typ.Size_ == goarch.PtrSize {
   565  		src = (1 << (dataSize / goarch.PtrSize)) - 1
   566  		// This object is all pointers, so scanSize is just dataSize.
   567  		scanSize = dataSize
   568  	} else {
   569  		// N.B. We rely on dataSize being an exact multiple of the type size.
   570  		// The alternative is to be defensive and mask out src to the length
   571  		// of dataSize. The purpose is to save on one additional masking operation.
   572  		if doubleCheckHeapSetType && !asanenabled && dataSize%typ.Size_ != 0 {
   573  			throw("runtime: (*mspan).writeHeapBitsSmall: dataSize is not a multiple of typ.Size_")
   574  		}
   575  		scanSize = typ.PtrBytes
   576  		for i := typ.Size_; i < dataSize; i += typ.Size_ {
   577  			src |= src0 << (i / goarch.PtrSize)
   578  			scanSize += typ.Size_
   579  		}
   580  	}
   581  
   582  	// Since we're never writing more than one uintptr's worth of bits, we're either going
   583  	// to do one or two writes.
   584  	dstBase, _ := spanHeapBitsRange(span.base(), pageSize, elemsize)
   585  	dst := unsafe.Pointer(dstBase)
   586  	o := (x - span.base()) / goarch.PtrSize
   587  	i := o / ptrBits
   588  	j := o % ptrBits
   589  	var bits uintptr = elemsize / goarch.PtrSize
   590  	// In the if statement below, we have to do two uintptr writes if the bits
   591  	// we need to write straddle across two different memory locations. But if
   592  	// the number of bits we're writing divides evenly into the number of bits
   593  	// in the uintptr we're writing, this can never happen. Since bitsIsPowerOfTwo
   594  	// is a compile-time constant in the generated code, in the case where the size is
   595  	// a power of two less than or equal to ptrBits, the compiler can remove the
   596  	// 'two writes' branch of the if statement and always do only one write without
   597  	// the check.
   598  	var bitsIsPowerOfTwo = bits&(bits-1) == 0
   599  	if bits > ptrBits || (!bitsIsPowerOfTwo && j+bits > ptrBits) {
   600  		// Two writes.
   601  		bits0 := ptrBits - j
   602  		bits1 := bits - bits0
   603  		dst0 := (*uintptr)(add(dst, (i+0)*goarch.PtrSize))
   604  		dst1 := (*uintptr)(add(dst, (i+1)*goarch.PtrSize))
   605  		*dst0 = (*dst0)&(^uintptr(0)>>bits0) | (src << j)
   606  		*dst1 = (*dst1)&^((1<<bits1)-1) | (src >> bits0)
   607  	} else {
   608  		// One write.
   609  		dst := (*uintptr)(add(dst, i*goarch.PtrSize))
   610  		*dst = (*dst)&^(((1<<(min(bits, ptrBits)))-1)<<j) | (src << j) // We're taking the min so this compiles on 32 bit platforms. But if bits > ptrbits we always take the other branch
   611  	}
   612  
   613  	const doubleCheck = false
   614  	if doubleCheck {
   615  		writeHeapBitsDoubleCheck(span, x, dataSize, src, src0, i, j, bits, typ)
   616  	}
   617  	return scanSize
   618  }
   619  
   620  func writeHeapBitsDoubleCheck(span *mspan, x, dataSize, src, src0, i, j, bits uintptr, typ *_type) {
   621  	srcRead := span.heapBitsSmallForAddr(x)
   622  	if srcRead != src {
   623  		print("runtime: x=", hex(x), " i=", i, " j=", j, " bits=", bits, "\n")
   624  		print("runtime: dataSize=", dataSize, " typ.Size_=", typ.Size_, " typ.PtrBytes=", typ.PtrBytes, "\n")
   625  		print("runtime: src0=", hex(src0), " src=", hex(src), " srcRead=", hex(srcRead), "\n")
   626  		throw("bad pointer bits written for small object")
   627  	}
   628  }
   629  

View as plain text