Source file src/runtime/mgcsweep.go

     1  // Copyright 2009 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  // Garbage collector: sweeping
     6  
     7  // The sweeper consists of two different algorithms:
     8  //
     9  // * The object reclaimer finds and frees unmarked slots in spans. It
    10  //   can free a whole span if none of the objects are marked, but that
    11  //   isn't its goal. This can be driven either synchronously by
    12  //   mcentral.cacheSpan for mcentral spans, or asynchronously by
    13  //   sweepone, which looks at all the mcentral lists.
    14  //
    15  // * The span reclaimer looks for spans that contain no marked objects
    16  //   and frees whole spans. This is a separate algorithm because
    17  //   freeing whole spans is the hardest task for the object reclaimer,
    18  //   but is critical when allocating new spans. The entry point for
    19  //   this is mheap_.reclaim and it's driven by a sequential scan of
    20  //   the page marks bitmap in the heap arenas.
    21  //
    22  // Both algorithms ultimately call mspan.sweep, which sweeps a single
    23  // heap span.
    24  
    25  package runtime
    26  
    27  import (
    28  	"runtime/internal/atomic"
    29  	"unsafe"
    30  )
    31  
    32  var sweep sweepdata
    33  
    34  // State of background sweep.
    35  type sweepdata struct {
    36  	lock   mutex
    37  	g      *g
    38  	parked bool
    39  
    40  	nbgsweep    uint32
    41  	npausesweep uint32
    42  
    43  	// active tracks outstanding sweepers and the sweep
    44  	// termination condition.
    45  	active activeSweep
    46  
    47  	// centralIndex is the current unswept span class.
    48  	// It represents an index into the mcentral span
    49  	// sets. Accessed and updated via its load and
    50  	// update methods. Not protected by a lock.
    51  	//
    52  	// Reset at mark termination.
    53  	// Used by mheap.nextSpanForSweep.
    54  	centralIndex sweepClass
    55  }
    56  
    57  // sweepClass is a spanClass and one bit to represent whether we're currently
    58  // sweeping partial or full spans.
    59  type sweepClass uint32
    60  
    61  const (
    62  	numSweepClasses            = numSpanClasses * 2
    63  	sweepClassDone  sweepClass = sweepClass(^uint32(0))
    64  )
    65  
    66  func (s *sweepClass) load() sweepClass {
    67  	return sweepClass(atomic.Load((*uint32)(s)))
    68  }
    69  
    70  func (s *sweepClass) update(sNew sweepClass) {
    71  	// Only update *s if its current value is less than sNew,
    72  	// since *s increases monotonically.
    73  	sOld := s.load()
    74  	for sOld < sNew && !atomic.Cas((*uint32)(s), uint32(sOld), uint32(sNew)) {
    75  		sOld = s.load()
    76  	}
    77  	// TODO(mknyszek): This isn't the only place we have
    78  	// an atomic monotonically increasing counter. It would
    79  	// be nice to have an "atomic max" which is just implemented
    80  	// as the above on most architectures. Some architectures
    81  	// like RISC-V however have native support for an atomic max.
    82  }
    83  
    84  func (s *sweepClass) clear() {
    85  	atomic.Store((*uint32)(s), 0)
    86  }
    87  
    88  // split returns the underlying span class as well as
    89  // whether we're interested in the full or partial
    90  // unswept lists for that class, indicated as a boolean
    91  // (true means "full").
    92  func (s sweepClass) split() (spc spanClass, full bool) {
    93  	return spanClass(s >> 1), s&1 == 0
    94  }
    95  
    96  // nextSpanForSweep finds and pops the next span for sweeping from the
    97  // central sweep buffers. It returns ownership of the span to the caller.
    98  // Returns nil if no such span exists.
    99  func (h *mheap) nextSpanForSweep() *mspan {
   100  	sg := h.sweepgen
   101  	for sc := sweep.centralIndex.load(); sc < numSweepClasses; sc++ {
   102  		spc, full := sc.split()
   103  		c := &h.central[spc].mcentral
   104  		var s *mspan
   105  		if full {
   106  			s = c.fullUnswept(sg).pop()
   107  		} else {
   108  			s = c.partialUnswept(sg).pop()
   109  		}
   110  		if s != nil {
   111  			// Write down that we found something so future sweepers
   112  			// can start from here.
   113  			sweep.centralIndex.update(sc)
   114  			return s
   115  		}
   116  	}
   117  	// Write down that we found nothing.
   118  	sweep.centralIndex.update(sweepClassDone)
   119  	return nil
   120  }
   121  
   122  const sweepDrainedMask = 1 << 31
   123  
   124  // activeSweep is a type that captures whether sweeping
   125  // is done, and whether there are any outstanding sweepers.
   126  //
   127  // Every potential sweeper must call begin() before they look
   128  // for work, and end() after they've finished sweeping.
   129  type activeSweep struct {
   130  	// state is divided into two parts.
   131  	//
   132  	// The top bit (masked by sweepDrainedMask) is a boolean
   133  	// value indicating whether all the sweep work has been
   134  	// drained from the queue.
   135  	//
   136  	// The rest of the bits are a counter, indicating the
   137  	// number of outstanding concurrent sweepers.
   138  	state atomic.Uint32
   139  }
   140  
   141  // begin registers a new sweeper. Returns a sweepLocker
   142  // for acquiring spans for sweeping. Any outstanding sweeper blocks
   143  // sweep termination.
   144  //
   145  // If the sweepLocker is invalid, the caller can be sure that all
   146  // outstanding sweep work has been drained, so there is nothing left
   147  // to sweep. Note that there may be sweepers currently running, so
   148  // this does not indicate that all sweeping has completed.
   149  //
   150  // Even if the sweepLocker is invalid, its sweepGen is always valid.
   151  func (a *activeSweep) begin() sweepLocker {
   152  	for {
   153  		state := a.state.Load()
   154  		if state&sweepDrainedMask != 0 {
   155  			return sweepLocker{mheap_.sweepgen, false}
   156  		}
   157  		if a.state.CompareAndSwap(state, state+1) {
   158  			return sweepLocker{mheap_.sweepgen, true}
   159  		}
   160  	}
   161  }
   162  
   163  // end deregisters a sweeper. Must be called once for each time
   164  // begin is called if the sweepLocker is valid.
   165  func (a *activeSweep) end(sl sweepLocker) {
   166  	if sl.sweepGen != mheap_.sweepgen {
   167  		throw("sweeper left outstanding across sweep generations")
   168  	}
   169  	for {
   170  		state := a.state.Load()
   171  		if (state&^sweepDrainedMask)-1 >= sweepDrainedMask {
   172  			throw("mismatched begin/end of activeSweep")
   173  		}
   174  		if a.state.CompareAndSwap(state, state-1) {
   175  			if state != sweepDrainedMask {
   176  				return
   177  			}
   178  			if debug.gcpacertrace > 0 {
   179  				live := gcController.heapLive.Load()
   180  				print("pacer: sweep done at heap size ", live>>20, "MB; allocated ", (live-mheap_.sweepHeapLiveBasis)>>20, "MB during sweep; swept ", mheap_.pagesSwept.Load(), " pages at ", mheap_.sweepPagesPerByte, " pages/byte\n")
   181  			}
   182  			return
   183  		}
   184  	}
   185  }
   186  
   187  // markDrained marks the active sweep cycle as having drained
   188  // all remaining work. This is safe to be called concurrently
   189  // with all other methods of activeSweep, though may race.
   190  //
   191  // Returns true if this call was the one that actually performed
   192  // the mark.
   193  func (a *activeSweep) markDrained() bool {
   194  	for {
   195  		state := a.state.Load()
   196  		if state&sweepDrainedMask != 0 {
   197  			return false
   198  		}
   199  		if a.state.CompareAndSwap(state, state|sweepDrainedMask) {
   200  			return true
   201  		}
   202  	}
   203  }
   204  
   205  // sweepers returns the current number of active sweepers.
   206  func (a *activeSweep) sweepers() uint32 {
   207  	return a.state.Load() &^ sweepDrainedMask
   208  }
   209  
   210  // isDone returns true if all sweep work has been drained and no more
   211  // outstanding sweepers exist. That is, when the sweep phase is
   212  // completely done.
   213  func (a *activeSweep) isDone() bool {
   214  	return a.state.Load() == sweepDrainedMask
   215  }
   216  
   217  // reset sets up the activeSweep for the next sweep cycle.
   218  //
   219  // The world must be stopped.
   220  func (a *activeSweep) reset() {
   221  	assertWorldStopped()
   222  	a.state.Store(0)
   223  }
   224  
   225  // finishsweep_m ensures that all spans are swept.
   226  //
   227  // The world must be stopped. This ensures there are no sweeps in
   228  // progress.
   229  //
   230  //go:nowritebarrier
   231  func finishsweep_m() {
   232  	assertWorldStopped()
   233  
   234  	// Sweeping must be complete before marking commences, so
   235  	// sweep any unswept spans. If this is a concurrent GC, there
   236  	// shouldn't be any spans left to sweep, so this should finish
   237  	// instantly. If GC was forced before the concurrent sweep
   238  	// finished, there may be spans to sweep.
   239  	for sweepone() != ^uintptr(0) {
   240  		sweep.npausesweep++
   241  	}
   242  
   243  	// Make sure there aren't any outstanding sweepers left.
   244  	// At this point, with the world stopped, it means one of two
   245  	// things. Either we were able to preempt a sweeper, or that
   246  	// a sweeper didn't call sweep.active.end when it should have.
   247  	// Both cases indicate a bug, so throw.
   248  	if sweep.active.sweepers() != 0 {
   249  		throw("active sweepers found at start of mark phase")
   250  	}
   251  
   252  	// Reset all the unswept buffers, which should be empty.
   253  	// Do this in sweep termination as opposed to mark termination
   254  	// so that we can catch unswept spans and reclaim blocks as
   255  	// soon as possible.
   256  	sg := mheap_.sweepgen
   257  	for i := range mheap_.central {
   258  		c := &mheap_.central[i].mcentral
   259  		c.partialUnswept(sg).reset()
   260  		c.fullUnswept(sg).reset()
   261  	}
   262  
   263  	// Sweeping is done, so if the scavenger isn't already awake,
   264  	// wake it up. There's definitely work for it to do at this
   265  	// point.
   266  	scavenger.wake()
   267  
   268  	nextMarkBitArenaEpoch()
   269  }
   270  
   271  func bgsweep(c chan int) {
   272  	sweep.g = getg()
   273  
   274  	lockInit(&sweep.lock, lockRankSweep)
   275  	lock(&sweep.lock)
   276  	sweep.parked = true
   277  	c <- 1
   278  	goparkunlock(&sweep.lock, waitReasonGCSweepWait, traceEvGoBlock, 1)
   279  
   280  	for {
   281  		// bgsweep attempts to be a "low priority" goroutine by intentionally
   282  		// yielding time. It's OK if it doesn't run, because goroutines allocating
   283  		// memory will sweep and ensure that all spans are swept before the next
   284  		// GC cycle. We really only want to run when we're idle.
   285  		//
   286  		// However, calling Gosched after each span swept produces a tremendous
   287  		// amount of tracing events, sometimes up to 50% of events in a trace. It's
   288  		// also inefficient to call into the scheduler so much because sweeping a
   289  		// single span is in general a very fast operation, taking as little as 30 ns
   290  		// on modern hardware. (See #54767.)
   291  		//
   292  		// As a result, bgsweep sweeps in batches, and only calls into the scheduler
   293  		// at the end of every batch. Furthermore, it only yields its time if there
   294  		// isn't spare idle time available on other cores. If there's available idle
   295  		// time, helping to sweep can reduce allocation latencies by getting ahead of
   296  		// the proportional sweeper and having spans ready to go for allocation.
   297  		const sweepBatchSize = 10
   298  		nSwept := 0
   299  		for sweepone() != ^uintptr(0) {
   300  			sweep.nbgsweep++
   301  			nSwept++
   302  			if nSwept%sweepBatchSize == 0 {
   303  				goschedIfBusy()
   304  			}
   305  		}
   306  		for freeSomeWbufs(true) {
   307  			// N.B. freeSomeWbufs is already batched internally.
   308  			goschedIfBusy()
   309  		}
   310  		lock(&sweep.lock)
   311  		if !isSweepDone() {
   312  			// This can happen if a GC runs between
   313  			// gosweepone returning ^0 above
   314  			// and the lock being acquired.
   315  			unlock(&sweep.lock)
   316  			continue
   317  		}
   318  		sweep.parked = true
   319  		goparkunlock(&sweep.lock, waitReasonGCSweepWait, traceEvGoBlock, 1)
   320  	}
   321  }
   322  
   323  // sweepLocker acquires sweep ownership of spans.
   324  type sweepLocker struct {
   325  	// sweepGen is the sweep generation of the heap.
   326  	sweepGen uint32
   327  	valid    bool
   328  }
   329  
   330  // sweepLocked represents sweep ownership of a span.
   331  type sweepLocked struct {
   332  	*mspan
   333  }
   334  
   335  // tryAcquire attempts to acquire sweep ownership of span s. If it
   336  // successfully acquires ownership, it blocks sweep completion.
   337  func (l *sweepLocker) tryAcquire(s *mspan) (sweepLocked, bool) {
   338  	if !l.valid {
   339  		throw("use of invalid sweepLocker")
   340  	}
   341  	// Check before attempting to CAS.
   342  	if atomic.Load(&s.sweepgen) != l.sweepGen-2 {
   343  		return sweepLocked{}, false
   344  	}
   345  	// Attempt to acquire sweep ownership of s.
   346  	if !atomic.Cas(&s.sweepgen, l.sweepGen-2, l.sweepGen-1) {
   347  		return sweepLocked{}, false
   348  	}
   349  	return sweepLocked{s}, true
   350  }
   351  
   352  // sweepone sweeps some unswept heap span and returns the number of pages returned
   353  // to the heap, or ^uintptr(0) if there was nothing to sweep.
   354  func sweepone() uintptr {
   355  	gp := getg()
   356  
   357  	// Increment locks to ensure that the goroutine is not preempted
   358  	// in the middle of sweep thus leaving the span in an inconsistent state for next GC
   359  	gp.m.locks++
   360  
   361  	// TODO(austin): sweepone is almost always called in a loop;
   362  	// lift the sweepLocker into its callers.
   363  	sl := sweep.active.begin()
   364  	if !sl.valid {
   365  		gp.m.locks--
   366  		return ^uintptr(0)
   367  	}
   368  
   369  	// Find a span to sweep.
   370  	npages := ^uintptr(0)
   371  	var noMoreWork bool
   372  	for {
   373  		s := mheap_.nextSpanForSweep()
   374  		if s == nil {
   375  			noMoreWork = sweep.active.markDrained()
   376  			break
   377  		}
   378  		if state := s.state.get(); state != mSpanInUse {
   379  			// This can happen if direct sweeping already
   380  			// swept this span, but in that case the sweep
   381  			// generation should always be up-to-date.
   382  			if !(s.sweepgen == sl.sweepGen || s.sweepgen == sl.sweepGen+3) {
   383  				print("runtime: bad span s.state=", state, " s.sweepgen=", s.sweepgen, " sweepgen=", sl.sweepGen, "\n")
   384  				throw("non in-use span in unswept list")
   385  			}
   386  			continue
   387  		}
   388  		if s, ok := sl.tryAcquire(s); ok {
   389  			// Sweep the span we found.
   390  			npages = s.npages
   391  			if s.sweep(false) {
   392  				// Whole span was freed. Count it toward the
   393  				// page reclaimer credit since these pages can
   394  				// now be used for span allocation.
   395  				mheap_.reclaimCredit.Add(npages)
   396  			} else {
   397  				// Span is still in-use, so this returned no
   398  				// pages to the heap and the span needs to
   399  				// move to the swept in-use list.
   400  				npages = 0
   401  			}
   402  			break
   403  		}
   404  	}
   405  	sweep.active.end(sl)
   406  
   407  	if noMoreWork {
   408  		// The sweep list is empty. There may still be
   409  		// concurrent sweeps running, but we're at least very
   410  		// close to done sweeping.
   411  
   412  		// Move the scavenge gen forward (signaling
   413  		// that there's new work to do) and wake the scavenger.
   414  		//
   415  		// The scavenger is signaled by the last sweeper because once
   416  		// sweeping is done, we will definitely have useful work for
   417  		// the scavenger to do, since the scavenger only runs over the
   418  		// heap once per GC cycle. This update is not done during sweep
   419  		// termination because in some cases there may be a long delay
   420  		// between sweep done and sweep termination (e.g. not enough
   421  		// allocations to trigger a GC) which would be nice to fill in
   422  		// with scavenging work.
   423  		if debug.scavtrace > 0 {
   424  			systemstack(func() {
   425  				lock(&mheap_.lock)
   426  				released := atomic.Loaduintptr(&mheap_.pages.scav.released)
   427  				printScavTrace(released, false)
   428  				atomic.Storeuintptr(&mheap_.pages.scav.released, 0)
   429  				unlock(&mheap_.lock)
   430  			})
   431  		}
   432  		scavenger.ready()
   433  	}
   434  
   435  	gp.m.locks--
   436  	return npages
   437  }
   438  
   439  // isSweepDone reports whether all spans are swept.
   440  //
   441  // Note that this condition may transition from false to true at any
   442  // time as the sweeper runs. It may transition from true to false if a
   443  // GC runs; to prevent that the caller must be non-preemptible or must
   444  // somehow block GC progress.
   445  func isSweepDone() bool {
   446  	return sweep.active.isDone()
   447  }
   448  
   449  // Returns only when span s has been swept.
   450  //
   451  //go:nowritebarrier
   452  func (s *mspan) ensureSwept() {
   453  	// Caller must disable preemption.
   454  	// Otherwise when this function returns the span can become unswept again
   455  	// (if GC is triggered on another goroutine).
   456  	gp := getg()
   457  	if gp.m.locks == 0 && gp.m.mallocing == 0 && gp != gp.m.g0 {
   458  		throw("mspan.ensureSwept: m is not locked")
   459  	}
   460  
   461  	// If this operation fails, then that means that there are
   462  	// no more spans to be swept. In this case, either s has already
   463  	// been swept, or is about to be acquired for sweeping and swept.
   464  	sl := sweep.active.begin()
   465  	if sl.valid {
   466  		// The caller must be sure that the span is a mSpanInUse span.
   467  		if s, ok := sl.tryAcquire(s); ok {
   468  			s.sweep(false)
   469  			sweep.active.end(sl)
   470  			return
   471  		}
   472  		sweep.active.end(sl)
   473  	}
   474  
   475  	// Unfortunately we can't sweep the span ourselves. Somebody else
   476  	// got to it first. We don't have efficient means to wait, but that's
   477  	// OK, it will be swept fairly soon.
   478  	for {
   479  		spangen := atomic.Load(&s.sweepgen)
   480  		if spangen == sl.sweepGen || spangen == sl.sweepGen+3 {
   481  			break
   482  		}
   483  		osyield()
   484  	}
   485  }
   486  
   487  // Sweep frees or collects finalizers for blocks not marked in the mark phase.
   488  // It clears the mark bits in preparation for the next GC round.
   489  // Returns true if the span was returned to heap.
   490  // If preserve=true, don't return it to heap nor relink in mcentral lists;
   491  // caller takes care of it.
   492  func (sl *sweepLocked) sweep(preserve bool) bool {
   493  	// It's critical that we enter this function with preemption disabled,
   494  	// GC must not start while we are in the middle of this function.
   495  	gp := getg()
   496  	if gp.m.locks == 0 && gp.m.mallocing == 0 && gp != gp.m.g0 {
   497  		throw("mspan.sweep: m is not locked")
   498  	}
   499  
   500  	s := sl.mspan
   501  	if !preserve {
   502  		// We'll release ownership of this span. Nil it out to
   503  		// prevent the caller from accidentally using it.
   504  		sl.mspan = nil
   505  	}
   506  
   507  	sweepgen := mheap_.sweepgen
   508  	if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 {
   509  		print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
   510  		throw("mspan.sweep: bad span state")
   511  	}
   512  
   513  	if trace.enabled {
   514  		traceGCSweepSpan(s.npages * _PageSize)
   515  	}
   516  
   517  	mheap_.pagesSwept.Add(int64(s.npages))
   518  
   519  	spc := s.spanclass
   520  	size := s.elemsize
   521  
   522  	// The allocBits indicate which unmarked objects don't need to be
   523  	// processed since they were free at the end of the last GC cycle
   524  	// and were not allocated since then.
   525  	// If the allocBits index is >= s.freeindex and the bit
   526  	// is not marked then the object remains unallocated
   527  	// since the last GC.
   528  	// This situation is analogous to being on a freelist.
   529  
   530  	// Unlink & free special records for any objects we're about to free.
   531  	// Two complications here:
   532  	// 1. An object can have both finalizer and profile special records.
   533  	//    In such case we need to queue finalizer for execution,
   534  	//    mark the object as live and preserve the profile special.
   535  	// 2. A tiny object can have several finalizers setup for different offsets.
   536  	//    If such object is not marked, we need to queue all finalizers at once.
   537  	// Both 1 and 2 are possible at the same time.
   538  	hadSpecials := s.specials != nil
   539  	siter := newSpecialsIter(s)
   540  	for siter.valid() {
   541  		// A finalizer can be set for an inner byte of an object, find object beginning.
   542  		objIndex := uintptr(siter.s.offset) / size
   543  		p := s.base() + objIndex*size
   544  		mbits := s.markBitsForIndex(objIndex)
   545  		if !mbits.isMarked() {
   546  			// This object is not marked and has at least one special record.
   547  			// Pass 1: see if it has at least one finalizer.
   548  			hasFin := false
   549  			endOffset := p - s.base() + size
   550  			for tmp := siter.s; tmp != nil && uintptr(tmp.offset) < endOffset; tmp = tmp.next {
   551  				if tmp.kind == _KindSpecialFinalizer {
   552  					// Stop freeing of object if it has a finalizer.
   553  					mbits.setMarkedNonAtomic()
   554  					hasFin = true
   555  					break
   556  				}
   557  			}
   558  			// Pass 2: queue all finalizers _or_ handle profile record.
   559  			for siter.valid() && uintptr(siter.s.offset) < endOffset {
   560  				// Find the exact byte for which the special was setup
   561  				// (as opposed to object beginning).
   562  				special := siter.s
   563  				p := s.base() + uintptr(special.offset)
   564  				if special.kind == _KindSpecialFinalizer || !hasFin {
   565  					siter.unlinkAndNext()
   566  					freeSpecial(special, unsafe.Pointer(p), size)
   567  				} else {
   568  					// The object has finalizers, so we're keeping it alive.
   569  					// All other specials only apply when an object is freed,
   570  					// so just keep the special record.
   571  					siter.next()
   572  				}
   573  			}
   574  		} else {
   575  			// object is still live
   576  			if siter.s.kind == _KindSpecialReachable {
   577  				special := siter.unlinkAndNext()
   578  				(*specialReachable)(unsafe.Pointer(special)).reachable = true
   579  				freeSpecial(special, unsafe.Pointer(p), size)
   580  			} else {
   581  				// keep special record
   582  				siter.next()
   583  			}
   584  		}
   585  	}
   586  	if hadSpecials && s.specials == nil {
   587  		spanHasNoSpecials(s)
   588  	}
   589  
   590  	if debug.allocfreetrace != 0 || debug.clobberfree != 0 || raceenabled || msanenabled || asanenabled {
   591  		// Find all newly freed objects. This doesn't have to
   592  		// efficient; allocfreetrace has massive overhead.
   593  		mbits := s.markBitsForBase()
   594  		abits := s.allocBitsForIndex(0)
   595  		for i := uintptr(0); i < s.nelems; i++ {
   596  			if !mbits.isMarked() && (abits.index < s.freeindex || abits.isMarked()) {
   597  				x := s.base() + i*s.elemsize
   598  				if debug.allocfreetrace != 0 {
   599  					tracefree(unsafe.Pointer(x), size)
   600  				}
   601  				if debug.clobberfree != 0 {
   602  					clobberfree(unsafe.Pointer(x), size)
   603  				}
   604  				// User arenas are handled on explicit free.
   605  				if raceenabled && !s.isUserArenaChunk {
   606  					racefree(unsafe.Pointer(x), size)
   607  				}
   608  				if msanenabled && !s.isUserArenaChunk {
   609  					msanfree(unsafe.Pointer(x), size)
   610  				}
   611  				if asanenabled && !s.isUserArenaChunk {
   612  					asanpoison(unsafe.Pointer(x), size)
   613  				}
   614  			}
   615  			mbits.advance()
   616  			abits.advance()
   617  		}
   618  	}
   619  
   620  	// Check for zombie objects.
   621  	if s.freeindex < s.nelems {
   622  		// Everything < freeindex is allocated and hence
   623  		// cannot be zombies.
   624  		//
   625  		// Check the first bitmap byte, where we have to be
   626  		// careful with freeindex.
   627  		obj := s.freeindex
   628  		if (*s.gcmarkBits.bytep(obj / 8)&^*s.allocBits.bytep(obj / 8))>>(obj%8) != 0 {
   629  			s.reportZombies()
   630  		}
   631  		// Check remaining bytes.
   632  		for i := obj/8 + 1; i < divRoundUp(s.nelems, 8); i++ {
   633  			if *s.gcmarkBits.bytep(i)&^*s.allocBits.bytep(i) != 0 {
   634  				s.reportZombies()
   635  			}
   636  		}
   637  	}
   638  
   639  	// Count the number of free objects in this span.
   640  	nalloc := uint16(s.countAlloc())
   641  	nfreed := s.allocCount - nalloc
   642  	if nalloc > s.allocCount {
   643  		// The zombie check above should have caught this in
   644  		// more detail.
   645  		print("runtime: nelems=", s.nelems, " nalloc=", nalloc, " previous allocCount=", s.allocCount, " nfreed=", nfreed, "\n")
   646  		throw("sweep increased allocation count")
   647  	}
   648  
   649  	s.allocCount = nalloc
   650  	s.freeindex = 0 // reset allocation index to start of span.
   651  	s.freeIndexForScan = 0
   652  	if trace.enabled {
   653  		getg().m.p.ptr().traceReclaimed += uintptr(nfreed) * s.elemsize
   654  	}
   655  
   656  	// gcmarkBits becomes the allocBits.
   657  	// get a fresh cleared gcmarkBits in preparation for next GC
   658  	s.allocBits = s.gcmarkBits
   659  	s.gcmarkBits = newMarkBits(s.nelems)
   660  
   661  	// Initialize alloc bits cache.
   662  	s.refillAllocCache(0)
   663  
   664  	// The span must be in our exclusive ownership until we update sweepgen,
   665  	// check for potential races.
   666  	if state := s.state.get(); state != mSpanInUse || s.sweepgen != sweepgen-1 {
   667  		print("mspan.sweep: state=", state, " sweepgen=", s.sweepgen, " mheap.sweepgen=", sweepgen, "\n")
   668  		throw("mspan.sweep: bad span state after sweep")
   669  	}
   670  	if s.sweepgen == sweepgen+1 || s.sweepgen == sweepgen+3 {
   671  		throw("swept cached span")
   672  	}
   673  
   674  	// We need to set s.sweepgen = h.sweepgen only when all blocks are swept,
   675  	// because of the potential for a concurrent free/SetFinalizer.
   676  	//
   677  	// But we need to set it before we make the span available for allocation
   678  	// (return it to heap or mcentral), because allocation code assumes that a
   679  	// span is already swept if available for allocation.
   680  	//
   681  	// Serialization point.
   682  	// At this point the mark bits are cleared and allocation ready
   683  	// to go so release the span.
   684  	atomic.Store(&s.sweepgen, sweepgen)
   685  
   686  	if s.isUserArenaChunk {
   687  		if preserve {
   688  			// This is a case that should never be handled by a sweeper that
   689  			// preserves the span for reuse.
   690  			throw("sweep: tried to preserve a user arena span")
   691  		}
   692  		if nalloc > 0 {
   693  			// There still exist pointers into the span or the span hasn't been
   694  			// freed yet. It's not ready to be reused. Put it back on the
   695  			// full swept list for the next cycle.
   696  			mheap_.central[spc].mcentral.fullSwept(sweepgen).push(s)
   697  			return false
   698  		}
   699  
   700  		// It's only at this point that the sweeper doesn't actually need to look
   701  		// at this arena anymore, so subtract from pagesInUse now.
   702  		mheap_.pagesInUse.Add(-s.npages)
   703  		s.state.set(mSpanDead)
   704  
   705  		// The arena is ready to be recycled. Remove it from the quarantine list
   706  		// and place it on the ready list. Don't add it back to any sweep lists.
   707  		systemstack(func() {
   708  			// It's the arena code's responsibility to get the chunk on the quarantine
   709  			// list by the time all references to the chunk are gone.
   710  			if s.list != &mheap_.userArena.quarantineList {
   711  				throw("user arena span is on the wrong list")
   712  			}
   713  			lock(&mheap_.lock)
   714  			mheap_.userArena.quarantineList.remove(s)
   715  			mheap_.userArena.readyList.insert(s)
   716  			unlock(&mheap_.lock)
   717  		})
   718  		return false
   719  	}
   720  
   721  	if spc.sizeclass() != 0 {
   722  		// Handle spans for small objects.
   723  		if nfreed > 0 {
   724  			// Only mark the span as needing zeroing if we've freed any
   725  			// objects, because a fresh span that had been allocated into,
   726  			// wasn't totally filled, but then swept, still has all of its
   727  			// free slots zeroed.
   728  			s.needzero = 1
   729  			stats := memstats.heapStats.acquire()
   730  			atomic.Xadd64(&stats.smallFreeCount[spc.sizeclass()], int64(nfreed))
   731  			memstats.heapStats.release()
   732  
   733  			// Count the frees in the inconsistent, internal stats.
   734  			gcController.totalFree.Add(int64(nfreed) * int64(s.elemsize))
   735  		}
   736  		if !preserve {
   737  			// The caller may not have removed this span from whatever
   738  			// unswept set its on but taken ownership of the span for
   739  			// sweeping by updating sweepgen. If this span still is in
   740  			// an unswept set, then the mcentral will pop it off the
   741  			// set, check its sweepgen, and ignore it.
   742  			if nalloc == 0 {
   743  				// Free totally free span directly back to the heap.
   744  				mheap_.freeSpan(s)
   745  				return true
   746  			}
   747  			// Return span back to the right mcentral list.
   748  			if uintptr(nalloc) == s.nelems {
   749  				mheap_.central[spc].mcentral.fullSwept(sweepgen).push(s)
   750  			} else {
   751  				mheap_.central[spc].mcentral.partialSwept(sweepgen).push(s)
   752  			}
   753  		}
   754  	} else if !preserve {
   755  		// Handle spans for large objects.
   756  		if nfreed != 0 {
   757  			// Free large object span to heap.
   758  
   759  			// NOTE(rsc,dvyukov): The original implementation of efence
   760  			// in CL 22060046 used sysFree instead of sysFault, so that
   761  			// the operating system would eventually give the memory
   762  			// back to us again, so that an efence program could run
   763  			// longer without running out of memory. Unfortunately,
   764  			// calling sysFree here without any kind of adjustment of the
   765  			// heap data structures means that when the memory does
   766  			// come back to us, we have the wrong metadata for it, either in
   767  			// the mspan structures or in the garbage collection bitmap.
   768  			// Using sysFault here means that the program will run out of
   769  			// memory fairly quickly in efence mode, but at least it won't
   770  			// have mysterious crashes due to confused memory reuse.
   771  			// It should be possible to switch back to sysFree if we also
   772  			// implement and then call some kind of mheap.deleteSpan.
   773  			if debug.efence > 0 {
   774  				s.limit = 0 // prevent mlookup from finding this span
   775  				sysFault(unsafe.Pointer(s.base()), size)
   776  			} else {
   777  				mheap_.freeSpan(s)
   778  			}
   779  
   780  			// Count the free in the consistent, external stats.
   781  			stats := memstats.heapStats.acquire()
   782  			atomic.Xadd64(&stats.largeFreeCount, 1)
   783  			atomic.Xadd64(&stats.largeFree, int64(size))
   784  			memstats.heapStats.release()
   785  
   786  			// Count the free in the inconsistent, internal stats.
   787  			gcController.totalFree.Add(int64(size))
   788  
   789  			return true
   790  		}
   791  
   792  		// Add a large span directly onto the full+swept list.
   793  		mheap_.central[spc].mcentral.fullSwept(sweepgen).push(s)
   794  	}
   795  	return false
   796  }
   797  
   798  // reportZombies reports any marked but free objects in s and throws.
   799  //
   800  // This generally means one of the following:
   801  //
   802  // 1. User code converted a pointer to a uintptr and then back
   803  // unsafely, and a GC ran while the uintptr was the only reference to
   804  // an object.
   805  //
   806  // 2. User code (or a compiler bug) constructed a bad pointer that
   807  // points to a free slot, often a past-the-end pointer.
   808  //
   809  // 3. The GC two cycles ago missed a pointer and freed a live object,
   810  // but it was still live in the last cycle, so this GC cycle found a
   811  // pointer to that object and marked it.
   812  func (s *mspan) reportZombies() {
   813  	printlock()
   814  	print("runtime: marked free object in span ", s, ", elemsize=", s.elemsize, " freeindex=", s.freeindex, " (bad use of unsafe.Pointer? try -d=checkptr)\n")
   815  	mbits := s.markBitsForBase()
   816  	abits := s.allocBitsForIndex(0)
   817  	for i := uintptr(0); i < s.nelems; i++ {
   818  		addr := s.base() + i*s.elemsize
   819  		print(hex(addr))
   820  		alloc := i < s.freeindex || abits.isMarked()
   821  		if alloc {
   822  			print(" alloc")
   823  		} else {
   824  			print(" free ")
   825  		}
   826  		if mbits.isMarked() {
   827  			print(" marked  ")
   828  		} else {
   829  			print(" unmarked")
   830  		}
   831  		zombie := mbits.isMarked() && !alloc
   832  		if zombie {
   833  			print(" zombie")
   834  		}
   835  		print("\n")
   836  		if zombie {
   837  			length := s.elemsize
   838  			if length > 1024 {
   839  				length = 1024
   840  			}
   841  			hexdumpWords(addr, addr+length, nil)
   842  		}
   843  		mbits.advance()
   844  		abits.advance()
   845  	}
   846  	throw("found pointer to free object")
   847  }
   848  
   849  // deductSweepCredit deducts sweep credit for allocating a span of
   850  // size spanBytes. This must be performed *before* the span is
   851  // allocated to ensure the system has enough credit. If necessary, it
   852  // performs sweeping to prevent going in to debt. If the caller will
   853  // also sweep pages (e.g., for a large allocation), it can pass a
   854  // non-zero callerSweepPages to leave that many pages unswept.
   855  //
   856  // deductSweepCredit makes a worst-case assumption that all spanBytes
   857  // bytes of the ultimately allocated span will be available for object
   858  // allocation.
   859  //
   860  // deductSweepCredit is the core of the "proportional sweep" system.
   861  // It uses statistics gathered by the garbage collector to perform
   862  // enough sweeping so that all pages are swept during the concurrent
   863  // sweep phase between GC cycles.
   864  //
   865  // mheap_ must NOT be locked.
   866  func deductSweepCredit(spanBytes uintptr, callerSweepPages uintptr) {
   867  	if mheap_.sweepPagesPerByte == 0 {
   868  		// Proportional sweep is done or disabled.
   869  		return
   870  	}
   871  
   872  	if trace.enabled {
   873  		traceGCSweepStart()
   874  	}
   875  
   876  retry:
   877  	sweptBasis := mheap_.pagesSweptBasis.Load()
   878  
   879  	// Fix debt if necessary.
   880  	newHeapLive := uintptr(gcController.heapLive.Load()-mheap_.sweepHeapLiveBasis) + spanBytes
   881  	pagesTarget := int64(mheap_.sweepPagesPerByte*float64(newHeapLive)) - int64(callerSweepPages)
   882  	for pagesTarget > int64(mheap_.pagesSwept.Load()-sweptBasis) {
   883  		if sweepone() == ^uintptr(0) {
   884  			mheap_.sweepPagesPerByte = 0
   885  			break
   886  		}
   887  		if mheap_.pagesSweptBasis.Load() != sweptBasis {
   888  			// Sweep pacing changed. Recompute debt.
   889  			goto retry
   890  		}
   891  	}
   892  
   893  	if trace.enabled {
   894  		traceGCSweepDone()
   895  	}
   896  }
   897  
   898  // clobberfree sets the memory content at x to bad content, for debugging
   899  // purposes.
   900  func clobberfree(x unsafe.Pointer, size uintptr) {
   901  	// size (span.elemsize) is always a multiple of 4.
   902  	for i := uintptr(0); i < size; i += 4 {
   903  		*(*uint32)(add(x, i)) = 0xdeadbeef
   904  	}
   905  }
   906  
   907  // gcPaceSweeper updates the sweeper's pacing parameters.
   908  //
   909  // Must be called whenever the GC's pacing is updated.
   910  //
   911  // The world must be stopped, or mheap_.lock must be held.
   912  func gcPaceSweeper(trigger uint64) {
   913  	assertWorldStoppedOrLockHeld(&mheap_.lock)
   914  
   915  	// Update sweep pacing.
   916  	if isSweepDone() {
   917  		mheap_.sweepPagesPerByte = 0
   918  	} else {
   919  		// Concurrent sweep needs to sweep all of the in-use
   920  		// pages by the time the allocated heap reaches the GC
   921  		// trigger. Compute the ratio of in-use pages to sweep
   922  		// per byte allocated, accounting for the fact that
   923  		// some might already be swept.
   924  		heapLiveBasis := gcController.heapLive.Load()
   925  		heapDistance := int64(trigger) - int64(heapLiveBasis)
   926  		// Add a little margin so rounding errors and
   927  		// concurrent sweep are less likely to leave pages
   928  		// unswept when GC starts.
   929  		heapDistance -= 1024 * 1024
   930  		if heapDistance < _PageSize {
   931  			// Avoid setting the sweep ratio extremely high
   932  			heapDistance = _PageSize
   933  		}
   934  		pagesSwept := mheap_.pagesSwept.Load()
   935  		pagesInUse := mheap_.pagesInUse.Load()
   936  		sweepDistancePages := int64(pagesInUse) - int64(pagesSwept)
   937  		if sweepDistancePages <= 0 {
   938  			mheap_.sweepPagesPerByte = 0
   939  		} else {
   940  			mheap_.sweepPagesPerByte = float64(sweepDistancePages) / float64(heapDistance)
   941  			mheap_.sweepHeapLiveBasis = heapLiveBasis
   942  			// Write pagesSweptBasis last, since this
   943  			// signals concurrent sweeps to recompute
   944  			// their debt.
   945  			mheap_.pagesSweptBasis.Store(pagesSwept)
   946  		}
   947  	}
   948  }
   949  

View as plain text