Source file src/runtime/mgcpacer.go

     1  // Copyright 2021 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime
     6  
     7  import (
     8  	"internal/cpu"
     9  	"internal/goexperiment"
    10  	"runtime/internal/atomic"
    11  	"unsafe"
    12  )
    13  
    14  // go119MemoryLimitSupport is a feature flag for a number of changes
    15  // related to the memory limit feature (#48409). Disabling this flag
    16  // disables those features, as well as the memory limit mechanism,
    17  // which becomes a no-op.
    18  const go119MemoryLimitSupport = true
    19  
    20  const (
    21  	// gcGoalUtilization is the goal CPU utilization for
    22  	// marking as a fraction of GOMAXPROCS.
    23  	//
    24  	// Increasing the goal utilization will shorten GC cycles as the GC
    25  	// has more resources behind it, lessening costs from the write barrier,
    26  	// but comes at the cost of increasing mutator latency.
    27  	gcGoalUtilization = gcBackgroundUtilization
    28  
    29  	// gcBackgroundUtilization is the fixed CPU utilization for background
    30  	// marking. It must be <= gcGoalUtilization. The difference between
    31  	// gcGoalUtilization and gcBackgroundUtilization will be made up by
    32  	// mark assists. The scheduler will aim to use within 50% of this
    33  	// goal.
    34  	//
    35  	// As a general rule, there's little reason to set gcBackgroundUtilization
    36  	// < gcGoalUtilization. One reason might be in mostly idle applications,
    37  	// where goroutines are unlikely to assist at all, so the actual
    38  	// utilization will be lower than the goal. But this is moot point
    39  	// because the idle mark workers already soak up idle CPU resources.
    40  	// These two values are still kept separate however because they are
    41  	// distinct conceptually, and in previous iterations of the pacer the
    42  	// distinction was more important.
    43  	gcBackgroundUtilization = 0.25
    44  
    45  	// gcCreditSlack is the amount of scan work credit that can
    46  	// accumulate locally before updating gcController.heapScanWork and,
    47  	// optionally, gcController.bgScanCredit. Lower values give a more
    48  	// accurate assist ratio and make it more likely that assists will
    49  	// successfully steal background credit. Higher values reduce memory
    50  	// contention.
    51  	gcCreditSlack = 2000
    52  
    53  	// gcAssistTimeSlack is the nanoseconds of mutator assist time that
    54  	// can accumulate on a P before updating gcController.assistTime.
    55  	gcAssistTimeSlack = 5000
    56  
    57  	// gcOverAssistWork determines how many extra units of scan work a GC
    58  	// assist does when an assist happens. This amortizes the cost of an
    59  	// assist by pre-paying for this many bytes of future allocations.
    60  	gcOverAssistWork = 64 << 10
    61  
    62  	// defaultHeapMinimum is the value of heapMinimum for GOGC==100.
    63  	defaultHeapMinimum = (goexperiment.HeapMinimum512KiBInt)*(512<<10) +
    64  		(1-goexperiment.HeapMinimum512KiBInt)*(4<<20)
    65  
    66  	// maxStackScanSlack is the bytes of stack space allocated or freed
    67  	// that can accumulate on a P before updating gcController.stackSize.
    68  	maxStackScanSlack = 8 << 10
    69  
    70  	// memoryLimitHeapGoalHeadroom is the amount of headroom the pacer gives to
    71  	// the heap goal when operating in the memory-limited regime. That is,
    72  	// it'll reduce the heap goal by this many extra bytes off of the base
    73  	// calculation.
    74  	memoryLimitHeapGoalHeadroom = 1 << 20
    75  )
    76  
    77  func init() {
    78  	if offset := unsafe.Offsetof(gcController.heapLive); offset%8 != 0 {
    79  		println(offset)
    80  		throw("gcController.heapLive not aligned to 8 bytes")
    81  	}
    82  }
    83  
    84  // gcController implements the GC pacing controller that determines
    85  // when to trigger concurrent garbage collection and how much marking
    86  // work to do in mutator assists and background marking.
    87  //
    88  // It calculates the ratio between the allocation rate (in terms of CPU
    89  // time) and the GC scan throughput to determine the heap size at which to
    90  // trigger a GC cycle such that no GC assists are required to finish on time.
    91  // This algorithm thus optimizes GC CPU utilization to the dedicated background
    92  // mark utilization of 25% of GOMAXPROCS by minimizing GC assists.
    93  // GOMAXPROCS. The high-level design of this algorithm is documented
    94  // at https://github.com/golang/proposal/blob/master/design/44167-gc-pacer-redesign.md.
    95  // See https://golang.org/s/go15gcpacing for additional historical context.
    96  var gcController gcControllerState
    97  
    98  type gcControllerState struct {
    99  	// Initialized from GOGC. GOGC=off means no GC.
   100  	gcPercent atomic.Int32
   101  
   102  	_ uint32 // padding so following 64-bit values are 8-byte aligned
   103  
   104  	// memoryLimit is the soft memory limit in bytes.
   105  	//
   106  	// Initialized from GOMEMLIMIT. GOMEMLIMIT=off is equivalent to MaxInt64
   107  	// which means no soft memory limit in practice.
   108  	//
   109  	// This is an int64 instead of a uint64 to more easily maintain parity with
   110  	// the SetMemoryLimit API, which sets a maximum at MaxInt64. This value
   111  	// should never be negative.
   112  	memoryLimit atomic.Int64
   113  
   114  	// heapMinimum is the minimum heap size at which to trigger GC.
   115  	// For small heaps, this overrides the usual GOGC*live set rule.
   116  	//
   117  	// When there is a very small live set but a lot of allocation, simply
   118  	// collecting when the heap reaches GOGC*live results in many GC
   119  	// cycles and high total per-GC overhead. This minimum amortizes this
   120  	// per-GC overhead while keeping the heap reasonably small.
   121  	//
   122  	// During initialization this is set to 4MB*GOGC/100. In the case of
   123  	// GOGC==0, this will set heapMinimum to 0, resulting in constant
   124  	// collection even when the heap size is small, which is useful for
   125  	// debugging.
   126  	heapMinimum uint64
   127  
   128  	// runway is the amount of runway in heap bytes allocated by the
   129  	// application that we want to give the GC once it starts.
   130  	//
   131  	// This is computed from consMark during mark termination.
   132  	runway atomic.Uint64
   133  
   134  	// consMark is the estimated per-CPU consMark ratio for the application.
   135  	//
   136  	// It represents the ratio between the application's allocation
   137  	// rate, as bytes allocated per CPU-time, and the GC's scan rate,
   138  	// as bytes scanned per CPU-time.
   139  	// The units of this ratio are (B / cpu-ns) / (B / cpu-ns).
   140  	//
   141  	// At a high level, this value is computed as the bytes of memory
   142  	// allocated (cons) per unit of scan work completed (mark) in a GC
   143  	// cycle, divided by the CPU time spent on each activity.
   144  	//
   145  	// Updated at the end of each GC cycle, in endCycle.
   146  	consMark float64
   147  
   148  	// consMarkController holds the state for the mark-cons ratio
   149  	// estimation over time.
   150  	//
   151  	// Its purpose is to smooth out noisiness in the computation of
   152  	// consMark; see consMark for details.
   153  	consMarkController piController
   154  
   155  	_ uint32 // Padding for atomics on 32-bit platforms.
   156  
   157  	// gcPercentHeapGoal is the goal heapLive for when next GC ends derived
   158  	// from gcPercent.
   159  	//
   160  	// Set to ^uint64(0) if gcPercent is disabled.
   161  	gcPercentHeapGoal atomic.Uint64
   162  
   163  	// sweepDistMinTrigger is the minimum trigger to ensure a minimum
   164  	// sweep distance.
   165  	//
   166  	// This bound is also special because it applies to both the trigger
   167  	// *and* the goal (all other trigger bounds must be based *on* the goal).
   168  	//
   169  	// It is computed ahead of time, at commit time. The theory is that,
   170  	// absent a sudden change to a parameter like gcPercent, the trigger
   171  	// will be chosen to always give the sweeper enough headroom. However,
   172  	// such a change might dramatically and suddenly move up the trigger,
   173  	// in which case we need to ensure the sweeper still has enough headroom.
   174  	sweepDistMinTrigger atomic.Uint64
   175  
   176  	// triggered is the point at which the current GC cycle actually triggered.
   177  	// Only valid during the mark phase of a GC cycle, otherwise set to ^uint64(0).
   178  	//
   179  	// Updated while the world is stopped.
   180  	triggered uint64
   181  
   182  	// lastHeapGoal is the value of heapGoal at the moment the last GC
   183  	// ended. Note that this is distinct from the last value heapGoal had,
   184  	// because it could change if e.g. gcPercent changes.
   185  	//
   186  	// Read and written with the world stopped or with mheap_.lock held.
   187  	lastHeapGoal uint64
   188  
   189  	// heapLive is the number of bytes considered live by the GC.
   190  	// That is: retained by the most recent GC plus allocated
   191  	// since then. heapLive ≤ memstats.totalAlloc-memstats.totalFree, since
   192  	// heapAlloc includes unmarked objects that have not yet been swept (and
   193  	// hence goes up as we allocate and down as we sweep) while heapLive
   194  	// excludes these objects (and hence only goes up between GCs).
   195  	//
   196  	// This is updated atomically without locking. To reduce
   197  	// contention, this is updated only when obtaining a span from
   198  	// an mcentral and at this point it counts all of the
   199  	// unallocated slots in that span (which will be allocated
   200  	// before that mcache obtains another span from that
   201  	// mcentral). Hence, it slightly overestimates the "true" live
   202  	// heap size. It's better to overestimate than to
   203  	// underestimate because 1) this triggers the GC earlier than
   204  	// necessary rather than potentially too late and 2) this
   205  	// leads to a conservative GC rate rather than a GC rate that
   206  	// is potentially too low.
   207  	//
   208  	// Reads should likewise be atomic (or during STW).
   209  	//
   210  	// Whenever this is updated, call traceHeapAlloc() and
   211  	// this gcControllerState's revise() method.
   212  	heapLive uint64
   213  
   214  	// heapScan is the number of bytes of "scannable" heap. This
   215  	// is the live heap (as counted by heapLive), but omitting
   216  	// no-scan objects and no-scan tails of objects.
   217  	//
   218  	// This value is fixed at the start of a GC cycle, so during a
   219  	// GC cycle it is safe to read without atomics, and it represents
   220  	// the maximum scannable heap.
   221  	heapScan uint64
   222  
   223  	// lastHeapScan is the number of bytes of heap that were scanned
   224  	// last GC cycle. It is the same as heapMarked, but only
   225  	// includes the "scannable" parts of objects.
   226  	//
   227  	// Updated when the world is stopped.
   228  	lastHeapScan uint64
   229  
   230  	// lastStackScan is the number of bytes of stack that were scanned
   231  	// last GC cycle.
   232  	lastStackScan uint64
   233  
   234  	// maxStackScan is the amount of allocated goroutine stack space in
   235  	// use by goroutines.
   236  	//
   237  	// This number tracks allocated goroutine stack space rather than used
   238  	// goroutine stack space (i.e. what is actually scanned) because used
   239  	// goroutine stack space is much harder to measure cheaply. By using
   240  	// allocated space, we make an overestimate; this is OK, it's better
   241  	// to conservatively overcount than undercount.
   242  	//
   243  	// Read and updated atomically.
   244  	maxStackScan uint64
   245  
   246  	// globalsScan is the total amount of global variable space
   247  	// that is scannable.
   248  	//
   249  	// Read and updated atomically.
   250  	globalsScan uint64
   251  
   252  	// heapMarked is the number of bytes marked by the previous
   253  	// GC. After mark termination, heapLive == heapMarked, but
   254  	// unlike heapLive, heapMarked does not change until the
   255  	// next mark termination.
   256  	heapMarked uint64
   257  
   258  	// heapScanWork is the total heap scan work performed this cycle.
   259  	// stackScanWork is the total stack scan work performed this cycle.
   260  	// globalsScanWork is the total globals scan work performed this cycle.
   261  	//
   262  	// These are updated atomically during the cycle. Updates occur in
   263  	// bounded batches, since they are both written and read
   264  	// throughout the cycle. At the end of the cycle, heapScanWork is how
   265  	// much of the retained heap is scannable.
   266  	//
   267  	// Currently these are measured in bytes. For most uses, this is an
   268  	// opaque unit of work, but for estimation the definition is important.
   269  	//
   270  	// Note that stackScanWork includes only stack space scanned, not all
   271  	// of the allocated stack.
   272  	heapScanWork    atomic.Int64
   273  	stackScanWork   atomic.Int64
   274  	globalsScanWork atomic.Int64
   275  
   276  	// bgScanCredit is the scan work credit accumulated by the
   277  	// concurrent background scan. This credit is accumulated by
   278  	// the background scan and stolen by mutator assists. This is
   279  	// updated atomically. Updates occur in bounded batches, since
   280  	// it is both written and read throughout the cycle.
   281  	bgScanCredit int64
   282  
   283  	// assistTime is the nanoseconds spent in mutator assists
   284  	// during this cycle. This is updated atomically, and must also
   285  	// be updated atomically even during a STW, because it is read
   286  	// by sysmon. Updates occur in bounded batches, since it is both
   287  	// written and read throughout the cycle.
   288  	assistTime atomic.Int64
   289  
   290  	// dedicatedMarkTime is the nanoseconds spent in dedicated
   291  	// mark workers during this cycle. This is updated atomically
   292  	// at the end of the concurrent mark phase.
   293  	dedicatedMarkTime int64
   294  
   295  	// fractionalMarkTime is the nanoseconds spent in the
   296  	// fractional mark worker during this cycle. This is updated
   297  	// atomically throughout the cycle and will be up-to-date if
   298  	// the fractional mark worker is not currently running.
   299  	fractionalMarkTime int64
   300  
   301  	// idleMarkTime is the nanoseconds spent in idle marking
   302  	// during this cycle. This is updated atomically throughout
   303  	// the cycle.
   304  	idleMarkTime int64
   305  
   306  	// markStartTime is the absolute start time in nanoseconds
   307  	// that assists and background mark workers started.
   308  	markStartTime int64
   309  
   310  	// dedicatedMarkWorkersNeeded is the number of dedicated mark
   311  	// workers that need to be started. This is computed at the
   312  	// beginning of each cycle and decremented atomically as
   313  	// dedicated mark workers get started.
   314  	dedicatedMarkWorkersNeeded int64
   315  
   316  	// idleMarkWorkers is two packed int32 values in a single uint64.
   317  	// These two values are always updated simultaneously.
   318  	//
   319  	// The bottom int32 is the current number of idle mark workers executing.
   320  	//
   321  	// The top int32 is the maximum number of idle mark workers allowed to
   322  	// execute concurrently. Normally, this number is just gomaxprocs. However,
   323  	// during periodic GC cycles it is set to 0 because the system is idle
   324  	// anyway; there's no need to go full blast on all of GOMAXPROCS.
   325  	//
   326  	// The maximum number of idle mark workers is used to prevent new workers
   327  	// from starting, but it is not a hard maximum. It is possible (but
   328  	// exceedingly rare) for the current number of idle mark workers to
   329  	// transiently exceed the maximum. This could happen if the maximum changes
   330  	// just after a GC ends, and an M with no P.
   331  	//
   332  	// Note that if we have no dedicated mark workers, we set this value to
   333  	// 1 in this case we only have fractional GC workers which aren't scheduled
   334  	// strictly enough to ensure GC progress. As a result, idle-priority mark
   335  	// workers are vital to GC progress in these situations.
   336  	//
   337  	// For example, consider a situation in which goroutines block on the GC
   338  	// (such as via runtime.GOMAXPROCS) and only fractional mark workers are
   339  	// scheduled (e.g. GOMAXPROCS=1). Without idle-priority mark workers, the
   340  	// last running M might skip scheduling a fractional mark worker if its
   341  	// utilization goal is met, such that once it goes to sleep (because there's
   342  	// nothing to do), there will be nothing else to spin up a new M for the
   343  	// fractional worker in the future, stalling GC progress and causing a
   344  	// deadlock. However, idle-priority workers will *always* run when there is
   345  	// nothing left to do, ensuring the GC makes progress.
   346  	//
   347  	// See github.com/golang/go/issues/44163 for more details.
   348  	idleMarkWorkers atomic.Uint64
   349  
   350  	// assistWorkPerByte is the ratio of scan work to allocated
   351  	// bytes that should be performed by mutator assists. This is
   352  	// computed at the beginning of each cycle and updated every
   353  	// time heapScan is updated.
   354  	assistWorkPerByte atomic.Float64
   355  
   356  	// assistBytesPerWork is 1/assistWorkPerByte.
   357  	//
   358  	// Note that because this is read and written independently
   359  	// from assistWorkPerByte users may notice a skew between
   360  	// the two values, and such a state should be safe.
   361  	assistBytesPerWork atomic.Float64
   362  
   363  	// fractionalUtilizationGoal is the fraction of wall clock
   364  	// time that should be spent in the fractional mark worker on
   365  	// each P that isn't running a dedicated worker.
   366  	//
   367  	// For example, if the utilization goal is 25% and there are
   368  	// no dedicated workers, this will be 0.25. If the goal is
   369  	// 25%, there is one dedicated worker, and GOMAXPROCS is 5,
   370  	// this will be 0.05 to make up the missing 5%.
   371  	//
   372  	// If this is zero, no fractional workers are needed.
   373  	fractionalUtilizationGoal float64
   374  
   375  	// These memory stats are effectively duplicates of fields from
   376  	// memstats.heapStats but are updated atomically or with the world
   377  	// stopped and don't provide the same consistency guarantees.
   378  	//
   379  	// Because the runtime is responsible for managing a memory limit, it's
   380  	// useful to couple these stats more tightly to the gcController, which
   381  	// is intimately connected to how that memory limit is maintained.
   382  	heapInUse    sysMemStat    // bytes in mSpanInUse spans
   383  	heapReleased sysMemStat    // bytes released to the OS
   384  	heapFree     sysMemStat    // bytes not in any span, but not released to the OS
   385  	totalAlloc   atomic.Uint64 // total bytes allocated
   386  	totalFree    atomic.Uint64 // total bytes freed
   387  	mappedReady  atomic.Uint64 // total virtual memory in the Ready state (see mem.go).
   388  
   389  	// test indicates that this is a test-only copy of gcControllerState.
   390  	test bool
   391  
   392  	_ cpu.CacheLinePad
   393  }
   394  
   395  func (c *gcControllerState) init(gcPercent int32, memoryLimit int64) {
   396  	c.heapMinimum = defaultHeapMinimum
   397  	c.triggered = ^uint64(0)
   398  
   399  	c.consMarkController = piController{
   400  		// Tuned first via the Ziegler-Nichols process in simulation,
   401  		// then the integral time was manually tuned against real-world
   402  		// applications to deal with noisiness in the measured cons/mark
   403  		// ratio.
   404  		kp: 0.9,
   405  		ti: 4.0,
   406  
   407  		// Set a high reset time in GC cycles.
   408  		// This is inversely proportional to the rate at which we
   409  		// accumulate error from clipping. By making this very high
   410  		// we make the accumulation slow. In general, clipping is
   411  		// OK in our situation, hence the choice.
   412  		//
   413  		// Tune this if we get unintended effects from clipping for
   414  		// a long time.
   415  		tt:  1000,
   416  		min: -1000,
   417  		max: 1000,
   418  	}
   419  
   420  	c.setGCPercent(gcPercent)
   421  	c.setMemoryLimit(memoryLimit)
   422  	c.commit(true) // No sweep phase in the first GC cycle.
   423  	// N.B. Don't bother calling traceHeapGoal. Tracing is never enabled at
   424  	// initialization time.
   425  	// N.B. No need to call revise; there's no GC enabled during
   426  	// initialization.
   427  }
   428  
   429  // startCycle resets the GC controller's state and computes estimates
   430  // for a new GC cycle. The caller must hold worldsema and the world
   431  // must be stopped.
   432  func (c *gcControllerState) startCycle(markStartTime int64, procs int, trigger gcTrigger) {
   433  	c.heapScanWork.Store(0)
   434  	c.stackScanWork.Store(0)
   435  	c.globalsScanWork.Store(0)
   436  	c.bgScanCredit = 0
   437  	c.assistTime.Store(0)
   438  	c.dedicatedMarkTime = 0
   439  	c.fractionalMarkTime = 0
   440  	c.idleMarkTime = 0
   441  	c.markStartTime = markStartTime
   442  
   443  	// TODO(mknyszek): This is supposed to be the actual trigger point for the heap, but
   444  	// causes regressions in memory use. The cause is that the PI controller used to smooth
   445  	// the cons/mark ratio measurements tends to flail when using the less accurate precomputed
   446  	// trigger for the cons/mark calculation, and this results in the controller being more
   447  	// conservative about steady-states it tries to find in the future.
   448  	//
   449  	// This conservatism is transient, but these transient states tend to matter for short-lived
   450  	// programs, especially because the PI controller is overdamped, partially because it is
   451  	// configured with a relatively large time constant.
   452  	//
   453  	// Ultimately, I think this is just two mistakes piled on one another: the choice of a swingy
   454  	// smoothing function that recalls a fairly long history (due to its overdamped time constant)
   455  	// coupled with an inaccurate cons/mark calculation. It just so happens this works better
   456  	// today, and it makes it harder to change things in the future.
   457  	//
   458  	// This is described in #53738. Fix this for #53892 by changing back to the actual trigger
   459  	// point and simplifying the smoothing function.
   460  	heapTrigger, heapGoal := c.trigger()
   461  	c.triggered = heapTrigger
   462  
   463  	// Compute the background mark utilization goal. In general,
   464  	// this may not come out exactly. We round the number of
   465  	// dedicated workers so that the utilization is closest to
   466  	// 25%. For small GOMAXPROCS, this would introduce too much
   467  	// error, so we add fractional workers in that case.
   468  	totalUtilizationGoal := float64(procs) * gcBackgroundUtilization
   469  	c.dedicatedMarkWorkersNeeded = int64(totalUtilizationGoal + 0.5)
   470  	utilError := float64(c.dedicatedMarkWorkersNeeded)/totalUtilizationGoal - 1
   471  	const maxUtilError = 0.3
   472  	if utilError < -maxUtilError || utilError > maxUtilError {
   473  		// Rounding put us more than 30% off our goal. With
   474  		// gcBackgroundUtilization of 25%, this happens for
   475  		// GOMAXPROCS<=3 or GOMAXPROCS=6. Enable fractional
   476  		// workers to compensate.
   477  		if float64(c.dedicatedMarkWorkersNeeded) > totalUtilizationGoal {
   478  			// Too many dedicated workers.
   479  			c.dedicatedMarkWorkersNeeded--
   480  		}
   481  		c.fractionalUtilizationGoal = (totalUtilizationGoal - float64(c.dedicatedMarkWorkersNeeded)) / float64(procs)
   482  	} else {
   483  		c.fractionalUtilizationGoal = 0
   484  	}
   485  
   486  	// In STW mode, we just want dedicated workers.
   487  	if debug.gcstoptheworld > 0 {
   488  		c.dedicatedMarkWorkersNeeded = int64(procs)
   489  		c.fractionalUtilizationGoal = 0
   490  	}
   491  
   492  	// Clear per-P state
   493  	for _, p := range allp {
   494  		p.gcAssistTime = 0
   495  		p.gcFractionalMarkTime = 0
   496  	}
   497  
   498  	if trigger.kind == gcTriggerTime {
   499  		// During a periodic GC cycle, reduce the number of idle mark workers
   500  		// required. However, we need at least one dedicated mark worker or
   501  		// idle GC worker to ensure GC progress in some scenarios (see comment
   502  		// on maxIdleMarkWorkers).
   503  		if c.dedicatedMarkWorkersNeeded > 0 {
   504  			c.setMaxIdleMarkWorkers(0)
   505  		} else {
   506  			// TODO(mknyszek): The fundamental reason why we need this is because
   507  			// we can't count on the fractional mark worker to get scheduled.
   508  			// Fix that by ensuring it gets scheduled according to its quota even
   509  			// if the rest of the application is idle.
   510  			c.setMaxIdleMarkWorkers(1)
   511  		}
   512  	} else {
   513  		// N.B. gomaxprocs and dedicatedMarkWorkersNeeded is guaranteed not to
   514  		// change during a GC cycle.
   515  		c.setMaxIdleMarkWorkers(int32(procs) - int32(c.dedicatedMarkWorkersNeeded))
   516  	}
   517  
   518  	// Compute initial values for controls that are updated
   519  	// throughout the cycle.
   520  	c.revise()
   521  
   522  	if debug.gcpacertrace > 0 {
   523  		assistRatio := c.assistWorkPerByte.Load()
   524  		print("pacer: assist ratio=", assistRatio,
   525  			" (scan ", gcController.heapScan>>20, " MB in ",
   526  			work.initialHeapLive>>20, "->",
   527  			heapGoal>>20, " MB)",
   528  			" workers=", c.dedicatedMarkWorkersNeeded,
   529  			"+", c.fractionalUtilizationGoal, "\n")
   530  	}
   531  }
   532  
   533  // revise updates the assist ratio during the GC cycle to account for
   534  // improved estimates. This should be called whenever gcController.heapScan,
   535  // gcController.heapLive, or if any inputs to gcController.heapGoal are
   536  // updated. It is safe to call concurrently, but it may race with other
   537  // calls to revise.
   538  //
   539  // The result of this race is that the two assist ratio values may not line
   540  // up or may be stale. In practice this is OK because the assist ratio
   541  // moves slowly throughout a GC cycle, and the assist ratio is a best-effort
   542  // heuristic anyway. Furthermore, no part of the heuristic depends on
   543  // the two assist ratio values being exact reciprocals of one another, since
   544  // the two values are used to convert values from different sources.
   545  //
   546  // The worst case result of this raciness is that we may miss a larger shift
   547  // in the ratio (say, if we decide to pace more aggressively against the
   548  // hard heap goal) but even this "hard goal" is best-effort (see #40460).
   549  // The dedicated GC should ensure we don't exceed the hard goal by too much
   550  // in the rare case we do exceed it.
   551  //
   552  // It should only be called when gcBlackenEnabled != 0 (because this
   553  // is when assists are enabled and the necessary statistics are
   554  // available).
   555  func (c *gcControllerState) revise() {
   556  	gcPercent := c.gcPercent.Load()
   557  	if gcPercent < 0 {
   558  		// If GC is disabled but we're running a forced GC,
   559  		// act like GOGC is huge for the below calculations.
   560  		gcPercent = 100000
   561  	}
   562  	live := atomic.Load64(&c.heapLive)
   563  	scan := atomic.Load64(&c.heapScan)
   564  	work := c.heapScanWork.Load() + c.stackScanWork.Load() + c.globalsScanWork.Load()
   565  
   566  	// Assume we're under the soft goal. Pace GC to complete at
   567  	// heapGoal assuming the heap is in steady-state.
   568  	heapGoal := int64(c.heapGoal())
   569  
   570  	// The expected scan work is computed as the amount of bytes scanned last
   571  	// GC cycle (both heap and stack), plus our estimate of globals work for this cycle.
   572  	scanWorkExpected := int64(c.lastHeapScan + c.lastStackScan + c.globalsScan)
   573  
   574  	// maxScanWork is a worst-case estimate of the amount of scan work that
   575  	// needs to be performed in this GC cycle. Specifically, it represents
   576  	// the case where *all* scannable memory turns out to be live, and
   577  	// *all* allocated stack space is scannable.
   578  	maxStackScan := atomic.Load64(&c.maxStackScan)
   579  	maxScanWork := int64(scan + maxStackScan + c.globalsScan)
   580  	if work > scanWorkExpected {
   581  		// We've already done more scan work than expected. Because our expectation
   582  		// is based on a steady-state scannable heap size, we assume this means our
   583  		// heap is growing. Compute a new heap goal that takes our existing runway
   584  		// computed for scanWorkExpected and extrapolates it to maxScanWork, the worst-case
   585  		// scan work. This keeps our assist ratio stable if the heap continues to grow.
   586  		//
   587  		// The effect of this mechanism is that assists stay flat in the face of heap
   588  		// growths. It's OK to use more memory this cycle to scan all the live heap,
   589  		// because the next GC cycle is inevitably going to use *at least* that much
   590  		// memory anyway.
   591  		extHeapGoal := int64(float64(heapGoal-int64(c.triggered))/float64(scanWorkExpected)*float64(maxScanWork)) + int64(c.triggered)
   592  		scanWorkExpected = maxScanWork
   593  
   594  		// hardGoal is a hard limit on the amount that we're willing to push back the
   595  		// heap goal, and that's twice the heap goal (i.e. if GOGC=100 and the heap and/or
   596  		// stacks and/or globals grow to twice their size, this limits the current GC cycle's
   597  		// growth to 4x the original live heap's size).
   598  		//
   599  		// This maintains the invariant that we use no more memory than the next GC cycle
   600  		// will anyway.
   601  		hardGoal := int64((1.0 + float64(gcPercent)/100.0) * float64(heapGoal))
   602  		if extHeapGoal > hardGoal {
   603  			extHeapGoal = hardGoal
   604  		}
   605  		heapGoal = extHeapGoal
   606  	}
   607  	if int64(live) > heapGoal {
   608  		// We're already past our heap goal, even the extrapolated one.
   609  		// Leave ourselves some extra runway, so in the worst case we
   610  		// finish by that point.
   611  		const maxOvershoot = 1.1
   612  		heapGoal = int64(float64(heapGoal) * maxOvershoot)
   613  
   614  		// Compute the upper bound on the scan work remaining.
   615  		scanWorkExpected = maxScanWork
   616  	}
   617  
   618  	// Compute the remaining scan work estimate.
   619  	//
   620  	// Note that we currently count allocations during GC as both
   621  	// scannable heap (heapScan) and scan work completed
   622  	// (scanWork), so allocation will change this difference
   623  	// slowly in the soft regime and not at all in the hard
   624  	// regime.
   625  	scanWorkRemaining := scanWorkExpected - work
   626  	if scanWorkRemaining < 1000 {
   627  		// We set a somewhat arbitrary lower bound on
   628  		// remaining scan work since if we aim a little high,
   629  		// we can miss by a little.
   630  		//
   631  		// We *do* need to enforce that this is at least 1,
   632  		// since marking is racy and double-scanning objects
   633  		// may legitimately make the remaining scan work
   634  		// negative, even in the hard goal regime.
   635  		scanWorkRemaining = 1000
   636  	}
   637  
   638  	// Compute the heap distance remaining.
   639  	heapRemaining := heapGoal - int64(live)
   640  	if heapRemaining <= 0 {
   641  		// This shouldn't happen, but if it does, avoid
   642  		// dividing by zero or setting the assist negative.
   643  		heapRemaining = 1
   644  	}
   645  
   646  	// Compute the mutator assist ratio so by the time the mutator
   647  	// allocates the remaining heap bytes up to heapGoal, it will
   648  	// have done (or stolen) the remaining amount of scan work.
   649  	// Note that the assist ratio values are updated atomically
   650  	// but not together. This means there may be some degree of
   651  	// skew between the two values. This is generally OK as the
   652  	// values shift relatively slowly over the course of a GC
   653  	// cycle.
   654  	assistWorkPerByte := float64(scanWorkRemaining) / float64(heapRemaining)
   655  	assistBytesPerWork := float64(heapRemaining) / float64(scanWorkRemaining)
   656  	c.assistWorkPerByte.Store(assistWorkPerByte)
   657  	c.assistBytesPerWork.Store(assistBytesPerWork)
   658  }
   659  
   660  // endCycle computes the consMark estimate for the next cycle.
   661  // userForced indicates whether the current GC cycle was forced
   662  // by the application.
   663  func (c *gcControllerState) endCycle(now int64, procs int, userForced bool) {
   664  	// Record last heap goal for the scavenger.
   665  	// We'll be updating the heap goal soon.
   666  	gcController.lastHeapGoal = c.heapGoal()
   667  
   668  	// Compute the duration of time for which assists were turned on.
   669  	assistDuration := now - c.markStartTime
   670  
   671  	// Assume background mark hit its utilization goal.
   672  	utilization := gcBackgroundUtilization
   673  	// Add assist utilization; avoid divide by zero.
   674  	if assistDuration > 0 {
   675  		utilization += float64(c.assistTime.Load()) / float64(assistDuration*int64(procs))
   676  	}
   677  
   678  	if c.heapLive <= c.triggered {
   679  		// Shouldn't happen, but let's be very safe about this in case the
   680  		// GC is somehow extremely short.
   681  		//
   682  		// In this case though, the only reasonable value for c.heapLive-c.triggered
   683  		// would be 0, which isn't really all that useful, i.e. the GC was so short
   684  		// that it didn't matter.
   685  		//
   686  		// Ignore this case and don't update anything.
   687  		return
   688  	}
   689  	idleUtilization := 0.0
   690  	if assistDuration > 0 {
   691  		idleUtilization = float64(c.idleMarkTime) / float64(assistDuration*int64(procs))
   692  	}
   693  	// Determine the cons/mark ratio.
   694  	//
   695  	// The units we want for the numerator and denominator are both B / cpu-ns.
   696  	// We get this by taking the bytes allocated or scanned, and divide by the amount of
   697  	// CPU time it took for those operations. For allocations, that CPU time is
   698  	//
   699  	//    assistDuration * procs * (1 - utilization)
   700  	//
   701  	// Where utilization includes just background GC workers and assists. It does *not*
   702  	// include idle GC work time, because in theory the mutator is free to take that at
   703  	// any point.
   704  	//
   705  	// For scanning, that CPU time is
   706  	//
   707  	//    assistDuration * procs * (utilization + idleUtilization)
   708  	//
   709  	// In this case, we *include* idle utilization, because that is additional CPU time that the
   710  	// the GC had available to it.
   711  	//
   712  	// In effect, idle GC time is sort of double-counted here, but it's very weird compared
   713  	// to other kinds of GC work, because of how fluid it is. Namely, because the mutator is
   714  	// *always* free to take it.
   715  	//
   716  	// So this calculation is really:
   717  	//     (heapLive-trigger) / (assistDuration * procs * (1-utilization)) /
   718  	//         (scanWork) / (assistDuration * procs * (utilization+idleUtilization)
   719  	//
   720  	// Note that because we only care about the ratio, assistDuration and procs cancel out.
   721  	scanWork := c.heapScanWork.Load() + c.stackScanWork.Load() + c.globalsScanWork.Load()
   722  	currentConsMark := (float64(c.heapLive-c.triggered) * (utilization + idleUtilization)) /
   723  		(float64(scanWork) * (1 - utilization))
   724  
   725  	// Update cons/mark controller. The time period for this is 1 GC cycle.
   726  	//
   727  	// This use of a PI controller might seem strange. So, here's an explanation:
   728  	//
   729  	// currentConsMark represents the consMark we *should've* had to be perfectly
   730  	// on-target for this cycle. Given that we assume the next GC will be like this
   731  	// one in the steady-state, it stands to reason that we should just pick that
   732  	// as our next consMark. In practice, however, currentConsMark is too noisy:
   733  	// we're going to be wildly off-target in each GC cycle if we do that.
   734  	//
   735  	// What we do instead is make a long-term assumption: there is some steady-state
   736  	// consMark value, but it's obscured by noise. By constantly shooting for this
   737  	// noisy-but-perfect consMark value, the controller will bounce around a bit,
   738  	// but its average behavior, in aggregate, should be less noisy and closer to
   739  	// the true long-term consMark value, provided its tuned to be slightly overdamped.
   740  	var ok bool
   741  	oldConsMark := c.consMark
   742  	c.consMark, ok = c.consMarkController.next(c.consMark, currentConsMark, 1.0)
   743  	if !ok {
   744  		// The error spiraled out of control. This is incredibly unlikely seeing
   745  		// as this controller is essentially just a smoothing function, but it might
   746  		// mean that something went very wrong with how currentConsMark was calculated.
   747  		// Just reset consMark and keep going.
   748  		c.consMark = 0
   749  	}
   750  
   751  	if debug.gcpacertrace > 0 {
   752  		printlock()
   753  		goal := gcGoalUtilization * 100
   754  		print("pacer: ", int(utilization*100), "% CPU (", int(goal), " exp.) for ")
   755  		print(c.heapScanWork.Load(), "+", c.stackScanWork.Load(), "+", c.globalsScanWork.Load(), " B work (", c.lastHeapScan+c.lastStackScan+c.globalsScan, " B exp.) ")
   756  		print("in ", c.triggered, " B -> ", c.heapLive, " B (∆goal ", int64(c.heapLive)-int64(c.lastHeapGoal), ", cons/mark ", oldConsMark, ")")
   757  		if !ok {
   758  			print("[controller reset]")
   759  		}
   760  		println()
   761  		printunlock()
   762  	}
   763  }
   764  
   765  // enlistWorker encourages another dedicated mark worker to start on
   766  // another P if there are spare worker slots. It is used by putfull
   767  // when more work is made available.
   768  //
   769  //go:nowritebarrier
   770  func (c *gcControllerState) enlistWorker() {
   771  	// If there are idle Ps, wake one so it will run an idle worker.
   772  	// NOTE: This is suspected of causing deadlocks. See golang.org/issue/19112.
   773  	//
   774  	//	if atomic.Load(&sched.npidle) != 0 && atomic.Load(&sched.nmspinning) == 0 {
   775  	//		wakep()
   776  	//		return
   777  	//	}
   778  
   779  	// There are no idle Ps. If we need more dedicated workers,
   780  	// try to preempt a running P so it will switch to a worker.
   781  	if c.dedicatedMarkWorkersNeeded <= 0 {
   782  		return
   783  	}
   784  	// Pick a random other P to preempt.
   785  	if gomaxprocs <= 1 {
   786  		return
   787  	}
   788  	gp := getg()
   789  	if gp == nil || gp.m == nil || gp.m.p == 0 {
   790  		return
   791  	}
   792  	myID := gp.m.p.ptr().id
   793  	for tries := 0; tries < 5; tries++ {
   794  		id := int32(fastrandn(uint32(gomaxprocs - 1)))
   795  		if id >= myID {
   796  			id++
   797  		}
   798  		p := allp[id]
   799  		if p.status != _Prunning {
   800  			continue
   801  		}
   802  		if preemptone(p) {
   803  			return
   804  		}
   805  	}
   806  }
   807  
   808  // findRunnableGCWorker returns a background mark worker for _p_ if it
   809  // should be run. This must only be called when gcBlackenEnabled != 0.
   810  func (c *gcControllerState) findRunnableGCWorker(_p_ *p, now int64) (*g, int64) {
   811  	if gcBlackenEnabled == 0 {
   812  		throw("gcControllerState.findRunnable: blackening not enabled")
   813  	}
   814  
   815  	// Since we have the current time, check if the GC CPU limiter
   816  	// hasn't had an update in a while. This check is necessary in
   817  	// case the limiter is on but hasn't been checked in a while and
   818  	// so may have left sufficient headroom to turn off again.
   819  	if now == 0 {
   820  		now = nanotime()
   821  	}
   822  	if gcCPULimiter.needUpdate(now) {
   823  		gcCPULimiter.update(now)
   824  	}
   825  
   826  	if !gcMarkWorkAvailable(_p_) {
   827  		// No work to be done right now. This can happen at
   828  		// the end of the mark phase when there are still
   829  		// assists tapering off. Don't bother running a worker
   830  		// now because it'll just return immediately.
   831  		return nil, now
   832  	}
   833  
   834  	// Grab a worker before we commit to running below.
   835  	node := (*gcBgMarkWorkerNode)(gcBgMarkWorkerPool.pop())
   836  	if node == nil {
   837  		// There is at least one worker per P, so normally there are
   838  		// enough workers to run on all Ps, if necessary. However, once
   839  		// a worker enters gcMarkDone it may park without rejoining the
   840  		// pool, thus freeing a P with no corresponding worker.
   841  		// gcMarkDone never depends on another worker doing work, so it
   842  		// is safe to simply do nothing here.
   843  		//
   844  		// If gcMarkDone bails out without completing the mark phase,
   845  		// it will always do so with queued global work. Thus, that P
   846  		// will be immediately eligible to re-run the worker G it was
   847  		// just using, ensuring work can complete.
   848  		return nil, now
   849  	}
   850  
   851  	decIfPositive := func(ptr *int64) bool {
   852  		for {
   853  			v := atomic.Loadint64(ptr)
   854  			if v <= 0 {
   855  				return false
   856  			}
   857  
   858  			if atomic.Casint64(ptr, v, v-1) {
   859  				return true
   860  			}
   861  		}
   862  	}
   863  
   864  	if decIfPositive(&c.dedicatedMarkWorkersNeeded) {
   865  		// This P is now dedicated to marking until the end of
   866  		// the concurrent mark phase.
   867  		_p_.gcMarkWorkerMode = gcMarkWorkerDedicatedMode
   868  	} else if c.fractionalUtilizationGoal == 0 {
   869  		// No need for fractional workers.
   870  		gcBgMarkWorkerPool.push(&node.node)
   871  		return nil, now
   872  	} else {
   873  		// Is this P behind on the fractional utilization
   874  		// goal?
   875  		//
   876  		// This should be kept in sync with pollFractionalWorkerExit.
   877  		delta := now - c.markStartTime
   878  		if delta > 0 && float64(_p_.gcFractionalMarkTime)/float64(delta) > c.fractionalUtilizationGoal {
   879  			// Nope. No need to run a fractional worker.
   880  			gcBgMarkWorkerPool.push(&node.node)
   881  			return nil, now
   882  		}
   883  		// Run a fractional worker.
   884  		_p_.gcMarkWorkerMode = gcMarkWorkerFractionalMode
   885  	}
   886  
   887  	// Run the background mark worker.
   888  	gp := node.gp.ptr()
   889  	casgstatus(gp, _Gwaiting, _Grunnable)
   890  	if trace.enabled {
   891  		traceGoUnpark(gp, 0)
   892  	}
   893  	return gp, now
   894  }
   895  
   896  // resetLive sets up the controller state for the next mark phase after the end
   897  // of the previous one. Must be called after endCycle and before commit, before
   898  // the world is started.
   899  //
   900  // The world must be stopped.
   901  func (c *gcControllerState) resetLive(bytesMarked uint64) {
   902  	c.heapMarked = bytesMarked
   903  	c.heapLive = bytesMarked
   904  	c.heapScan = uint64(c.heapScanWork.Load())
   905  	c.lastHeapScan = uint64(c.heapScanWork.Load())
   906  	c.lastStackScan = uint64(c.stackScanWork.Load())
   907  	c.triggered = ^uint64(0) // Reset triggered.
   908  
   909  	// heapLive was updated, so emit a trace event.
   910  	if trace.enabled {
   911  		traceHeapAlloc()
   912  	}
   913  }
   914  
   915  // markWorkerStop must be called whenever a mark worker stops executing.
   916  //
   917  // It updates mark work accounting in the controller by a duration of
   918  // work in nanoseconds and other bookkeeping.
   919  //
   920  // Safe to execute at any time.
   921  func (c *gcControllerState) markWorkerStop(mode gcMarkWorkerMode, duration int64) {
   922  	switch mode {
   923  	case gcMarkWorkerDedicatedMode:
   924  		atomic.Xaddint64(&c.dedicatedMarkTime, duration)
   925  		atomic.Xaddint64(&c.dedicatedMarkWorkersNeeded, 1)
   926  	case gcMarkWorkerFractionalMode:
   927  		atomic.Xaddint64(&c.fractionalMarkTime, duration)
   928  	case gcMarkWorkerIdleMode:
   929  		atomic.Xaddint64(&c.idleMarkTime, duration)
   930  		c.removeIdleMarkWorker()
   931  	default:
   932  		throw("markWorkerStop: unknown mark worker mode")
   933  	}
   934  }
   935  
   936  func (c *gcControllerState) update(dHeapLive, dHeapScan int64) {
   937  	if dHeapLive != 0 {
   938  		atomic.Xadd64(&gcController.heapLive, dHeapLive)
   939  		if trace.enabled {
   940  			// gcController.heapLive changed.
   941  			traceHeapAlloc()
   942  		}
   943  	}
   944  	if gcBlackenEnabled == 0 {
   945  		// Update heapScan when we're not in a current GC. It is fixed
   946  		// at the beginning of a cycle.
   947  		if dHeapScan != 0 {
   948  			atomic.Xadd64(&gcController.heapScan, dHeapScan)
   949  		}
   950  	} else {
   951  		// gcController.heapLive changed.
   952  		c.revise()
   953  	}
   954  }
   955  
   956  func (c *gcControllerState) addScannableStack(pp *p, amount int64) {
   957  	if pp == nil {
   958  		atomic.Xadd64(&c.maxStackScan, amount)
   959  		return
   960  	}
   961  	pp.maxStackScanDelta += amount
   962  	if pp.maxStackScanDelta >= maxStackScanSlack || pp.maxStackScanDelta <= -maxStackScanSlack {
   963  		atomic.Xadd64(&c.maxStackScan, pp.maxStackScanDelta)
   964  		pp.maxStackScanDelta = 0
   965  	}
   966  }
   967  
   968  func (c *gcControllerState) addGlobals(amount int64) {
   969  	atomic.Xadd64(&c.globalsScan, amount)
   970  }
   971  
   972  // heapGoal returns the current heap goal.
   973  func (c *gcControllerState) heapGoal() uint64 {
   974  	goal, _ := c.heapGoalInternal()
   975  	return goal
   976  }
   977  
   978  // heapGoalInternal is the implementation of heapGoal which returns additional
   979  // information that is necessary for computing the trigger.
   980  //
   981  // The returned minTrigger is always <= goal.
   982  func (c *gcControllerState) heapGoalInternal() (goal, minTrigger uint64) {
   983  	// Start with the goal calculated for gcPercent.
   984  	goal = c.gcPercentHeapGoal.Load()
   985  
   986  	// Check if the memory-limit-based goal is smaller, and if so, pick that.
   987  	if newGoal := c.memoryLimitHeapGoal(); go119MemoryLimitSupport && newGoal < goal {
   988  		goal = newGoal
   989  	} else {
   990  		// We're not limited by the memory limit goal, so perform a series of
   991  		// adjustments that might move the goal forward in a variety of circumstances.
   992  
   993  		sweepDistTrigger := c.sweepDistMinTrigger.Load()
   994  		if sweepDistTrigger > goal {
   995  			// Set the goal to maintain a minimum sweep distance since
   996  			// the last call to commit. Note that we never want to do this
   997  			// if we're in the memory limit regime, because it could push
   998  			// the goal up.
   999  			goal = sweepDistTrigger
  1000  		}
  1001  		// Since we ignore the sweep distance trigger in the memory
  1002  		// limit regime, we need to ensure we don't propagate it to
  1003  		// the trigger, because it could cause a violation of the
  1004  		// invariant that the trigger < goal.
  1005  		minTrigger = sweepDistTrigger
  1006  
  1007  		// Ensure that the heap goal is at least a little larger than
  1008  		// the point at which we triggered. This may not be the case if GC
  1009  		// start is delayed or if the allocation that pushed gcController.heapLive
  1010  		// over trigger is large or if the trigger is really close to
  1011  		// GOGC. Assist is proportional to this distance, so enforce a
  1012  		// minimum distance, even if it means going over the GOGC goal
  1013  		// by a tiny bit.
  1014  		//
  1015  		// Ignore this if we're in the memory limit regime: we'd prefer to
  1016  		// have the GC respond hard about how close we are to the goal than to
  1017  		// push the goal back in such a manner that it could cause us to exceed
  1018  		// the memory limit.
  1019  		const minRunway = 64 << 10
  1020  		if c.triggered != ^uint64(0) && goal < c.triggered+minRunway {
  1021  			goal = c.triggered + minRunway
  1022  		}
  1023  	}
  1024  	return
  1025  }
  1026  
  1027  // memoryLimitHeapGoal returns a heap goal derived from memoryLimit.
  1028  func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
  1029  	// Start by pulling out some values we'll need. Be careful about overflow.
  1030  	var heapFree, heapAlloc, mappedReady uint64
  1031  	for {
  1032  		heapFree = c.heapFree.load()                         // Free and unscavenged memory.
  1033  		heapAlloc = c.totalAlloc.Load() - c.totalFree.Load() // Heap object bytes in use.
  1034  		mappedReady = c.mappedReady.Load()                   // Total unreleased mapped memory.
  1035  		if heapFree+heapAlloc <= mappedReady {
  1036  			break
  1037  		}
  1038  		// It is impossible for total unreleased mapped memory to exceed heap memory, but
  1039  		// because these stats are updated independently, we may observe a partial update
  1040  		// including only some values. Thus, we appear to break the invariant. However,
  1041  		// this condition is necessarily transient, so just try again. In the case of a
  1042  		// persistent accounting error, we'll deadlock here.
  1043  	}
  1044  
  1045  	// Below we compute a goal from memoryLimit. There are a few things to be aware of.
  1046  	// Firstly, the memoryLimit does not easily compare to the heap goal: the former
  1047  	// is total mapped memory by the runtime that hasn't been released, while the latter is
  1048  	// only heap object memory. Intuitively, the way we convert from one to the other is to
  1049  	// subtract everything from memoryLimit that both contributes to the memory limit (so,
  1050  	// ignore scavenged memory) and doesn't contain heap objects. This isn't quite what
  1051  	// lines up with reality, but it's a good starting point.
  1052  	//
  1053  	// In practice this computation looks like the following:
  1054  	//
  1055  	//    memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0)) - memoryLimitHeapGoalHeadroom
  1056  	//                    ^1                                    ^2                                   ^3
  1057  	//
  1058  	// Let's break this down.
  1059  	//
  1060  	// The first term (marker 1) is everything that contributes to the memory limit and isn't
  1061  	// or couldn't become heap objects. It represents, broadly speaking, non-heap overheads.
  1062  	// One oddity you may have noticed is that we also subtract out heapFree, i.e. unscavenged
  1063  	// memory that may contain heap objects in the future.
  1064  	//
  1065  	// Let's take a step back. In an ideal world, this term would look something like just
  1066  	// the heap goal. That is, we "reserve" enough space for the heap to grow to the heap
  1067  	// goal, and subtract out everything else. This is of course impossible; the definition
  1068  	// is circular! However, this impossible definition contains a key insight: the amount
  1069  	// we're *going* to use matters just as much as whatever we're currently using.
  1070  	//
  1071  	// Consider if the heap shrinks to 1/10th its size, leaving behind lots of free and
  1072  	// unscavenged memory. mappedReady - heapAlloc will be quite large, because of that free
  1073  	// and unscavenged memory, pushing the goal down significantly.
  1074  	//
  1075  	// heapFree is also safe to exclude from the memory limit because in the steady-state, it's
  1076  	// just a pool of memory for future heap allocations, and making new allocations from heapFree
  1077  	// memory doesn't increase overall memory use. In transient states, the scavenger and the
  1078  	// allocator actively manage the pool of heapFree memory to maintain the memory limit.
  1079  	//
  1080  	// The second term (marker 2) is the amount of memory we've exceeded the limit by, and is
  1081  	// intended to help recover from such a situation. By pushing the heap goal down, we also
  1082  	// push the trigger down, triggering and finishing a GC sooner in order to make room for
  1083  	// other memory sources. Note that since we're effectively reducing the heap goal by X bytes,
  1084  	// we're actually giving more than X bytes of headroom back, because the heap goal is in
  1085  	// terms of heap objects, but it takes more than X bytes (e.g. due to fragmentation) to store
  1086  	// X bytes worth of objects.
  1087  	//
  1088  	// The third term (marker 3) subtracts an additional memoryLimitHeapGoalHeadroom bytes from the
  1089  	// heap goal. As the name implies, this is to provide additional headroom in the face of pacing
  1090  	// inaccuracies. This is a fixed number of bytes because these inaccuracies disproportionately
  1091  	// affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier. Shorter GC cycles
  1092  	// and less GC work means noisy external factors like the OS scheduler have a greater impact.
  1093  
  1094  	memoryLimit := uint64(c.memoryLimit.Load())
  1095  
  1096  	// Compute term 1.
  1097  	nonHeapMemory := mappedReady - heapFree - heapAlloc
  1098  
  1099  	// Compute term 2.
  1100  	var overage uint64
  1101  	if mappedReady > memoryLimit {
  1102  		overage = mappedReady - memoryLimit
  1103  	}
  1104  
  1105  	if nonHeapMemory+overage >= memoryLimit {
  1106  		// We're at a point where non-heap memory exceeds the memory limit on its own.
  1107  		// There's honestly not much we can do here but just trigger GCs continuously
  1108  		// and let the CPU limiter reign that in. Something has to give at this point.
  1109  		// Set it to heapMarked, the lowest possible goal.
  1110  		return c.heapMarked
  1111  	}
  1112  
  1113  	// Compute the goal.
  1114  	goal := memoryLimit - (nonHeapMemory + overage)
  1115  
  1116  	// Apply some headroom to the goal to account for pacing inaccuracies.
  1117  	// Be careful about small limits.
  1118  	if goal < memoryLimitHeapGoalHeadroom || goal-memoryLimitHeapGoalHeadroom < memoryLimitHeapGoalHeadroom {
  1119  		goal = memoryLimitHeapGoalHeadroom
  1120  	} else {
  1121  		goal = goal - memoryLimitHeapGoalHeadroom
  1122  	}
  1123  	// Don't let us go below the live heap. A heap goal below the live heap doesn't make sense.
  1124  	if goal < c.heapMarked {
  1125  		goal = c.heapMarked
  1126  	}
  1127  	return goal
  1128  }
  1129  
  1130  const (
  1131  	// These constants determine the bounds on the GC trigger as a fraction
  1132  	// of heap bytes allocated between the start of a GC (heapLive == heapMarked)
  1133  	// and the end of a GC (heapLive == heapGoal).
  1134  	//
  1135  	// The constants are obscured in this way for efficiency. The denominator
  1136  	// of the fraction is always a power-of-two for a quick division, so that
  1137  	// the numerator is a single constant integer multiplication.
  1138  	triggerRatioDen = 64
  1139  
  1140  	// The minimum trigger constant was chosen empirically: given a sufficiently
  1141  	// fast/scalable allocator with 48 Ps that could drive the trigger ratio
  1142  	// to <0.05, this constant causes applications to retain the same peak
  1143  	// RSS compared to not having this allocator.
  1144  	minTriggerRatioNum = 45 // ~0.7
  1145  
  1146  	// The maximum trigger constant is chosen somewhat arbitrarily, but the
  1147  	// current constant has served us well over the years.
  1148  	maxTriggerRatioNum = 61 // ~0.95
  1149  )
  1150  
  1151  // trigger returns the current point at which a GC should trigger along with
  1152  // the heap goal.
  1153  //
  1154  // The returned value may be compared against heapLive to determine whether
  1155  // the GC should trigger. Thus, the GC trigger condition should be (but may
  1156  // not be, in the case of small movements for efficiency) checked whenever
  1157  // the heap goal may change.
  1158  func (c *gcControllerState) trigger() (uint64, uint64) {
  1159  	goal, minTrigger := c.heapGoalInternal()
  1160  
  1161  	// Invariant: the trigger must always be less than the heap goal.
  1162  	//
  1163  	// Note that the memory limit sets a hard maximum on our heap goal,
  1164  	// but the live heap may grow beyond it.
  1165  
  1166  	if c.heapMarked >= goal {
  1167  		// The goal should never be smaller than heapMarked, but let's be
  1168  		// defensive about it. The only reasonable trigger here is one that
  1169  		// causes a continuous GC cycle at heapMarked, but respect the goal
  1170  		// if it came out as smaller than that.
  1171  		return goal, goal
  1172  	}
  1173  
  1174  	// Below this point, c.heapMarked < goal.
  1175  
  1176  	// heapMarked is our absolute minimum, and it's possible the trigger
  1177  	// bound we get from heapGoalinternal is less than that.
  1178  	if minTrigger < c.heapMarked {
  1179  		minTrigger = c.heapMarked
  1180  	}
  1181  
  1182  	// If we let the trigger go too low, then if the application
  1183  	// is allocating very rapidly we might end up in a situation
  1184  	// where we're allocating black during a nearly always-on GC.
  1185  	// The result of this is a growing heap and ultimately an
  1186  	// increase in RSS. By capping us at a point >0, we're essentially
  1187  	// saying that we're OK using more CPU during the GC to prevent
  1188  	// this growth in RSS.
  1189  	triggerLowerBound := uint64(((goal-c.heapMarked)/triggerRatioDen)*minTriggerRatioNum) + c.heapMarked
  1190  	if minTrigger < triggerLowerBound {
  1191  		minTrigger = triggerLowerBound
  1192  	}
  1193  
  1194  	// For small heaps, set the max trigger point at maxTriggerRatio of the way
  1195  	// from the live heap to the heap goal. This ensures we always have *some*
  1196  	// headroom when the GC actually starts. For larger heaps, set the max trigger
  1197  	// point at the goal, minus the minimum heap size.
  1198  	//
  1199  	// This choice follows from the fact that the minimum heap size is chosen
  1200  	// to reflect the costs of a GC with no work to do. With a large heap but
  1201  	// very little scan work to perform, this gives us exactly as much runway
  1202  	// as we would need, in the worst case.
  1203  	maxTrigger := uint64(((goal-c.heapMarked)/triggerRatioDen)*maxTriggerRatioNum) + c.heapMarked
  1204  	if goal > defaultHeapMinimum && goal-defaultHeapMinimum > maxTrigger {
  1205  		maxTrigger = goal - defaultHeapMinimum
  1206  	}
  1207  	if maxTrigger < minTrigger {
  1208  		maxTrigger = minTrigger
  1209  	}
  1210  
  1211  	// Compute the trigger from our bounds and the runway stored by commit.
  1212  	var trigger uint64
  1213  	runway := c.runway.Load()
  1214  	if runway > goal {
  1215  		trigger = minTrigger
  1216  	} else {
  1217  		trigger = goal - runway
  1218  	}
  1219  	if trigger < minTrigger {
  1220  		trigger = minTrigger
  1221  	}
  1222  	if trigger > maxTrigger {
  1223  		trigger = maxTrigger
  1224  	}
  1225  	if trigger > goal {
  1226  		print("trigger=", trigger, " heapGoal=", goal, "\n")
  1227  		print("minTrigger=", minTrigger, " maxTrigger=", maxTrigger, "\n")
  1228  		throw("produced a trigger greater than the heap goal")
  1229  	}
  1230  	return trigger, goal
  1231  }
  1232  
  1233  // commit recomputes all pacing parameters needed to derive the
  1234  // trigger and the heap goal. Namely, the gcPercent-based heap goal,
  1235  // and the amount of runway we want to give the GC this cycle.
  1236  //
  1237  // This can be called any time. If GC is the in the middle of a
  1238  // concurrent phase, it will adjust the pacing of that phase.
  1239  //
  1240  // isSweepDone should be the result of calling isSweepDone(),
  1241  // unless we're testing or we know we're executing during a GC cycle.
  1242  //
  1243  // This depends on gcPercent, gcController.heapMarked, and
  1244  // gcController.heapLive. These must be up to date.
  1245  //
  1246  // Callers must call gcControllerState.revise after calling this
  1247  // function if the GC is enabled.
  1248  //
  1249  // mheap_.lock must be held or the world must be stopped.
  1250  func (c *gcControllerState) commit(isSweepDone bool) {
  1251  	if !c.test {
  1252  		assertWorldStoppedOrLockHeld(&mheap_.lock)
  1253  	}
  1254  
  1255  	if isSweepDone {
  1256  		// The sweep is done, so there aren't any restrictions on the trigger
  1257  		// we need to think about.
  1258  		c.sweepDistMinTrigger.Store(0)
  1259  	} else {
  1260  		// Concurrent sweep happens in the heap growth
  1261  		// from gcController.heapLive to trigger. Make sure we
  1262  		// give the sweeper some runway if it doesn't have enough.
  1263  		c.sweepDistMinTrigger.Store(atomic.Load64(&c.heapLive) + sweepMinHeapDistance)
  1264  	}
  1265  
  1266  	// Compute the next GC goal, which is when the allocated heap
  1267  	// has grown by GOGC/100 over where it started the last cycle,
  1268  	// plus additional runway for non-heap sources of GC work.
  1269  	gcPercentHeapGoal := ^uint64(0)
  1270  	if gcPercent := c.gcPercent.Load(); gcPercent >= 0 {
  1271  		gcPercentHeapGoal = c.heapMarked + (c.heapMarked+atomic.Load64(&c.lastStackScan)+atomic.Load64(&c.globalsScan))*uint64(gcPercent)/100
  1272  	}
  1273  	// Apply the minimum heap size here. It's defined in terms of gcPercent
  1274  	// and is only updated by functions that call commit.
  1275  	if gcPercentHeapGoal < c.heapMinimum {
  1276  		gcPercentHeapGoal = c.heapMinimum
  1277  	}
  1278  	c.gcPercentHeapGoal.Store(gcPercentHeapGoal)
  1279  
  1280  	// Compute the amount of runway we want the GC to have by using our
  1281  	// estimate of the cons/mark ratio.
  1282  	//
  1283  	// The idea is to take our expected scan work, and multiply it by
  1284  	// the cons/mark ratio to determine how long it'll take to complete
  1285  	// that scan work in terms of bytes allocated. This gives us our GC's
  1286  	// runway.
  1287  	//
  1288  	// However, the cons/mark ratio is a ratio of rates per CPU-second, but
  1289  	// here we care about the relative rates for some division of CPU
  1290  	// resources among the mutator and the GC.
  1291  	//
  1292  	// To summarize, we have B / cpu-ns, and we want B / ns. We get that
  1293  	// by multiplying by our desired division of CPU resources. We choose
  1294  	// to express CPU resources as GOMAPROCS*fraction. Note that because
  1295  	// we're working with a ratio here, we can omit the number of CPU cores,
  1296  	// because they'll appear in the numerator and denominator and cancel out.
  1297  	// As a result, this is basically just "weighing" the cons/mark ratio by
  1298  	// our desired division of resources.
  1299  	//
  1300  	// Furthermore, by setting the runway so that CPU resources are divided
  1301  	// this way, assuming that the cons/mark ratio is correct, we make that
  1302  	// division a reality.
  1303  	c.runway.Store(uint64((c.consMark * (1 - gcGoalUtilization) / (gcGoalUtilization)) * float64(c.lastHeapScan+c.lastStackScan+c.globalsScan)))
  1304  }
  1305  
  1306  // setGCPercent updates gcPercent. commit must be called after.
  1307  // Returns the old value of gcPercent.
  1308  //
  1309  // The world must be stopped, or mheap_.lock must be held.
  1310  func (c *gcControllerState) setGCPercent(in int32) int32 {
  1311  	if !c.test {
  1312  		assertWorldStoppedOrLockHeld(&mheap_.lock)
  1313  	}
  1314  
  1315  	out := c.gcPercent.Load()
  1316  	if in < 0 {
  1317  		in = -1
  1318  	}
  1319  	c.heapMinimum = defaultHeapMinimum * uint64(in) / 100
  1320  	c.gcPercent.Store(in)
  1321  
  1322  	return out
  1323  }
  1324  
  1325  //go:linkname setGCPercent runtime/debug.setGCPercent
  1326  func setGCPercent(in int32) (out int32) {
  1327  	// Run on the system stack since we grab the heap lock.
  1328  	systemstack(func() {
  1329  		lock(&mheap_.lock)
  1330  		out = gcController.setGCPercent(in)
  1331  		gcControllerCommit()
  1332  		unlock(&mheap_.lock)
  1333  	})
  1334  
  1335  	// If we just disabled GC, wait for any concurrent GC mark to
  1336  	// finish so we always return with no GC running.
  1337  	if in < 0 {
  1338  		gcWaitOnMark(atomic.Load(&work.cycles))
  1339  	}
  1340  
  1341  	return out
  1342  }
  1343  
  1344  func readGOGC() int32 {
  1345  	p := gogetenv("GOGC")
  1346  	if p == "off" {
  1347  		return -1
  1348  	}
  1349  	if n, ok := atoi32(p); ok {
  1350  		return n
  1351  	}
  1352  	return 100
  1353  }
  1354  
  1355  // setMemoryLimit updates memoryLimit. commit must be called after
  1356  // Returns the old value of memoryLimit.
  1357  //
  1358  // The world must be stopped, or mheap_.lock must be held.
  1359  func (c *gcControllerState) setMemoryLimit(in int64) int64 {
  1360  	if !c.test {
  1361  		assertWorldStoppedOrLockHeld(&mheap_.lock)
  1362  	}
  1363  
  1364  	out := c.memoryLimit.Load()
  1365  	if in >= 0 {
  1366  		c.memoryLimit.Store(in)
  1367  	}
  1368  
  1369  	return out
  1370  }
  1371  
  1372  //go:linkname setMemoryLimit runtime/debug.setMemoryLimit
  1373  func setMemoryLimit(in int64) (out int64) {
  1374  	// Run on the system stack since we grab the heap lock.
  1375  	systemstack(func() {
  1376  		lock(&mheap_.lock)
  1377  		out = gcController.setMemoryLimit(in)
  1378  		if in < 0 || out == in {
  1379  			// If we're just checking the value or not changing
  1380  			// it, there's no point in doing the rest.
  1381  			unlock(&mheap_.lock)
  1382  			return
  1383  		}
  1384  		gcControllerCommit()
  1385  		unlock(&mheap_.lock)
  1386  	})
  1387  	return out
  1388  }
  1389  
  1390  func readGOMEMLIMIT() int64 {
  1391  	p := gogetenv("GOMEMLIMIT")
  1392  	if p == "" || p == "off" {
  1393  		return maxInt64
  1394  	}
  1395  	n, ok := parseByteCount(p)
  1396  	if !ok {
  1397  		print("GOMEMLIMIT=", p, "\n")
  1398  		throw("malformed GOMEMLIMIT; see `go doc runtime/debug.SetMemoryLimit`")
  1399  	}
  1400  	return n
  1401  }
  1402  
  1403  type piController struct {
  1404  	kp float64 // Proportional constant.
  1405  	ti float64 // Integral time constant.
  1406  	tt float64 // Reset time.
  1407  
  1408  	min, max float64 // Output boundaries.
  1409  
  1410  	// PI controller state.
  1411  
  1412  	errIntegral float64 // Integral of the error from t=0 to now.
  1413  
  1414  	// Error flags.
  1415  	errOverflow   bool // Set if errIntegral ever overflowed.
  1416  	inputOverflow bool // Set if an operation with the input overflowed.
  1417  }
  1418  
  1419  // next provides a new sample to the controller.
  1420  //
  1421  // input is the sample, setpoint is the desired point, and period is how much
  1422  // time (in whatever unit makes the most sense) has passed since the last sample.
  1423  //
  1424  // Returns a new value for the variable it's controlling, and whether the operation
  1425  // completed successfully. One reason this might fail is if error has been growing
  1426  // in an unbounded manner, to the point of overflow.
  1427  //
  1428  // In the specific case of an error overflow occurs, the errOverflow field will be
  1429  // set and the rest of the controller's internal state will be fully reset.
  1430  func (c *piController) next(input, setpoint, period float64) (float64, bool) {
  1431  	// Compute the raw output value.
  1432  	prop := c.kp * (setpoint - input)
  1433  	rawOutput := prop + c.errIntegral
  1434  
  1435  	// Clamp rawOutput into output.
  1436  	output := rawOutput
  1437  	if isInf(output) || isNaN(output) {
  1438  		// The input had a large enough magnitude that either it was already
  1439  		// overflowed, or some operation with it overflowed.
  1440  		// Set a flag and reset. That's the safest thing to do.
  1441  		c.reset()
  1442  		c.inputOverflow = true
  1443  		return c.min, false
  1444  	}
  1445  	if output < c.min {
  1446  		output = c.min
  1447  	} else if output > c.max {
  1448  		output = c.max
  1449  	}
  1450  
  1451  	// Update the controller's state.
  1452  	if c.ti != 0 && c.tt != 0 {
  1453  		c.errIntegral += (c.kp*period/c.ti)*(setpoint-input) + (period/c.tt)*(output-rawOutput)
  1454  		if isInf(c.errIntegral) || isNaN(c.errIntegral) {
  1455  			// So much error has accumulated that we managed to overflow.
  1456  			// The assumptions around the controller have likely broken down.
  1457  			// Set a flag and reset. That's the safest thing to do.
  1458  			c.reset()
  1459  			c.errOverflow = true
  1460  			return c.min, false
  1461  		}
  1462  	}
  1463  	return output, true
  1464  }
  1465  
  1466  // reset resets the controller state, except for controller error flags.
  1467  func (c *piController) reset() {
  1468  	c.errIntegral = 0
  1469  }
  1470  
  1471  // addIdleMarkWorker attempts to add a new idle mark worker.
  1472  //
  1473  // If this returns true, the caller must become an idle mark worker unless
  1474  // there's no background mark worker goroutines in the pool. This case is
  1475  // harmless because there are already background mark workers running.
  1476  // If this returns false, the caller must NOT become an idle mark worker.
  1477  //
  1478  // nosplit because it may be called without a P.
  1479  //
  1480  //go:nosplit
  1481  func (c *gcControllerState) addIdleMarkWorker() bool {
  1482  	for {
  1483  		old := c.idleMarkWorkers.Load()
  1484  		n, max := int32(old&uint64(^uint32(0))), int32(old>>32)
  1485  		if n >= max {
  1486  			// See the comment on idleMarkWorkers for why
  1487  			// n > max is tolerated.
  1488  			return false
  1489  		}
  1490  		if n < 0 {
  1491  			print("n=", n, " max=", max, "\n")
  1492  			throw("negative idle mark workers")
  1493  		}
  1494  		new := uint64(uint32(n+1)) | (uint64(max) << 32)
  1495  		if c.idleMarkWorkers.CompareAndSwap(old, new) {
  1496  			return true
  1497  		}
  1498  	}
  1499  }
  1500  
  1501  // needIdleMarkWorker is a hint as to whether another idle mark worker is needed.
  1502  //
  1503  // The caller must still call addIdleMarkWorker to become one. This is mainly
  1504  // useful for a quick check before an expensive operation.
  1505  //
  1506  // nosplit because it may be called without a P.
  1507  //
  1508  //go:nosplit
  1509  func (c *gcControllerState) needIdleMarkWorker() bool {
  1510  	p := c.idleMarkWorkers.Load()
  1511  	n, max := int32(p&uint64(^uint32(0))), int32(p>>32)
  1512  	return n < max
  1513  }
  1514  
  1515  // removeIdleMarkWorker must be called when an new idle mark worker stops executing.
  1516  func (c *gcControllerState) removeIdleMarkWorker() {
  1517  	for {
  1518  		old := c.idleMarkWorkers.Load()
  1519  		n, max := int32(old&uint64(^uint32(0))), int32(old>>32)
  1520  		if n-1 < 0 {
  1521  			print("n=", n, " max=", max, "\n")
  1522  			throw("negative idle mark workers")
  1523  		}
  1524  		new := uint64(uint32(n-1)) | (uint64(max) << 32)
  1525  		if c.idleMarkWorkers.CompareAndSwap(old, new) {
  1526  			return
  1527  		}
  1528  	}
  1529  }
  1530  
  1531  // setMaxIdleMarkWorkers sets the maximum number of idle mark workers allowed.
  1532  //
  1533  // This method is optimistic in that it does not wait for the number of
  1534  // idle mark workers to reduce to max before returning; it assumes the workers
  1535  // will deschedule themselves.
  1536  func (c *gcControllerState) setMaxIdleMarkWorkers(max int32) {
  1537  	for {
  1538  		old := c.idleMarkWorkers.Load()
  1539  		n := int32(old & uint64(^uint32(0)))
  1540  		if n < 0 {
  1541  			print("n=", n, " max=", max, "\n")
  1542  			throw("negative idle mark workers")
  1543  		}
  1544  		new := uint64(uint32(n)) | (uint64(max) << 32)
  1545  		if c.idleMarkWorkers.CompareAndSwap(old, new) {
  1546  			return
  1547  		}
  1548  	}
  1549  }
  1550  
  1551  // gcControllerCommit is gcController.commit, but passes arguments from live
  1552  // (non-test) data. It also updates any consumers of the GC pacing, such as
  1553  // sweep pacing and the background scavenger.
  1554  //
  1555  // Calls gcController.commit.
  1556  //
  1557  // The heap lock must be held, so this must be executed on the system stack.
  1558  //
  1559  //go:systemstack
  1560  func gcControllerCommit() {
  1561  	assertWorldStoppedOrLockHeld(&mheap_.lock)
  1562  
  1563  	gcController.commit(isSweepDone())
  1564  
  1565  	// Update mark pacing.
  1566  	if gcphase != _GCoff {
  1567  		gcController.revise()
  1568  	}
  1569  
  1570  	// TODO(mknyszek): This isn't really accurate any longer because the heap
  1571  	// goal is computed dynamically. Still useful to snapshot, but not as useful.
  1572  	if trace.enabled {
  1573  		traceHeapGoal()
  1574  	}
  1575  
  1576  	trigger, heapGoal := gcController.trigger()
  1577  	gcPaceSweeper(trigger)
  1578  	gcPaceScavenger(gcController.memoryLimit.Load(), heapGoal, gcController.lastHeapGoal)
  1579  }
  1580  

View as plain text