goroutineleakprofile_test.go

     1  // Copyright 2025 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package runtime_test
     6  
     7  import (
     8  	"fmt"
     9  	"internal/testenv"
    10  	"os"
    11  	"regexp"
    12  	"strings"
    13  	"testing"
    14  )
    15  
    16  func TestGoroutineLeakProfile(t *testing.T) {
    17  	// Some tests have false negatives under mayMoreStackPreempt and mayMoreStackMove.
    18  	// This may be a test-only issue in that they're just sensitive to scheduling, but it
    19  	// needs more investigation.
    20  	for _, cfg := range []string{"mayMoreStackPreempt", "mayMoreStackMove"} {
    21  		if strings.Contains(os.Getenv("GOFLAGS"), cfg) {
    22  			testenv.SkipFlaky(t, 75729)
    23  		}
    24  	}
    25  
    26  	// Goroutine leak test case.
    27  	//
    28  	// Test cases can be configured with test name, the name of the entry point function,
    29  	// a set of expected leaks identified by regular expressions, and the number of times
    30  	// the test should be repeated.
    31  	//
    32  	// Repeated runs reduce flakiness in some tests.
    33  	type testCase struct {
    34  		name          string
    35  		simple        bool
    36  		repetitions   int
    37  		expectedLeaks map[*regexp.Regexp]bool
    38  
    39  		// flakyLeaks are goroutine leaks that are too flaky to be reliably detected.
    40  		// Still, they might pop up every once in a while. The test will pass regardless
    41  		// if they occur or nor, as they are not unexpected.
    42  		//
    43  		// Note that all flaky leaks are true positives, i.e. real goroutine leaks,
    44  		// and it is only their detection that is unreliable due to scheduling
    45  		// non-determinism.
    46  		flakyLeaks map[*regexp.Regexp]struct{}
    47  	}
    48  
    49  	// makeAnyTest is a short-hand for creating test cases.
    50  	// Each of the leaks in the list is identified by a regular expression.
    51  	// If a leak is flaky, it is added to the flakyLeaks map.
    52  	makeAnyTest := func(name string, flaky bool, repetitions int, leaks ...string) testCase {
    53  		tc := testCase{
    54  			name:          name,
    55  			expectedLeaks: make(map[*regexp.Regexp]bool, len(leaks)),
    56  			flakyLeaks:    make(map[*regexp.Regexp]struct{}, len(leaks)),
    57  			// Make sure the test is repeated at least once.
    58  			repetitions: repetitions | 1,
    59  		}
    60  
    61  		for _, leak := range leaks {
    62  			if !flaky {
    63  				tc.expectedLeaks[regexp.MustCompile(leak)] = false
    64  			} else {
    65  				tc.flakyLeaks[regexp.MustCompile(leak)] = struct{}{}
    66  			}
    67  		}
    68  
    69  		return tc
    70  	}
    71  
    72  	// makeTest is a short-hand for creating non-flaky test cases.
    73  	makeTest := func(name string, leaks ...string) testCase {
    74  		tcase := makeAnyTest(name, false, 2, leaks...)
    75  		tcase.simple = true
    76  		return tcase
    77  	}
    78  
    79  	// makeFlakyTest is a short-hand for creating flaky test cases.
    80  	makeFlakyTest := func(name string, leaks ...string) testCase {
    81  		if testing.Short() {
    82  			return makeAnyTest(name, true, 2, leaks...)
    83  		}
    84  		return makeAnyTest(name, true, 10, leaks...)
    85  	}
    86  
    87  	goroutineHeader := regexp.MustCompile(`goroutine \d+ \[`)
    88  
    89  	// extractLeaks takes the output of a test and splits it into a
    90  	// list of strings denoting goroutine leaks.
    91  	//
    92  	// If the input is:
    93  	//
    94  	// goroutine 1 [wait reason (leaked)]:
    95  	// main.leaked()
    96  	// 	./testdata/testgoroutineleakprofile/foo.go:37 +0x100
    97  	// created by main.main()
    98  	// 	./testdata/testgoroutineleakprofile/main.go:10 +0x20
    99  	//
   100  	// goroutine 2 [wait reason (leaked)]:
   101  	// main.leaked2()
   102  	// 	./testdata/testgoroutineleakprofile/foo.go:37 +0x100
   103  	// created by main.main()
   104  	// 	./testdata/testgoroutineleakprofile/main.go:10 +0x20
   105  	//
   106  	// The output is (as a list of strings):
   107  	//
   108  	// leaked() [wait reason]
   109  	// leaked2() [wait reason]
   110  	extractLeaks := func(output string) []string {
   111  		stacks := strings.Split(output, "\n\ngoroutine")
   112  		var leaks []string
   113  		for _, stack := range stacks {
   114  			lines := strings.Split(stack, "\n")
   115  			if len(lines) < 5 {
   116  				// Expecting at least the following lines (where n=len(lines)-1):
   117  				//
   118  				// [0] goroutine n [wait reason (leaked)]
   119  				// ...
   120  				// [n-3] bottom.leak.frame(...)
   121  				// [n-2]  ./bottom/leak/frame/source.go:line
   122  				// [n-1] created by go.instruction()
   123  				// [n] 	  ./go/instruction/source.go:line
   124  				continue
   125  			}
   126  
   127  			if !strings.Contains(lines[0], "(leaked)") {
   128  				// Ignore non-leaked goroutines.
   129  				continue
   130  			}
   131  
   132  			// Get the wait reason from the goroutine header.
   133  			header := lines[0]
   134  			waitReason := goroutineHeader.ReplaceAllString(header, "[")
   135  			waitReason = strings.ReplaceAll(waitReason, " (leaked)", "")
   136  
   137  			// Get the function name from the stack trace (should be two lines above `created by`).
   138  			var funcName string
   139  			for i := len(lines) - 1; i >= 0; i-- {
   140  				if strings.Contains(lines[i], "created by") {
   141  					funcName = strings.TrimPrefix(lines[i-2], "main.")
   142  					break
   143  				}
   144  			}
   145  			if funcName == "" {
   146  				t.Fatalf("failed to extract function name from stack trace: %s", lines)
   147  			}
   148  
   149  			leaks = append(leaks, funcName+" "+waitReason)
   150  		}
   151  		return leaks
   152  	}
   153  
   154  	// Micro tests involve very simple leaks for each type of concurrency primitive operation.
   155  	microTests := []testCase{
   156  		makeTest("NilRecv",
   157  			`NilRecv\.func1\(.* \[chan receive \(nil chan\)\]`,
   158  		),
   159  		makeTest("NilSend",
   160  			`NilSend\.func1\(.* \[chan send \(nil chan\)\]`,
   161  		),
   162  		makeTest("SelectNoCases",
   163  			`SelectNoCases\.func1\(.* \[select \(no cases\)\]`,
   164  		),
   165  		makeTest("ChanRecv",
   166  			`ChanRecv\.func1\(.* \[chan receive\]`,
   167  		),
   168  		makeTest("ChanSend",
   169  			`ChanSend\.func1\(.* \[chan send\]`,
   170  		),
   171  		makeTest("Select",
   172  			`Select\.func1\(.* \[select\]`,
   173  		),
   174  		makeTest("WaitGroup",
   175  			`WaitGroup\.func1\(.* \[sync\.WaitGroup\.Wait\]`,
   176  		),
   177  		makeTest("MutexStack",
   178  			`MutexStack\.func1\(.* \[sync\.Mutex\.Lock\]`,
   179  		),
   180  		makeTest("MutexHeap",
   181  			`MutexHeap\.func1.1\(.* \[sync\.Mutex\.Lock\]`,
   182  		),
   183  		makeTest("Cond",
   184  			`Cond\.func1\(.* \[sync\.Cond\.Wait\]`,
   185  		),
   186  		makeTest("RWMutexRLock",
   187  			`RWMutexRLock\.func1\(.* \[sync\.RWMutex\.RLock\]`,
   188  		),
   189  		makeTest("RWMutexLock",
   190  			`RWMutexLock\.func1\(.* \[sync\.(RW)?Mutex\.Lock\]`,
   191  		),
   192  		makeTest("Mixed",
   193  			`Mixed\.func1\(.* \[sync\.WaitGroup\.Wait\]`,
   194  			`Mixed\.func1.1\(.* \[chan send\]`,
   195  		),
   196  		makeTest("NoLeakGlobal"),
   197  		// If the main goroutine is waiting on select{}, it should not be reported as a leak.
   198  		makeTest("SelectNoCasesMain",
   199  			// However, any goroutine leaking on a channel that is reachable from the main
   200  			// goroutine should be reported.
   201  			`SelectNoCasesMain\.func1\(.* \[chan receive\]`,
   202  		),
   203  	}
   204  
   205  	// Stress tests are flaky and we do not strictly care about their output.
   206  	// They are only intended to stress the goroutine leak detector and profiling
   207  	// infrastructure in interesting ways.
   208  	stressTestCases := []testCase{
   209  		makeFlakyTest("SpawnGC",
   210  			`spawnGC.func1\(.* \[chan receive\]`,
   211  		),
   212  		makeTest("DaisyChain"),
   213  	}
   214  
   215  	// Common goroutine leak patterns.
   216  	// Extracted from "Unveiling and Vanquishing Goroutine Leaks in Enterprise Microservices: A Dynamic Analysis Approach"
   217  	// doi:10.1109/CGO57630.2024.10444835
   218  	patternTestCases := []testCase{
   219  		makeTest("NoCloseRange",
   220  			`noCloseRange\(.* \[chan send\]`,
   221  			`noCloseRange\.func1\(.* \[chan receive\]`,
   222  		),
   223  		makeTest("MethodContractViolation",
   224  			`worker\.Start\.func1\(.* \[select\]`,
   225  		),
   226  		makeTest("DoubleSend",
   227  			`DoubleSend\.func3\(.* \[chan send\]`,
   228  		),
   229  		makeTest("EarlyReturn",
   230  			`earlyReturn\.func1\(.* \[chan send\]`,
   231  		),
   232  		makeTest("NCastLeak",
   233  			`nCastLeak\.func1\(.* \[chan send\]`,
   234  			`NCastLeak\.func2\(.* \[chan receive\]`,
   235  		),
   236  		makeTest("Timeout",
   237  			// (vsaioc): Timeout is *theoretically* flaky, but the
   238  			// pseudo-random choice for select case branches makes it
   239  			// practically impossible for it to fail.
   240  			`timeout\.func1\(.* \[chan send\]`,
   241  		),
   242  	}
   243  
   244  	// GoKer tests from "GoBench: A Benchmark Suite of Real-World Go Concurrency Bugs".
   245  	// Refer to testdata/testgoroutineleakprofile/goker/README.md.
   246  	//
   247  	// This list is curated for tests that are not excessively flaky.
   248  	// Some tests are also excluded because they are redundant.
   249  	//
   250  	// TODO(vsaioc): Some of these might be removable (their patterns may overlap).
   251  	gokerTestCases := []testCase{
   252  		makeFlakyTest("Cockroach584",
   253  			`Cockroach584\.func2\(.* \[sync\.Mutex\.Lock\]`,
   254  		),
   255  		makeFlakyTest("Cockroach1055",
   256  			`Cockroach1055\.func2\(.* \[chan receive\]`,
   257  			`Cockroach1055\.func2\.2\(.* \[sync\.WaitGroup\.Wait\]`,
   258  			`Cockroach1055\.func2\.1\(.* \[chan receive\]`,
   259  			`Cockroach1055\.func2\.1\(.* \[sync\.Mutex\.Lock\]`,
   260  		),
   261  		makeFlakyTest("Cockroach1462",
   262  			`\(\*Stopper_cockroach1462\)\.RunWorker\.func1\(.* \[chan send\]`,
   263  			`Cockroach1462\.func2\(.* \[sync\.WaitGroup\.Wait\]`,
   264  		),
   265  		makeFlakyTest("Cockroach2448",
   266  			`\(\*Store_cockroach2448\)\.processRaft\(.* \[select\]`,
   267  			`\(\*state_cockroach2448\)\.start\(.* \[select\]`,
   268  		),
   269  		makeFlakyTest("Cockroach3710",
   270  			`\(\*Store_cockroach3710\)\.ForceRaftLogScanAndProcess\(.* \[sync\.RWMutex\.RLock\]`,
   271  			`\(\*Store_cockroach3710\)\.processRaft\.func1\(.* \[sync\.RWMutex\.Lock\]`,
   272  		),
   273  		makeFlakyTest("Cockroach6181",
   274  			`testRangeCacheCoalescedRequests_cockroach6181\(.* \[sync\.WaitGroup\.Wait\]`,
   275  			`testRangeCacheCoalescedRequests_cockroach6181\.func1\.1\(.* \[sync\.(RW)?Mutex\.Lock\]`,
   276  			`testRangeCacheCoalescedRequests_cockroach6181\.func1\.1\(.* \[sync\.RWMutex\.RLock\]`,
   277  		),
   278  		makeTest("Cockroach7504",
   279  			`Cockroach7504\.func2\.1.* \[sync\.Mutex\.Lock\]`,
   280  			`Cockroach7504\.func2\.2.* \[sync\.Mutex\.Lock\]`,
   281  		),
   282  		makeFlakyTest("Cockroach9935",
   283  			`\(\*loggingT_cockroach9935\)\.outputLogEntry\(.* \[sync\.Mutex\.Lock\]`,
   284  		),
   285  		makeFlakyTest("Cockroach10214",
   286  			`\(*Store_cockroach10214\)\.sendQueuedHeartbeats\(.* \[sync\.Mutex\.Lock\]`,
   287  			`\(*Replica_cockroach10214\)\.tick\(.* \[sync\.Mutex\.Lock\]`,
   288  		),
   289  		makeFlakyTest("Cockroach10790",
   290  			`\(\*Replica_cockroach10790\)\.beginCmds\.func1\(.* \[chan receive\]`,
   291  		),
   292  		makeTest("Cockroach13197",
   293  			`\(\*Tx_cockroach13197\)\.awaitDone\(.* \[chan receive\]`,
   294  		),
   295  		makeTest("Cockroach13755",
   296  			`\(\*Rows_cockroach13755\)\.awaitDone\(.* \[chan receive\]`,
   297  		),
   298  		makeFlakyTest("Cockroach16167",
   299  			`Cockroach16167\.func2\(.* \[sync\.RWMutex\.RLock\]`,
   300  			`\(\*Executor_cockroach16167\)\.Start\(.* \[sync\.RWMutex\.Lock\]`,
   301  		),
   302  		makeFlakyTest("Cockroach18101",
   303  			`restore_cockroach18101\.func1\(.* \[chan send\]`,
   304  		),
   305  		makeTest("Cockroach24808",
   306  			`Cockroach24808\.func2\(.* \[chan send\]`,
   307  		),
   308  		makeTest("Cockroach25456",
   309  			`Cockroach25456\.func2\(.* \[chan receive\]`,
   310  		),
   311  		makeTest("Cockroach35073",
   312  			`Cockroach35073\.func2.1\(.* \[chan send\]`,
   313  			`Cockroach35073\.func2\(.* \[chan send\]`,
   314  		),
   315  		makeTest("Cockroach35931",
   316  			`Cockroach35931\.func2\(.* \[chan send\]`,
   317  		),
   318  		makeTest("Etcd5509",
   319  			`Etcd5509\.func2\(.* \[sync\.RWMutex\.Lock\]`,
   320  		),
   321  		makeTest("Etcd6708",
   322  			`Etcd6708\.func2\(.* \[sync\.RWMutex\.RLock\]`,
   323  		),
   324  		makeFlakyTest("Etcd6857",
   325  			`\(\*node_etcd6857\)\.Status\(.* \[chan send\]`,
   326  		),
   327  		makeFlakyTest("Etcd6873",
   328  			`\(\*watchBroadcasts_etcd6873\)\.stop\(.* \[chan receive\]`,
   329  			`newWatchBroadcasts_etcd6873\.func1\(.* \[sync\.Mutex\.Lock\]`,
   330  		),
   331  		makeFlakyTest("Etcd7492",
   332  			`Etcd7492\.func2\(.* \[sync\.WaitGroup\.Wait\]`,
   333  			`Etcd7492\.func2\.1\(.* \[chan send\]`,
   334  			`\(\*simpleTokenTTLKeeper_etcd7492\)\.run\(.* \[sync\.Mutex\.Lock\]`,
   335  		),
   336  		makeFlakyTest("Etcd7902",
   337  			`doRounds_etcd7902\.func1\(.* \[chan receive\]`,
   338  			`doRounds_etcd7902\.func1\(.* \[sync\.Mutex\.Lock\]`,
   339  			`runElectionFunc_etcd7902\(.* \[sync\.WaitGroup\.Wait\]`,
   340  		),
   341  		makeTest("Etcd10492",
   342  			`Etcd10492\.func2\(.* \[sync\.Mutex\.Lock\]`,
   343  		),
   344  		makeTest("Grpc660",
   345  			`\(\*benchmarkClient_grpc660\)\.doCloseLoopUnary\.func1\(.* \[chan send\]`,
   346  		),
   347  		makeFlakyTest("Grpc795",
   348  			`\(\*Server_grpc795\)\.Serve\(.* \[sync\.Mutex\.Lock\]`,
   349  			`testServerGracefulStopIdempotent_grpc795\(.* \[sync\.Mutex\.Lock\]`,
   350  		),
   351  		makeTest("Grpc862",
   352  			`DialContext_grpc862\.func2\(.* \[chan receive\]`),
   353  		makeTest("Grpc1275",
   354  			`testInflightStreamClosing_grpc1275\.func1\(.* \[chan receive\]`),
   355  		makeTest("Grpc1424",
   356  			`DialContext_grpc1424\.func1\(.* \[chan receive\]`),
   357  		makeFlakyTest("Grpc1460",
   358  			`\(\*http2Client_grpc1460\)\.keepalive\(.* \[chan receive\]`,
   359  			`\(\*http2Client_grpc1460\)\.NewStream\(.* \[sync\.Mutex\.Lock\]`,
   360  		),
   361  		makeFlakyTest("Grpc3017",
   362  			// grpc/3017 involves a goroutine leak that also simultaneously engages many GC assists.
   363  			`Grpc3017\.func2\(.* \[chan receive\]`,
   364  			`Grpc3017\.func2\.1\(.* \[sync\.Mutex\.Lock\]`,
   365  			`\(\*lbCacheClientConn_grpc3017\)\.RemoveSubConn\.func1\(.* \[sync\.Mutex\.Lock\]`,
   366  		),
   367  		makeFlakyTest("Hugo3251",
   368  			`Hugo3251\.func2\(.* \[sync\.WaitGroup\.Wait\]`,
   369  			`Hugo3251\.func2\.1\(.* \[sync\.Mutex\.Lock\]`,
   370  			`Hugo3251\.func2\.1\(.* \[sync\.RWMutex\.RLock\]`,
   371  		),
   372  		makeFlakyTest("Hugo5379",
   373  			`\(\*Page_hugo5379\)\.initContent\.func1\.1\(.* \[sync\.Mutex\.Lock\]`,
   374  			`pageRenderer_hugo5379\(.* \[sync\.Mutex\.Lock\]`,
   375  			`Hugo5379\.func2\(.* \[sync\.WaitGroup\.Wait\]`,
   376  		),
   377  		makeFlakyTest("Istio16224",
   378  			`Istio16224\.func2\(.* \[sync\.Mutex\.Lock\]`,
   379  			`\(\*controller_istio16224\)\.Run\(.* \[chan send\]`,
   380  			`\(\*controller_istio16224\)\.Run\(.* \[chan receive\]`,
   381  		),
   382  		makeFlakyTest("Istio17860",
   383  			`\(\*agent_istio17860\)\.runWait\(.* \[chan send\]`,
   384  		),
   385  		makeFlakyTest("Istio18454",
   386  			`\(\*Worker_istio18454\)\.Start\.func1\(.* \[chan receive\]`,
   387  			`\(\*Worker_istio18454\)\.Start\.func1\(.* \[chan send\]`,
   388  		),
   389  		// NOTE(vsaioc):
   390  		// Kubernetes/1321 is excluded due to a race condition in the original program
   391  		// that may, in extremely rare cases, lead to nil pointer dereference crashes.
   392  		// (Reproducible even with regular GC). Only kept here for posterity.
   393  		//
   394  		// makeTest(testCase{name: "Kubernetes1321"},
   395  		// 	`NewMux_kubernetes1321\.gowrap1\(.* \[chan send\]`,
   396  		// 	`testMuxWatcherClose_kubernetes1321\(.* \[sync\.Mutex\.Lock\]`),
   397  		makeTest("Kubernetes5316",
   398  			`finishRequest_kubernetes5316\.func1\(.* \[chan send\]`,
   399  		),
   400  		makeFlakyTest("Kubernetes6632",
   401  			`\(\*idleAwareFramer_kubernetes6632\)\.monitor\(.* \[sync\.Mutex\.Lock\]`,
   402  			`\(\*idleAwareFramer_kubernetes6632\)\.WriteFrame\(.* \[chan send\]`,
   403  		),
   404  		makeFlakyTest("Kubernetes10182",
   405  			`\(\*statusManager_kubernetes10182\)\.Start\.func1\(.* \[sync\.Mutex\.Lock\]`,
   406  			`\(\*statusManager_kubernetes10182\)\.SetPodStatus\(.* \[chan send\]`,
   407  		),
   408  		makeFlakyTest("Kubernetes11298",
   409  			`After_kubernetes11298\.func1\(.* \[chan receive\]`,
   410  			`After_kubernetes11298\.func1\(.* \[sync\.Cond\.Wait\]`,
   411  			`Kubernetes11298\.func2\(.* \[chan receive\]`,
   412  		),
   413  		makeFlakyTest("Kubernetes13135",
   414  			`Util_kubernetes13135\(.* \[sync\.Mutex\.Lock\]`,
   415  			`\(\*WatchCache_kubernetes13135\)\.Add\(.* \[sync\.Mutex\.Lock\]`,
   416  		),
   417  		makeTest("Kubernetes25331",
   418  			`\(\*watchChan_kubernetes25331\)\.run\(.* \[chan send\]`,
   419  		),
   420  		makeFlakyTest("Kubernetes26980",
   421  			`Kubernetes26980\.func2\(.* \[chan receive\]`,
   422  			`Kubernetes26980\.func2\.1\(.* \[sync\.Mutex\.Lock\]`,
   423  			`\(\*processorListener_kubernetes26980\)\.pop\(.* \[chan receive\]`,
   424  		),
   425  		makeFlakyTest("Kubernetes30872",
   426  			`\(\*DelayingDeliverer_kubernetes30872\)\.StartWithHandler\.func1\(.* \[sync\.Mutex\.Lock\]`,
   427  			`\(\*Controller_kubernetes30872\)\.Run\(.* \[sync\.Mutex\.Lock\]`,
   428  			`\(\*NamespaceController_kubernetes30872\)\.Run\.func1\(.* \[sync\.Mutex\.Lock\]`,
   429  		),
   430  		makeTest("Kubernetes38669",
   431  			`\(\*cacheWatcher_kubernetes38669\)\.process\(.* \[chan send\]`,
   432  		),
   433  		makeFlakyTest("Kubernetes58107",
   434  			`\(\*ResourceQuotaController_kubernetes58107\)\.worker\(.* \[sync\.Cond\.Wait\]`,
   435  			`\(\*ResourceQuotaController_kubernetes58107\)\.worker\(.* \[sync\.RWMutex\.RLock\]`,
   436  			`\(\*ResourceQuotaController_kubernetes58107\)\.Sync\(.* \[sync\.RWMutex\.Lock\]`,
   437  		),
   438  		makeFlakyTest("Kubernetes62464",
   439  			`\(\*manager_kubernetes62464\)\.reconcileState\(.* \[sync\.RWMutex\.RLock\]`,
   440  			`\(\*staticPolicy_kubernetes62464\)\.RemoveContainer\(.* \[sync\.(RW)?Mutex\.Lock\]`,
   441  		),
   442  		makeFlakyTest("Kubernetes70277",
   443  			`Kubernetes70277\.func2\(.* \[chan receive\]`,
   444  		),
   445  		makeFlakyTest("Moby4951",
   446  			`\(\*DeviceSet_moby4951\)\.DeleteDevice\(.* \[sync\.Mutex\.Lock\]`,
   447  		),
   448  		makeTest("Moby7559",
   449  			`\(\*UDPProxy_moby7559\)\.Run\(.* \[sync\.Mutex\.Lock\]`,
   450  		),
   451  		makeTest("Moby17176",
   452  			`testDevmapperLockReleasedDeviceDeletion_moby17176\.func1\(.* \[sync\.Mutex\.Lock\]`,
   453  		),
   454  		makeFlakyTest("Moby21233",
   455  			`\(\*Transfer_moby21233\)\.Watch\.func1\(.* \[chan send\]`,
   456  			`\(\*Transfer_moby21233\)\.Watch\.func1\(.* \[select\]`,
   457  			`testTransfer_moby21233\(.* \[chan receive\]`,
   458  		),
   459  		makeTest("Moby25348",
   460  			`\(\*Manager_moby25348\)\.init\(.* \[sync\.WaitGroup\.Wait\]`,
   461  		),
   462  		makeFlakyTest("Moby27782",
   463  			`\(\*JSONFileLogger_moby27782\)\.readLogs\(.* \[sync\.Cond\.Wait\]`,
   464  			`\(\*Watcher_moby27782\)\.readEvents\(.* \[select\]`,
   465  		),
   466  		makeFlakyTest("Moby28462",
   467  			`monitor_moby28462\(.* \[sync\.Mutex\.Lock\]`,
   468  			`\(\*Daemon_moby28462\)\.StateChanged\(.* \[chan send\]`,
   469  		),
   470  		makeTest("Moby30408",
   471  			`Moby30408\.func2\(.* \[chan receive\]`,
   472  			`testActive_moby30408\.func1\(.* \[sync\.Cond\.Wait\]`,
   473  		),
   474  		makeFlakyTest("Moby33781",
   475  			`monitor_moby33781\.func1\(.* \[chan send\]`,
   476  		),
   477  		makeFlakyTest("Moby36114",
   478  			`\(\*serviceVM_moby36114\)\.hotAddVHDsAtStart\(.* \[sync\.Mutex\.Lock\]`,
   479  		),
   480  		makeFlakyTest("Serving2137",
   481  			`\(\*Breaker_serving2137\)\.concurrentRequest\.func1\(.* \[chan send\]`,
   482  			`\(\*Breaker_serving2137\)\.concurrentRequest\.func1\(.* \[sync\.Mutex\.Lock\]`,
   483  			`Serving2137\.func2\(.* \[chan receive\]`,
   484  		),
   485  		makeTest("Syncthing4829",
   486  			`Syncthing4829\.func2\(.* \[sync\.RWMutex\.RLock\]`,
   487  		),
   488  		makeTest("Syncthing5795",
   489  			`\(\*rawConnection_syncthing5795\)\.dispatcherLoop\(.* \[chan receive\]`,
   490  			`Syncthing5795\.func2.* \[chan receive\]`,
   491  		),
   492  	}
   493  
   494  	// Combine all test cases into a single list.
   495  	testCases := append(microTests, stressTestCases...)
   496  	testCases = append(testCases, patternTestCases...)
   497  
   498  	runTests := func(exepath string, testCases []testCase) {
   499  
   500  		// Build the test program once.
   501  		exe, err := buildTestProg(t, exepath)
   502  		if err != nil {
   503  			t.Fatal(fmt.Sprintf("building testgoroutineleakprofile failed: %v", err))
   504  		}
   505  
   506  		for _, tcase := range testCases {
   507  			t.Run(tcase.name, func(t *testing.T) {
   508  				t.Parallel()
   509  
   510  				cmdEnv := []string{
   511  					"GODEBUG=asyncpreemptoff=1",
   512  				}
   513  
   514  				if tcase.simple {
   515  					// If the test is simple, set GOMAXPROCS=1 in order to better
   516  					// control the behavior of the scheduler.
   517  					cmdEnv = append(cmdEnv, "GOMAXPROCS=1")
   518  				}
   519  
   520  				var output string
   521  				for i := 0; i < tcase.repetitions; i++ {
   522  					// Run program for one repetition and get runOutput trace.
   523  					runOutput, err := runBuiltTestProgErr(t, exe, tcase.name, cmdEnv...)
   524  					if len(runOutput) == 0 {
   525  						t.Errorf("Test %s produced no output. Is the goroutine leak profile collected?", tcase.name)
   526  					}
   527  					// Test cases must not end in a non-zero exit code, or otherwise experience a failure to
   528  					// actually execute.
   529  					if err != nil {
   530  						t.Errorf("unexpected failure\noutput:\n%s\n\n", runOutput)
   531  					}
   532  
   533  					output += runOutput + "\n\n"
   534  				}
   535  
   536  				// Extract all the goroutine leaks
   537  				foundLeaks := extractLeaks(output)
   538  
   539  				// If the test case was not expected to produce leaks, but some were reported,
   540  				// stop the test immediately. Zero tolerance policy for false positives.
   541  				if len(tcase.expectedLeaks)+len(tcase.flakyLeaks) == 0 && len(foundLeaks) > 0 {
   542  					t.Errorf("output:\n%s\n\ngoroutines leaks detected in case with no leaks", output)
   543  				}
   544  
   545  				unexpectedLeaks := make([]string, 0, len(foundLeaks))
   546  
   547  				// Parse every leak and check if it is expected (maybe as a flaky leak).
   548  			leaks:
   549  				for _, leak := range foundLeaks {
   550  					// Check if the leak is expected.
   551  					// If it is, check whether it has been encountered before.
   552  					var foundNew bool
   553  					var leakPattern *regexp.Regexp
   554  
   555  					for expectedLeak, ok := range tcase.expectedLeaks {
   556  						if expectedLeak.MatchString(leak) {
   557  							if !ok {
   558  								foundNew = true
   559  							}
   560  
   561  							leakPattern = expectedLeak
   562  							break
   563  						}
   564  					}
   565  
   566  					if foundNew {
   567  						// Only bother writing if we found a new leak.
   568  						tcase.expectedLeaks[leakPattern] = true
   569  					}
   570  
   571  					if leakPattern == nil {
   572  						// We are dealing with a leak not marked as expected.
   573  						// Check if it is a flaky leak.
   574  						for flakyLeak := range tcase.flakyLeaks {
   575  							if flakyLeak.MatchString(leak) {
   576  								// The leak is flaky. Carry on to the next line.
   577  								continue leaks
   578  							}
   579  						}
   580  
   581  						unexpectedLeaks = append(unexpectedLeaks, leak)
   582  					}
   583  				}
   584  
   585  				missingLeakStrs := make([]string, 0, len(tcase.expectedLeaks))
   586  				for expectedLeak, found := range tcase.expectedLeaks {
   587  					if !found {
   588  						missingLeakStrs = append(missingLeakStrs, expectedLeak.String())
   589  					}
   590  				}
   591  
   592  				var errors []error
   593  				if len(unexpectedLeaks) > 0 {
   594  					errors = append(errors, fmt.Errorf("unexpected goroutine leaks:\n%s\n", strings.Join(unexpectedLeaks, "\n")))
   595  				}
   596  				if len(missingLeakStrs) > 0 {
   597  					errors = append(errors, fmt.Errorf("missing expected leaks:\n%s\n", strings.Join(missingLeakStrs, ", ")))
   598  				}
   599  				if len(errors) > 0 {
   600  					t.Fatalf("Failed with the following errors:\n%s\n\noutput:\n%s", errors, output)
   601  				}
   602  			})
   603  		}
   604  	}
   605  
   606  	runTests("testgoroutineleakprofile", testCases)
   607  	runTests("testgoroutineleakprofile/goker", gokerTestCases)
   608  }
   609
View as plain text