Source file
src/runtime/lockrank_on.go
1
2
3
4
5
6
7 package runtime
8
9 import (
10 "runtime/internal/atomic"
11 "unsafe"
12 )
13
14
15
16 var worldIsStopped uint32
17
18
19 type lockRankStruct struct {
20
21 rank lockRank
22
23
24 pad int
25 }
26
27 func lockInit(l *mutex, rank lockRank) {
28 l.rank = rank
29 }
30
31 func getLockRank(l *mutex) lockRank {
32 return l.rank
33 }
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48 func lockWithRank(l *mutex, rank lockRank) {
49 if l == &debuglock || l == &paniclk {
50
51
52
53
54
55
56
57
58
59 lock2(l)
60 return
61 }
62 if rank == 0 {
63 rank = lockRankLeafRank
64 }
65 gp := getg()
66
67 systemstack(func() {
68 i := gp.m.locksHeldLen
69 if i >= len(gp.m.locksHeld) {
70 throw("too many locks held concurrently for rank checking")
71 }
72 gp.m.locksHeld[i].rank = rank
73 gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l))
74 gp.m.locksHeldLen++
75
76
77 if i > 0 {
78 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
79 }
80 lock2(l)
81 })
82 }
83
84
85
86
87 func printHeldLocks(gp *g) {
88 if gp.m.locksHeldLen == 0 {
89 println("<none>")
90 return
91 }
92
93 for j, held := range gp.m.locksHeld[:gp.m.locksHeldLen] {
94 println(j, ":", held.rank.String(), held.rank, unsafe.Pointer(gp.m.locksHeld[j].lockAddr))
95 }
96 }
97
98
99
100
101
102
103 func acquireLockRank(rank lockRank) {
104 gp := getg()
105
106 systemstack(func() {
107 i := gp.m.locksHeldLen
108 if i >= len(gp.m.locksHeld) {
109 throw("too many locks held concurrently for rank checking")
110 }
111 gp.m.locksHeld[i].rank = rank
112 gp.m.locksHeld[i].lockAddr = 0
113 gp.m.locksHeldLen++
114
115
116 if i > 0 {
117 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
118 }
119 })
120 }
121
122
123
124
125
126 func checkRanks(gp *g, prevRank, rank lockRank) {
127 rankOK := false
128 if rank < prevRank {
129
130 rankOK = false
131 } else if rank == lockRankLeafRank {
132
133
134 rankOK = prevRank < lockRankLeafRank
135 } else {
136
137
138
139
140
141 list := lockPartialOrder[rank]
142 for _, entry := range list {
143 if entry == prevRank {
144 rankOK = true
145 break
146 }
147 }
148 }
149 if !rankOK {
150 printlock()
151 println(gp.m.procid, " ======")
152 printHeldLocks(gp)
153 throw("lock ordering problem")
154 }
155 }
156
157
158 func unlockWithRank(l *mutex) {
159 if l == &debuglock || l == &paniclk {
160
161 unlock2(l)
162 return
163 }
164 gp := getg()
165 systemstack(func() {
166 found := false
167 for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
168 if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) {
169 found = true
170 copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen])
171 gp.m.locksHeldLen--
172 break
173 }
174 }
175 if !found {
176 println(gp.m.procid, ":", l.rank.String(), l.rank, l)
177 throw("unlock without matching lock acquire")
178 }
179 unlock2(l)
180 })
181 }
182
183
184
185
186
187
188 func releaseLockRank(rank lockRank) {
189 gp := getg()
190 systemstack(func() {
191 found := false
192 for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
193 if gp.m.locksHeld[i].rank == rank && gp.m.locksHeld[i].lockAddr == 0 {
194 found = true
195 copy(gp.m.locksHeld[i:gp.m.locksHeldLen-1], gp.m.locksHeld[i+1:gp.m.locksHeldLen])
196 gp.m.locksHeldLen--
197 break
198 }
199 }
200 if !found {
201 println(gp.m.procid, ":", rank.String(), rank)
202 throw("lockRank release without matching lockRank acquire")
203 }
204 })
205 }
206
207
208 func lockWithRankMayAcquire(l *mutex, rank lockRank) {
209 gp := getg()
210 if gp.m.locksHeldLen == 0 {
211
212 return
213 }
214
215 systemstack(func() {
216 i := gp.m.locksHeldLen
217 if i >= len(gp.m.locksHeld) {
218 throw("too many locks held concurrently for rank checking")
219 }
220
221
222
223 gp.m.locksHeld[i].rank = rank
224 gp.m.locksHeld[i].lockAddr = uintptr(unsafe.Pointer(l))
225 gp.m.locksHeldLen++
226 checkRanks(gp, gp.m.locksHeld[i-1].rank, rank)
227 gp.m.locksHeldLen--
228 })
229 }
230
231
232
233
234 func checkLockHeld(gp *g, l *mutex) bool {
235 for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
236 if gp.m.locksHeld[i].lockAddr == uintptr(unsafe.Pointer(l)) {
237 return true
238 }
239 }
240 return false
241 }
242
243
244
245
246
247
248 func assertLockHeld(l *mutex) {
249 gp := getg()
250
251 held := checkLockHeld(gp, l)
252 if held {
253 return
254 }
255
256
257
258 systemstack(func() {
259 printlock()
260 print("caller requires lock ", l, " (rank ", l.rank.String(), "), holding:\n")
261 printHeldLocks(gp)
262 throw("not holding required lock!")
263 })
264 }
265
266
267
268
269
270
271
272
273
274 func assertRankHeld(r lockRank) {
275 gp := getg()
276
277 for i := gp.m.locksHeldLen - 1; i >= 0; i-- {
278 if gp.m.locksHeld[i].rank == r {
279 return
280 }
281 }
282
283
284
285 systemstack(func() {
286 printlock()
287 print("caller requires lock with rank ", r.String(), "), holding:\n")
288 printHeldLocks(gp)
289 throw("not holding required lock!")
290 })
291 }
292
293
294
295
296
297
298
299
300 func worldStopped() {
301 if stopped := atomic.Xadd(&worldIsStopped, 1); stopped != 1 {
302 systemstack(func() {
303 print("world stop count=", stopped, "\n")
304 throw("recursive world stop")
305 })
306 }
307 }
308
309
310
311
312
313
314
315
316 func worldStarted() {
317 if stopped := atomic.Xadd(&worldIsStopped, -1); stopped != 0 {
318 systemstack(func() {
319 print("world stop count=", stopped, "\n")
320 throw("released non-stopped world stop")
321 })
322 }
323 }
324
325
326
327
328 func checkWorldStopped() bool {
329 stopped := atomic.Load(&worldIsStopped)
330 if stopped > 1 {
331 systemstack(func() {
332 print("inconsistent world stop count=", stopped, "\n")
333 throw("inconsistent world stop count")
334 })
335 }
336
337 return stopped == 1
338 }
339
340
341
342
343
344
345
346 func assertWorldStopped() {
347 if checkWorldStopped() {
348 return
349 }
350
351 throw("world not stopped")
352 }
353
354
355
356
357
358
359
360 func assertWorldStoppedOrLockHeld(l *mutex) {
361 if checkWorldStopped() {
362 return
363 }
364
365 gp := getg()
366 held := checkLockHeld(gp, l)
367 if held {
368 return
369 }
370
371
372
373 systemstack(func() {
374 printlock()
375 print("caller requires world stop or lock ", l, " (rank ", l.rank.String(), "), holding:\n")
376 println("<no world stop>")
377 printHeldLocks(gp)
378 throw("no world stop or required lock!")
379 })
380 }
381
View as plain text