Source file
src/runtime/race.go
1
2
3
4
5
6
7 package runtime
8
9 import (
10 "internal/abi"
11 "unsafe"
12 )
13
14
15
16 func RaceRead(addr unsafe.Pointer)
17 func RaceWrite(addr unsafe.Pointer)
18 func RaceReadRange(addr unsafe.Pointer, len int)
19 func RaceWriteRange(addr unsafe.Pointer, len int)
20
21 func RaceErrors() int {
22 var n uint64
23 racecall(&__tsan_report_count, uintptr(unsafe.Pointer(&n)), 0, 0, 0)
24 return int(n)
25 }
26
27
28
29
30
31
32
33
34
35
36
37 func RaceAcquire(addr unsafe.Pointer) {
38 raceacquire(addr)
39 }
40
41
42
43
44
45
46
47
48 func RaceRelease(addr unsafe.Pointer) {
49 racerelease(addr)
50 }
51
52
53
54
55
56
57
58
59 func RaceReleaseMerge(addr unsafe.Pointer) {
60 racereleasemerge(addr)
61 }
62
63
64
65
66
67
68
69 func RaceDisable() {
70 _g_ := getg()
71 if _g_.raceignore == 0 {
72 racecall(&__tsan_go_ignore_sync_begin, _g_.racectx, 0, 0, 0)
73 }
74 _g_.raceignore++
75 }
76
77
78
79
80 func RaceEnable() {
81 _g_ := getg()
82 _g_.raceignore--
83 if _g_.raceignore == 0 {
84 racecall(&__tsan_go_ignore_sync_end, _g_.racectx, 0, 0, 0)
85 }
86 }
87
88
89
90 const raceenabled = true
91
92
93
94
95 func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
96 kind := t.kind & kindMask
97 if kind == kindArray || kind == kindStruct {
98
99
100 racereadrangepc(addr, t.size, callerpc, pc)
101 } else {
102
103
104 racereadpc(addr, callerpc, pc)
105 }
106 }
107
108 func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
109 kind := t.kind & kindMask
110 if kind == kindArray || kind == kindStruct {
111
112
113 racewriterangepc(addr, t.size, callerpc, pc)
114 } else {
115
116
117 racewritepc(addr, callerpc, pc)
118 }
119 }
120
121
122 func racereadpc(addr unsafe.Pointer, callpc, pc uintptr)
123
124
125 func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
126
127 type symbolizeCodeContext struct {
128 pc uintptr
129 fn *byte
130 file *byte
131 line uintptr
132 off uintptr
133 res uintptr
134 }
135
136 var qq = [...]byte{'?', '?', 0}
137 var dash = [...]byte{'-', 0}
138
139 const (
140 raceGetProcCmd = iota
141 raceSymbolizeCodeCmd
142 raceSymbolizeDataCmd
143 )
144
145
146 func racecallback(cmd uintptr, ctx unsafe.Pointer) {
147 switch cmd {
148 case raceGetProcCmd:
149 throw("should have been handled by racecallbackthunk")
150 case raceSymbolizeCodeCmd:
151 raceSymbolizeCode((*symbolizeCodeContext)(ctx))
152 case raceSymbolizeDataCmd:
153 raceSymbolizeData((*symbolizeDataContext)(ctx))
154 default:
155 throw("unknown command")
156 }
157 }
158
159
160
161
162
163
164
165
166
167
168
169
170
171 func raceSymbolizeCode(ctx *symbolizeCodeContext) {
172 pc := ctx.pc
173 fi := findfunc(pc)
174 f := fi._Func()
175 if f != nil {
176 file, line := f.FileLine(pc)
177 if line != 0 {
178 if inldata := funcdata(fi, _FUNCDATA_InlTree); inldata != nil {
179 inltree := (*[1 << 20]inlinedCall)(inldata)
180 for {
181 ix := pcdatavalue(fi, _PCDATA_InlTreeIndex, pc, nil)
182 if ix >= 0 {
183 if inltree[ix].funcID == funcID_wrapper {
184
185
186 pc = f.Entry() + uintptr(inltree[ix].parentPc)
187 continue
188 }
189 ctx.pc = f.Entry() + uintptr(inltree[ix].parentPc)
190 ctx.fn = cfuncnameFromNameoff(fi, inltree[ix].func_)
191 ctx.line = uintptr(line)
192 ctx.file = &bytes(file)[0]
193 ctx.off = pc - f.Entry()
194 ctx.res = 1
195 return
196 }
197 break
198 }
199 }
200 ctx.fn = cfuncname(fi)
201 ctx.line = uintptr(line)
202 ctx.file = &bytes(file)[0]
203 ctx.off = pc - f.Entry()
204 ctx.res = 1
205 return
206 }
207 }
208 ctx.fn = &qq[0]
209 ctx.file = &dash[0]
210 ctx.line = 0
211 ctx.off = ctx.pc
212 ctx.res = 1
213 }
214
215 type symbolizeDataContext struct {
216 addr uintptr
217 heap uintptr
218 start uintptr
219 size uintptr
220 name *byte
221 file *byte
222 line uintptr
223 res uintptr
224 }
225
226 func raceSymbolizeData(ctx *symbolizeDataContext) {
227 if base, span, _ := findObject(ctx.addr, 0, 0); base != 0 {
228 ctx.heap = 1
229 ctx.start = base
230 ctx.size = span.elemsize
231 ctx.res = 1
232 }
233 }
234
235
236
237
238 var __tsan_init byte
239
240
241 var __tsan_fini byte
242
243
244 var __tsan_proc_create byte
245
246
247 var __tsan_proc_destroy byte
248
249
250 var __tsan_map_shadow byte
251
252
253 var __tsan_finalizer_goroutine byte
254
255
256 var __tsan_go_start byte
257
258
259 var __tsan_go_end byte
260
261
262 var __tsan_malloc byte
263
264
265 var __tsan_free byte
266
267
268 var __tsan_acquire byte
269
270
271 var __tsan_release byte
272
273
274 var __tsan_release_acquire byte
275
276
277 var __tsan_release_merge byte
278
279
280 var __tsan_go_ignore_sync_begin byte
281
282
283 var __tsan_go_ignore_sync_end byte
284
285
286 var __tsan_report_count byte
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331 var racedatastart uintptr
332 var racedataend uintptr
333
334
335 var racearenastart uintptr
336 var racearenaend uintptr
337
338 func racefuncenter(callpc uintptr)
339 func racefuncenterfp(fp uintptr)
340 func racefuncexit()
341 func raceread(addr uintptr)
342 func racewrite(addr uintptr)
343 func racereadrange(addr, size uintptr)
344 func racewriterange(addr, size uintptr)
345 func racereadrangepc1(addr, size, pc uintptr)
346 func racewriterangepc1(addr, size, pc uintptr)
347 func racecallbackthunk(uintptr)
348
349
350
351 func racecall(fn *byte, arg0, arg1, arg2, arg3 uintptr)
352
353
354
355
356 func isvalidaddr(addr unsafe.Pointer) bool {
357 return racearenastart <= uintptr(addr) && uintptr(addr) < racearenaend ||
358 racedatastart <= uintptr(addr) && uintptr(addr) < racedataend
359 }
360
361
362 func raceinit() (gctx, pctx uintptr) {
363
364 if !iscgo {
365 throw("raceinit: race build must use cgo")
366 }
367
368 racecall(&__tsan_init, uintptr(unsafe.Pointer(&gctx)), uintptr(unsafe.Pointer(&pctx)), abi.FuncPCABI0(racecallbackthunk), 0)
369
370
371 start := ^uintptr(0)
372 end := uintptr(0)
373 if start > firstmoduledata.noptrdata {
374 start = firstmoduledata.noptrdata
375 }
376 if start > firstmoduledata.data {
377 start = firstmoduledata.data
378 }
379 if start > firstmoduledata.noptrbss {
380 start = firstmoduledata.noptrbss
381 }
382 if start > firstmoduledata.bss {
383 start = firstmoduledata.bss
384 }
385 if end < firstmoduledata.enoptrdata {
386 end = firstmoduledata.enoptrdata
387 }
388 if end < firstmoduledata.edata {
389 end = firstmoduledata.edata
390 }
391 if end < firstmoduledata.enoptrbss {
392 end = firstmoduledata.enoptrbss
393 }
394 if end < firstmoduledata.ebss {
395 end = firstmoduledata.ebss
396 }
397 size := alignUp(end-start, _PageSize)
398 racecall(&__tsan_map_shadow, start, size, 0, 0)
399 racedatastart = start
400 racedataend = start + size
401
402 return
403 }
404
405 var raceFiniLock mutex
406
407
408 func racefini() {
409
410
411
412
413
414 lock(&raceFiniLock)
415
416
417 osPreemptExtEnter(getg().m)
418 racecall(&__tsan_fini, 0, 0, 0, 0)
419 }
420
421
422 func raceproccreate() uintptr {
423 var ctx uintptr
424 racecall(&__tsan_proc_create, uintptr(unsafe.Pointer(&ctx)), 0, 0, 0)
425 return ctx
426 }
427
428
429 func raceprocdestroy(ctx uintptr) {
430 racecall(&__tsan_proc_destroy, ctx, 0, 0, 0)
431 }
432
433
434 func racemapshadow(addr unsafe.Pointer, size uintptr) {
435 if racearenastart == 0 {
436 racearenastart = uintptr(addr)
437 }
438 if racearenaend < uintptr(addr)+size {
439 racearenaend = uintptr(addr) + size
440 }
441 racecall(&__tsan_map_shadow, uintptr(addr), size, 0, 0)
442 }
443
444
445 func racemalloc(p unsafe.Pointer, sz uintptr) {
446 racecall(&__tsan_malloc, 0, 0, uintptr(p), sz)
447 }
448
449
450 func racefree(p unsafe.Pointer, sz uintptr) {
451 racecall(&__tsan_free, uintptr(p), sz, 0, 0)
452 }
453
454
455 func racegostart(pc uintptr) uintptr {
456 _g_ := getg()
457 var spawng *g
458 if _g_.m.curg != nil {
459 spawng = _g_.m.curg
460 } else {
461 spawng = _g_
462 }
463
464 var racectx uintptr
465 racecall(&__tsan_go_start, spawng.racectx, uintptr(unsafe.Pointer(&racectx)), pc, 0)
466 return racectx
467 }
468
469
470 func racegoend() {
471 racecall(&__tsan_go_end, getg().racectx, 0, 0, 0)
472 }
473
474
475 func racectxend(racectx uintptr) {
476 racecall(&__tsan_go_end, racectx, 0, 0, 0)
477 }
478
479
480 func racewriterangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
481 _g_ := getg()
482 if _g_ != _g_.m.curg {
483
484
485 return
486 }
487 if callpc != 0 {
488 racefuncenter(callpc)
489 }
490 racewriterangepc1(uintptr(addr), sz, pc)
491 if callpc != 0 {
492 racefuncexit()
493 }
494 }
495
496
497 func racereadrangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
498 _g_ := getg()
499 if _g_ != _g_.m.curg {
500
501
502 return
503 }
504 if callpc != 0 {
505 racefuncenter(callpc)
506 }
507 racereadrangepc1(uintptr(addr), sz, pc)
508 if callpc != 0 {
509 racefuncexit()
510 }
511 }
512
513
514 func raceacquire(addr unsafe.Pointer) {
515 raceacquireg(getg(), addr)
516 }
517
518
519 func raceacquireg(gp *g, addr unsafe.Pointer) {
520 if getg().raceignore != 0 || !isvalidaddr(addr) {
521 return
522 }
523 racecall(&__tsan_acquire, gp.racectx, uintptr(addr), 0, 0)
524 }
525
526
527 func raceacquirectx(racectx uintptr, addr unsafe.Pointer) {
528 if !isvalidaddr(addr) {
529 return
530 }
531 racecall(&__tsan_acquire, racectx, uintptr(addr), 0, 0)
532 }
533
534
535 func racerelease(addr unsafe.Pointer) {
536 racereleaseg(getg(), addr)
537 }
538
539
540 func racereleaseg(gp *g, addr unsafe.Pointer) {
541 if getg().raceignore != 0 || !isvalidaddr(addr) {
542 return
543 }
544 racecall(&__tsan_release, gp.racectx, uintptr(addr), 0, 0)
545 }
546
547
548 func racereleaseacquire(addr unsafe.Pointer) {
549 racereleaseacquireg(getg(), addr)
550 }
551
552
553 func racereleaseacquireg(gp *g, addr unsafe.Pointer) {
554 if getg().raceignore != 0 || !isvalidaddr(addr) {
555 return
556 }
557 racecall(&__tsan_release_acquire, gp.racectx, uintptr(addr), 0, 0)
558 }
559
560
561 func racereleasemerge(addr unsafe.Pointer) {
562 racereleasemergeg(getg(), addr)
563 }
564
565
566 func racereleasemergeg(gp *g, addr unsafe.Pointer) {
567 if getg().raceignore != 0 || !isvalidaddr(addr) {
568 return
569 }
570 racecall(&__tsan_release_merge, gp.racectx, uintptr(addr), 0, 0)
571 }
572
573
574 func racefingo() {
575 racecall(&__tsan_finalizer_goroutine, getg().racectx, 0, 0, 0)
576 }
577
578
579
580
581
582
583 func abigen_sync_atomic_LoadInt32(addr *int32) (val int32)
584
585
586 func abigen_sync_atomic_LoadInt64(addr *int64) (val int64)
587
588
589 func abigen_sync_atomic_LoadUint32(addr *uint32) (val uint32)
590
591
592 func abigen_sync_atomic_LoadUint64(addr *uint64) (val uint64)
593
594
595 func abigen_sync_atomic_LoadUintptr(addr *uintptr) (val uintptr)
596
597
598 func abigen_sync_atomic_LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer)
599
600
601 func abigen_sync_atomic_StoreInt32(addr *int32, val int32)
602
603
604 func abigen_sync_atomic_StoreInt64(addr *int64, val int64)
605
606
607 func abigen_sync_atomic_StoreUint32(addr *uint32, val uint32)
608
609
610 func abigen_sync_atomic_StoreUint64(addr *uint64, val uint64)
611
612
613 func abigen_sync_atomic_SwapInt32(addr *int32, new int32) (old int32)
614
615
616 func abigen_sync_atomic_SwapInt64(addr *int64, new int64) (old int64)
617
618
619 func abigen_sync_atomic_SwapUint32(addr *uint32, new uint32) (old uint32)
620
621
622 func abigen_sync_atomic_SwapUint64(addr *uint64, new uint64) (old uint64)
623
624
625 func abigen_sync_atomic_AddInt32(addr *int32, delta int32) (new int32)
626
627
628 func abigen_sync_atomic_AddUint32(addr *uint32, delta uint32) (new uint32)
629
630
631 func abigen_sync_atomic_AddInt64(addr *int64, delta int64) (new int64)
632
633
634 func abigen_sync_atomic_AddUint64(addr *uint64, delta uint64) (new uint64)
635
636
637 func abigen_sync_atomic_AddUintptr(addr *uintptr, delta uintptr) (new uintptr)
638
639
640 func abigen_sync_atomic_CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)
641
642
643 func abigen_sync_atomic_CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool)
644
645
646 func abigen_sync_atomic_CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool)
647
648
649 func abigen_sync_atomic_CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool)
650
View as plain text