Source file
src/runtime/netpoll.go
1
2
3
4
5
6
7 package runtime
8
9 import (
10 "runtime/internal/atomic"
11 "runtime/internal/sys"
12 "unsafe"
13 )
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42 const (
43 pollNoError = 0
44 pollErrClosing = 1
45 pollErrTimeout = 2
46 pollErrNotPollable = 3
47 )
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62 const (
63 pdNil uintptr = 0
64 pdReady uintptr = 1
65 pdWait uintptr = 2
66 )
67
68 const pollBlockSize = 4 * 1024
69
70
71
72
73 type pollDesc struct {
74 _ sys.NotInHeap
75 link *pollDesc
76 fd uintptr
77 fdseq atomic.Uintptr
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94 atomicInfo atomic.Uint32
95
96
97
98 rg atomic.Uintptr
99 wg atomic.Uintptr
100
101 lock mutex
102 closing bool
103 user uint32
104 rseq uintptr
105 rt timer
106 rd int64
107 wseq uintptr
108 wt timer
109 wd int64
110 self *pollDesc
111 }
112
113
114
115
116
117 type pollInfo uint32
118
119 const (
120 pollClosing = 1 << iota
121 pollEventErr
122 pollExpiredReadDeadline
123 pollExpiredWriteDeadline
124 pollFDSeq
125 )
126
127 const (
128 pollFDSeqBits = 20
129 pollFDSeqMask = 1<<pollFDSeqBits - 1
130 )
131
132 func (i pollInfo) closing() bool { return i&pollClosing != 0 }
133 func (i pollInfo) eventErr() bool { return i&pollEventErr != 0 }
134 func (i pollInfo) expiredReadDeadline() bool { return i&pollExpiredReadDeadline != 0 }
135 func (i pollInfo) expiredWriteDeadline() bool { return i&pollExpiredWriteDeadline != 0 }
136
137
138 func (pd *pollDesc) info() pollInfo {
139 return pollInfo(pd.atomicInfo.Load())
140 }
141
142
143
144
145
146
147
148
149 func (pd *pollDesc) publishInfo() {
150 var info uint32
151 if pd.closing {
152 info |= pollClosing
153 }
154 if pd.rd < 0 {
155 info |= pollExpiredReadDeadline
156 }
157 if pd.wd < 0 {
158 info |= pollExpiredWriteDeadline
159 }
160 info |= uint32(pd.fdseq.Load()&pollFDSeqMask) << pollFDSeq
161
162
163 x := pd.atomicInfo.Load()
164 for !pd.atomicInfo.CompareAndSwap(x, (x&pollEventErr)|info) {
165 x = pd.atomicInfo.Load()
166 }
167 }
168
169
170
171
172 func (pd *pollDesc) setEventErr(b bool, seq uintptr) {
173 mSeq := uint32(seq & pollFDSeqMask)
174 x := pd.atomicInfo.Load()
175 xSeq := (x >> pollFDSeq) & pollFDSeqMask
176 if seq != 0 && xSeq != mSeq {
177 return
178 }
179 for (x&pollEventErr != 0) != b && !pd.atomicInfo.CompareAndSwap(x, x^pollEventErr) {
180 x = pd.atomicInfo.Load()
181 xSeq := (x >> pollFDSeq) & pollFDSeqMask
182 if seq != 0 && xSeq != mSeq {
183 return
184 }
185 }
186 }
187
188 type pollCache struct {
189 lock mutex
190 first *pollDesc
191
192
193
194
195
196 }
197
198 var (
199 netpollInitLock mutex
200 netpollInited atomic.Uint32
201
202 pollcache pollCache
203 netpollWaiters atomic.Uint32
204 )
205
206
207 func poll_runtime_pollServerInit() {
208 netpollGenericInit()
209 }
210
211 func netpollGenericInit() {
212 if netpollInited.Load() == 0 {
213 lockInit(&netpollInitLock, lockRankNetpollInit)
214 lock(&netpollInitLock)
215 if netpollInited.Load() == 0 {
216 netpollinit()
217 netpollInited.Store(1)
218 }
219 unlock(&netpollInitLock)
220 }
221 }
222
223 func netpollinited() bool {
224 return netpollInited.Load() != 0
225 }
226
227
228
229
230
231 func poll_runtime_isPollServerDescriptor(fd uintptr) bool {
232 return netpollIsPollDescriptor(fd)
233 }
234
235
236 func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) {
237 pd := pollcache.alloc()
238 lock(&pd.lock)
239 wg := pd.wg.Load()
240 if wg != pdNil && wg != pdReady {
241 throw("runtime: blocked write on free polldesc")
242 }
243 rg := pd.rg.Load()
244 if rg != pdNil && rg != pdReady {
245 throw("runtime: blocked read on free polldesc")
246 }
247 pd.fd = fd
248 if pd.fdseq.Load() == 0 {
249
250 pd.fdseq.Store(1)
251 }
252 pd.closing = false
253 pd.setEventErr(false, 0)
254 pd.rseq++
255 pd.rg.Store(pdNil)
256 pd.rd = 0
257 pd.wseq++
258 pd.wg.Store(pdNil)
259 pd.wd = 0
260 pd.self = pd
261 pd.publishInfo()
262 unlock(&pd.lock)
263
264 errno := netpollopen(fd, pd)
265 if errno != 0 {
266 pollcache.free(pd)
267 return nil, int(errno)
268 }
269 return pd, 0
270 }
271
272
273 func poll_runtime_pollClose(pd *pollDesc) {
274 if !pd.closing {
275 throw("runtime: close polldesc w/o unblock")
276 }
277 wg := pd.wg.Load()
278 if wg != pdNil && wg != pdReady {
279 throw("runtime: blocked write on closing polldesc")
280 }
281 rg := pd.rg.Load()
282 if rg != pdNil && rg != pdReady {
283 throw("runtime: blocked read on closing polldesc")
284 }
285 netpollclose(pd.fd)
286 pollcache.free(pd)
287 }
288
289 func (c *pollCache) free(pd *pollDesc) {
290
291
292 lock(&pd.lock)
293
294
295
296 fdseq := pd.fdseq.Load()
297 fdseq = (fdseq + 1) & (1<<taggedPointerBits - 1)
298 pd.fdseq.Store(fdseq)
299
300 pd.publishInfo()
301
302 unlock(&pd.lock)
303
304 lock(&c.lock)
305 pd.link = c.first
306 c.first = pd
307 unlock(&c.lock)
308 }
309
310
311
312
313
314
315 func poll_runtime_pollReset(pd *pollDesc, mode int) int {
316 errcode := netpollcheckerr(pd, int32(mode))
317 if errcode != pollNoError {
318 return errcode
319 }
320 if mode == 'r' {
321 pd.rg.Store(pdNil)
322 } else if mode == 'w' {
323 pd.wg.Store(pdNil)
324 }
325 return pollNoError
326 }
327
328
329
330
331
332
333
334 func poll_runtime_pollWait(pd *pollDesc, mode int) int {
335 errcode := netpollcheckerr(pd, int32(mode))
336 if errcode != pollNoError {
337 return errcode
338 }
339
340 if GOOS == "solaris" || GOOS == "illumos" || GOOS == "aix" || GOOS == "wasip1" {
341 netpollarm(pd, mode)
342 }
343 for !netpollblock(pd, int32(mode), false) {
344 errcode = netpollcheckerr(pd, int32(mode))
345 if errcode != pollNoError {
346 return errcode
347 }
348
349
350
351 }
352 return pollNoError
353 }
354
355
356 func poll_runtime_pollWaitCanceled(pd *pollDesc, mode int) {
357
358
359 for !netpollblock(pd, int32(mode), true) {
360 }
361 }
362
363
364 func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) {
365 lock(&pd.lock)
366 if pd.closing {
367 unlock(&pd.lock)
368 return
369 }
370 rd0, wd0 := pd.rd, pd.wd
371 combo0 := rd0 > 0 && rd0 == wd0
372 if d > 0 {
373 d += nanotime()
374 if d <= 0 {
375
376
377 d = 1<<63 - 1
378 }
379 }
380 if mode == 'r' || mode == 'r'+'w' {
381 pd.rd = d
382 }
383 if mode == 'w' || mode == 'r'+'w' {
384 pd.wd = d
385 }
386 pd.publishInfo()
387 combo := pd.rd > 0 && pd.rd == pd.wd
388 rtf := netpollReadDeadline
389 if combo {
390 rtf = netpollDeadline
391 }
392 if pd.rt.f == nil {
393 if pd.rd > 0 {
394 pd.rt.f = rtf
395
396
397
398 pd.rt.arg = pd.makeArg()
399 pd.rt.seq = pd.rseq
400 resettimer(&pd.rt, pd.rd)
401 }
402 } else if pd.rd != rd0 || combo != combo0 {
403 pd.rseq++
404 if pd.rd > 0 {
405 modtimer(&pd.rt, pd.rd, 0, rtf, pd.makeArg(), pd.rseq)
406 } else {
407 deltimer(&pd.rt)
408 pd.rt.f = nil
409 }
410 }
411 if pd.wt.f == nil {
412 if pd.wd > 0 && !combo {
413 pd.wt.f = netpollWriteDeadline
414 pd.wt.arg = pd.makeArg()
415 pd.wt.seq = pd.wseq
416 resettimer(&pd.wt, pd.wd)
417 }
418 } else if pd.wd != wd0 || combo != combo0 {
419 pd.wseq++
420 if pd.wd > 0 && !combo {
421 modtimer(&pd.wt, pd.wd, 0, netpollWriteDeadline, pd.makeArg(), pd.wseq)
422 } else {
423 deltimer(&pd.wt)
424 pd.wt.f = nil
425 }
426 }
427
428
429 var rg, wg *g
430 if pd.rd < 0 {
431 rg = netpollunblock(pd, 'r', false)
432 }
433 if pd.wd < 0 {
434 wg = netpollunblock(pd, 'w', false)
435 }
436 unlock(&pd.lock)
437 if rg != nil {
438 netpollgoready(rg, 3)
439 }
440 if wg != nil {
441 netpollgoready(wg, 3)
442 }
443 }
444
445
446 func poll_runtime_pollUnblock(pd *pollDesc) {
447 lock(&pd.lock)
448 if pd.closing {
449 throw("runtime: unblock on closing polldesc")
450 }
451 pd.closing = true
452 pd.rseq++
453 pd.wseq++
454 var rg, wg *g
455 pd.publishInfo()
456 rg = netpollunblock(pd, 'r', false)
457 wg = netpollunblock(pd, 'w', false)
458 if pd.rt.f != nil {
459 deltimer(&pd.rt)
460 pd.rt.f = nil
461 }
462 if pd.wt.f != nil {
463 deltimer(&pd.wt)
464 pd.wt.f = nil
465 }
466 unlock(&pd.lock)
467 if rg != nil {
468 netpollgoready(rg, 3)
469 }
470 if wg != nil {
471 netpollgoready(wg, 3)
472 }
473 }
474
475
476
477
478
479
480
481
482
483
484 func netpollready(toRun *gList, pd *pollDesc, mode int32) {
485 var rg, wg *g
486 if mode == 'r' || mode == 'r'+'w' {
487 rg = netpollunblock(pd, 'r', true)
488 }
489 if mode == 'w' || mode == 'r'+'w' {
490 wg = netpollunblock(pd, 'w', true)
491 }
492 if rg != nil {
493 toRun.push(rg)
494 }
495 if wg != nil {
496 toRun.push(wg)
497 }
498 }
499
500 func netpollcheckerr(pd *pollDesc, mode int32) int {
501 info := pd.info()
502 if info.closing() {
503 return pollErrClosing
504 }
505 if (mode == 'r' && info.expiredReadDeadline()) || (mode == 'w' && info.expiredWriteDeadline()) {
506 return pollErrTimeout
507 }
508
509
510
511 if mode == 'r' && info.eventErr() {
512 return pollErrNotPollable
513 }
514 return pollNoError
515 }
516
517 func netpollblockcommit(gp *g, gpp unsafe.Pointer) bool {
518 r := atomic.Casuintptr((*uintptr)(gpp), pdWait, uintptr(unsafe.Pointer(gp)))
519 if r {
520
521
522
523 netpollWaiters.Add(1)
524 }
525 return r
526 }
527
528 func netpollgoready(gp *g, traceskip int) {
529 netpollWaiters.Add(-1)
530 goready(gp, traceskip+1)
531 }
532
533
534
535
536
537 func netpollblock(pd *pollDesc, mode int32, waitio bool) bool {
538 gpp := &pd.rg
539 if mode == 'w' {
540 gpp = &pd.wg
541 }
542
543
544 for {
545
546 if gpp.CompareAndSwap(pdReady, pdNil) {
547 return true
548 }
549 if gpp.CompareAndSwap(pdNil, pdWait) {
550 break
551 }
552
553
554
555 if v := gpp.Load(); v != pdReady && v != pdNil {
556 throw("runtime: double wait")
557 }
558 }
559
560
561
562
563 if waitio || netpollcheckerr(pd, mode) == pollNoError {
564 gopark(netpollblockcommit, unsafe.Pointer(gpp), waitReasonIOWait, traceBlockNet, 5)
565 }
566
567 old := gpp.Swap(pdNil)
568 if old > pdWait {
569 throw("runtime: corrupted polldesc")
570 }
571 return old == pdReady
572 }
573
574 func netpollunblock(pd *pollDesc, mode int32, ioready bool) *g {
575 gpp := &pd.rg
576 if mode == 'w' {
577 gpp = &pd.wg
578 }
579
580 for {
581 old := gpp.Load()
582 if old == pdReady {
583 return nil
584 }
585 if old == pdNil && !ioready {
586
587
588 return nil
589 }
590 var new uintptr
591 if ioready {
592 new = pdReady
593 }
594 if gpp.CompareAndSwap(old, new) {
595 if old == pdWait {
596 old = pdNil
597 }
598 return (*g)(unsafe.Pointer(old))
599 }
600 }
601 }
602
603 func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) {
604 lock(&pd.lock)
605
606
607 currentSeq := pd.rseq
608 if !read {
609 currentSeq = pd.wseq
610 }
611 if seq != currentSeq {
612
613 unlock(&pd.lock)
614 return
615 }
616 var rg *g
617 if read {
618 if pd.rd <= 0 || pd.rt.f == nil {
619 throw("runtime: inconsistent read deadline")
620 }
621 pd.rd = -1
622 pd.publishInfo()
623 rg = netpollunblock(pd, 'r', false)
624 }
625 var wg *g
626 if write {
627 if pd.wd <= 0 || pd.wt.f == nil && !read {
628 throw("runtime: inconsistent write deadline")
629 }
630 pd.wd = -1
631 pd.publishInfo()
632 wg = netpollunblock(pd, 'w', false)
633 }
634 unlock(&pd.lock)
635 if rg != nil {
636 netpollgoready(rg, 0)
637 }
638 if wg != nil {
639 netpollgoready(wg, 0)
640 }
641 }
642
643 func netpollDeadline(arg any, seq uintptr) {
644 netpolldeadlineimpl(arg.(*pollDesc), seq, true, true)
645 }
646
647 func netpollReadDeadline(arg any, seq uintptr) {
648 netpolldeadlineimpl(arg.(*pollDesc), seq, true, false)
649 }
650
651 func netpollWriteDeadline(arg any, seq uintptr) {
652 netpolldeadlineimpl(arg.(*pollDesc), seq, false, true)
653 }
654
655 func (c *pollCache) alloc() *pollDesc {
656 lock(&c.lock)
657 if c.first == nil {
658 const pdSize = unsafe.Sizeof(pollDesc{})
659 n := pollBlockSize / pdSize
660 if n == 0 {
661 n = 1
662 }
663
664
665 mem := persistentalloc(n*pdSize, 0, &memstats.other_sys)
666 for i := uintptr(0); i < n; i++ {
667 pd := (*pollDesc)(add(mem, i*pdSize))
668 pd.link = c.first
669 c.first = pd
670 }
671 }
672 pd := c.first
673 c.first = pd.link
674 lockInit(&pd.lock, lockRankPollDesc)
675 unlock(&c.lock)
676 return pd
677 }
678
679
680
681
682
683
684 func (pd *pollDesc) makeArg() (i any) {
685 x := (*eface)(unsafe.Pointer(&i))
686 x._type = pdType
687 x.data = unsafe.Pointer(&pd.self)
688 return
689 }
690
691 var (
692 pdEface any = (*pollDesc)(nil)
693 pdType *_type = efaceOf(&pdEface)._type
694 )
695
View as plain text