netpoll_epoll.go

     1  // Copyright 2013 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  //go:build linux
     6  
     7  package runtime
     8  
     9  import (
    10  	"internal/runtime/atomic"
    11  	"internal/runtime/syscall/linux"
    12  	"unsafe"
    13  )
    14  
    15  var (
    16  	epfd           int32         = -1 // epoll descriptor
    17  	netpollEventFd uintptr            // eventfd for netpollBreak
    18  	netpollWakeSig atomic.Uint32      // used to avoid duplicate calls of netpollBreak
    19  )
    20  
    21  // netpollPackData stores pd and its fdseq in the epoll data field of ev as
    22  // a taggedPointer. A nil pd marks the netpollBreak eventfd: netpollopen always
    23  // registers a non-nil *pollDesc, so netpollUnpackData can tell the two apart
    24  // by the pointer alone, on any word size or endianness.
    25  func netpollPackData(ev *linux.EpollEvent, pd *pollDesc, fdseq uintptr) {
    26  	*(*taggedPointer)(unsafe.Pointer(&ev.Data)) = taggedPointerPack(unsafe.Pointer(pd), fdseq)
    27  }
    28  
    29  // netpollUnpackData returns the pollDesc and fdseq stored by netpollPackData.
    30  // pd is nil for the netpollBreak eventfd.
    31  func netpollUnpackData(ev *linux.EpollEvent) (pd *pollDesc, fdseq uintptr) {
    32  	tp := *(*taggedPointer)(unsafe.Pointer(&ev.Data))
    33  	return (*pollDesc)(tp.pointer()), tp.tag()
    34  }
    35  
    36  func netpollinit() {
    37  	var errno uintptr
    38  	epfd, errno = linux.EpollCreate1(linux.EPOLL_CLOEXEC)
    39  	if errno != 0 {
    40  		println("runtime: epollcreate failed with", errno)
    41  		throw("runtime: netpollinit failed")
    42  	}
    43  	efd, errno := linux.Eventfd(0, linux.EFD_CLOEXEC|linux.EFD_NONBLOCK)
    44  	if errno != 0 {
    45  		println("runtime: eventfd failed with", errno)
    46  		throw("runtime: eventfd failed")
    47  	}
    48  	ev := linux.EpollEvent{
    49  		Events: linux.EPOLLIN,
    50  	}
    51  	netpollPackData(&ev, nil, 0)
    52  	errno = linux.EpollCtl(epfd, linux.EPOLL_CTL_ADD, efd, &ev)
    53  	if errno != 0 {
    54  		println("runtime: epollctl failed with", errno)
    55  		throw("runtime: epollctl failed")
    56  	}
    57  	netpollEventFd = uintptr(efd)
    58  }
    59  
    60  func netpollIsPollDescriptor(fd uintptr) bool {
    61  	return fd == uintptr(epfd) || fd == netpollEventFd
    62  }
    63  
    64  func netpollopen(fd uintptr, pd *pollDesc) uintptr {
    65  	var ev linux.EpollEvent
    66  	ev.Events = linux.EPOLLIN | linux.EPOLLOUT | linux.EPOLLRDHUP | linux.EPOLLET
    67  	netpollPackData(&ev, pd, pd.fdseq.Load())
    68  	return linux.EpollCtl(epfd, linux.EPOLL_CTL_ADD, int32(fd), &ev)
    69  }
    70  
    71  func netpollclose(fd uintptr) uintptr {
    72  	var ev linux.EpollEvent
    73  	return linux.EpollCtl(epfd, linux.EPOLL_CTL_DEL, int32(fd), &ev)
    74  }
    75  
    76  func netpollarm(pd *pollDesc, mode int) {
    77  	throw("runtime: unused")
    78  }
    79  
    80  // netpollBreak interrupts an epollwait.
    81  func netpollBreak() {
    82  	// Failing to cas indicates there is an in-flight wakeup, so we're done here.
    83  	if !netpollWakeSig.CompareAndSwap(0, 1) {
    84  		return
    85  	}
    86  
    87  	var one uint64 = 1
    88  	oneSize := int32(unsafe.Sizeof(one))
    89  	for {
    90  		n := write(netpollEventFd, noescape(unsafe.Pointer(&one)), oneSize)
    91  		if n == oneSize {
    92  			break
    93  		}
    94  		if n == -_EINTR {
    95  			continue
    96  		}
    97  		if n == -_EAGAIN {
    98  			return
    99  		}
   100  		println("runtime: netpollBreak write failed with", -n)
   101  		throw("runtime: netpollBreak write failed")
   102  	}
   103  }
   104  
   105  // netpoll checks for ready network connections.
   106  // Returns a list of goroutines that become runnable,
   107  // and a delta to add to netpollWaiters.
   108  // This must never return an empty list with a non-zero delta.
   109  //
   110  // delay < 0: blocks indefinitely
   111  // delay == 0: does not block, just polls
   112  // delay > 0: block for up to that many nanoseconds
   113  func netpoll(delay int64) (gList, int32) {
   114  	if epfd == -1 {
   115  		return gList{}, 0
   116  	}
   117  	var waitms int32
   118  	if delay < 0 {
   119  		waitms = -1
   120  	} else if delay == 0 {
   121  		waitms = 0
   122  	} else if delay < 1e6 {
   123  		waitms = 1
   124  	} else if delay < 1e15 {
   125  		waitms = int32(delay / 1e6)
   126  	} else {
   127  		// An arbitrary cap on how long to wait for a timer.
   128  		// 1e9 ms == ~11.5 days.
   129  		waitms = 1e9
   130  	}
   131  	var events [128]linux.EpollEvent
   132  retry:
   133  	n, errno := linux.EpollWait(epfd, events[:], int32(len(events)), waitms)
   134  	if errno != 0 {
   135  		if errno != _EINTR {
   136  			println("runtime: epollwait on fd", epfd, "failed with", errno)
   137  			throw("runtime: netpoll failed")
   138  		}
   139  		// If a timed sleep was interrupted, just return to
   140  		// recalculate how long we should sleep now.
   141  		if waitms > 0 {
   142  			return gList{}, 0
   143  		}
   144  		goto retry
   145  	}
   146  	var toRun gList
   147  	delta := int32(0)
   148  	for i := int32(0); i < n; i++ {
   149  		ev := events[i]
   150  		if ev.Events == 0 {
   151  			continue
   152  		}
   153  
   154  		pd, tag := netpollUnpackData(&ev)
   155  		// pd == nil denotes a netpollBreak eventfd.
   156  		if pd == nil {
   157  			if ev.Events != linux.EPOLLIN {
   158  				println("runtime: netpoll: eventfd ready for", ev.Events)
   159  				throw("runtime: netpoll: eventfd ready for something unexpected")
   160  			}
   161  			if delay != 0 {
   162  				// netpollBreak could be picked up by a
   163  				// nonblocking poll. Only read the 8-byte
   164  				// integer if blocking.
   165  				// Since EFD_SEMAPHORE was not specified,
   166  				// the eventfd counter will be reset to 0.
   167  				var one uint64
   168  				read(int32(netpollEventFd), noescape(unsafe.Pointer(&one)), int32(unsafe.Sizeof(one)))
   169  				netpollWakeSig.Store(0)
   170  			}
   171  			continue
   172  		}
   173  
   174  		var mode int32
   175  		if ev.Events&(linux.EPOLLIN|linux.EPOLLRDHUP|linux.EPOLLHUP|linux.EPOLLERR) != 0 {
   176  			mode += 'r'
   177  		}
   178  		if ev.Events&(linux.EPOLLOUT|linux.EPOLLHUP|linux.EPOLLERR) != 0 {
   179  			mode += 'w'
   180  		}
   181  		if mode != 0 {
   182  			if pd.fdseq.Load() == tag {
   183  				pd.setEventErr(ev.Events == linux.EPOLLERR, tag)
   184  				delta += netpollready(&toRun, pd, mode)
   185  			}
   186  		}
   187  	}
   188  	return toRun, delta
   189  }
   190
View as plain text