diff options
Diffstat (limited to 'src/runtime/netpoll.go')
-rw-r--r-- | src/runtime/netpoll.go | 129 |
1 files changed, 99 insertions, 30 deletions
diff --git a/src/runtime/netpoll.go b/src/runtime/netpoll.go index 4f8d24427c..72e47bfd29 100644 --- a/src/runtime/netpoll.go +++ b/src/runtime/netpoll.go @@ -72,31 +72,99 @@ const pollBlockSize = 4 * 1024 //go:notinheap type pollDesc struct { link *pollDesc // in pollcache, protected by pollcache.lock + fd uintptr // constant for pollDesc usage lifetime + + // atomicInfo holds bits from closing, rd, and wd, + // which are only ever written while holding the lock, + // summarized for use by netpollcheckerr, + // which cannot acquire the lock. + // After writing these fields under lock in a way that + // might change the summary, code must call publishInfo + // before releasing the lock. + // Code that changes fields and then calls netpollunblock + // (while still holding the lock) must call publishInfo + // before calling netpollunblock, because publishInfo is what + // stops netpollblock from blocking anew + // (by changing the result of netpollcheckerr). + // atomicInfo also holds the eventErr bit, + // recording whether a poll event on the fd got an error; + // atomicInfo is the only source of truth for that bit. + atomicInfo uint32 // atomic pollInfo + + // rg, wg are accessed atomically and hold g pointers. + // (Using atomic.Uintptr here is similar to using guintptr elsewhere.) + rg uintptr // pdReady, pdWait, G waiting for read or nil + wg uintptr // pdReady, pdWait, G waiting for write or nil - // The lock protects pollOpen, pollSetDeadline, pollUnblock and deadlineimpl operations. - // This fully covers seq, rt and wt variables. fd is constant throughout the PollDesc lifetime. - // pollReset, pollWait, pollWaitCanceled and runtime·netpollready (IO readiness notification) - // proceed w/o taking the lock. So closing, everr, rg, rd, wg and wd are manipulated - // in a lock-free way by all operations. - // TODO(golang.org/issue/49008): audit these lock-free fields for continued correctness. - // NOTE(dvyukov): the following code uses uintptr to store *g (rg/wg), - // that will blow up when GC starts moving objects. lock mutex // protects the following fields - fd uintptr closing bool - everr bool // marks event scanning error happened user uint32 // user settable cookie rseq uintptr // protects from stale read timers - rg uintptr // pdReady, pdWait, G waiting for read or nil. Accessed atomically. rt timer // read deadline timer (set if rt.f != nil) - rd int64 // read deadline + rd int64 // read deadline (a nanotime in the future, -1 when expired) wseq uintptr // protects from stale write timers - wg uintptr // pdReady, pdWait, G waiting for write or nil. Accessed atomically. wt timer // write deadline timer - wd int64 // write deadline + wd int64 // write deadline (a nanotime in the future, -1 when expired) self *pollDesc // storage for indirect interface. See (*pollDesc).makeArg. } +// pollInfo is the bits needed by netpollcheckerr, stored atomically, +// mostly duplicating state that is manipulated under lock in pollDesc. +// The one exception is the pollEventErr bit, which is maintained only +// in the pollInfo. +type pollInfo uint32 + +const ( + pollClosing = 1 << iota + pollEventErr + pollExpiredReadDeadline + pollExpiredWriteDeadline +) + +func (i pollInfo) closing() bool { return i&pollClosing != 0 } +func (i pollInfo) eventErr() bool { return i&pollEventErr != 0 } +func (i pollInfo) expiredReadDeadline() bool { return i&pollExpiredReadDeadline != 0 } +func (i pollInfo) expiredWriteDeadline() bool { return i&pollExpiredWriteDeadline != 0 } + +// info returns the pollInfo corresponding to pd. +func (pd *pollDesc) info() pollInfo { + return pollInfo(atomic.Load(&pd.atomicInfo)) +} + +// publishInfo updates pd.atomicInfo (returned by pd.info) +// using the other values in pd. +// It must be called while holding pd.lock, +// and it must be called after changing anything +// that might affect the info bits. +// In practice this means after changing closing +// or changing rd or wd from < 0 to >= 0. +func (pd *pollDesc) publishInfo() { + var info uint32 + if pd.closing { + info |= pollClosing + } + if pd.rd < 0 { + info |= pollExpiredReadDeadline + } + if pd.wd < 0 { + info |= pollExpiredWriteDeadline + } + + // Set all of x except the pollEventErr bit. + x := atomic.Load(&pd.atomicInfo) + for !atomic.Cas(&pd.atomicInfo, x, (x&pollEventErr)|info) { + x = atomic.Load(&pd.atomicInfo) + } +} + +// setEventErr sets the result of pd.info().eventErr() to b. +func (pd *pollDesc) setEventErr(b bool) { + x := atomic.Load(&pd.atomicInfo) + for (x&pollEventErr != 0) != b && !atomic.Cas(&pd.atomicInfo, x, x^pollEventErr) { + x = atomic.Load(&pd.atomicInfo) + } +} + type pollCache struct { lock mutex first *pollDesc @@ -158,7 +226,7 @@ func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) { } pd.fd = fd pd.closing = false - pd.everr = false + pd.setEventErr(false) pd.rseq++ atomic.Storeuintptr(&pd.rg, 0) pd.rd = 0 @@ -166,6 +234,7 @@ func poll_runtime_pollOpen(fd uintptr) (*pollDesc, int) { atomic.Storeuintptr(&pd.wg, 0) pd.wd = 0 pd.self = pd + pd.publishInfo() unlock(&pd.lock) errno := netpollopen(fd, pd) @@ -274,6 +343,7 @@ func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { if mode == 'w' || mode == 'r'+'w' { pd.wd = d } + pd.publishInfo() combo := pd.rd > 0 && pd.rd == pd.wd rtf := netpollReadDeadline if combo { @@ -315,15 +385,13 @@ func poll_runtime_pollSetDeadline(pd *pollDesc, d int64, mode int) { } } // If we set the new deadline in the past, unblock currently pending IO if any. + // Note that pd.publishInfo has already been called, above, immediately after modifying rd and wd. var rg, wg *g - if pd.rd < 0 || pd.wd < 0 { - atomic.StorepNoWB(noescape(unsafe.Pointer(&wg)), nil) // full memory barrier between stores to rd/wd and load of rg/wg in netpollunblock - if pd.rd < 0 { - rg = netpollunblock(pd, 'r', false) - } - if pd.wd < 0 { - wg = netpollunblock(pd, 'w', false) - } + if pd.rd < 0 { + rg = netpollunblock(pd, 'r', false) + } + if pd.wd < 0 { + wg = netpollunblock(pd, 'w', false) } unlock(&pd.lock) if rg != nil { @@ -344,7 +412,7 @@ func poll_runtime_pollUnblock(pd *pollDesc) { pd.rseq++ pd.wseq++ var rg, wg *g - atomic.StorepNoWB(noescape(unsafe.Pointer(&rg)), nil) // full memory barrier between store to closing and read of rg/wg in netpollunblock + pd.publishInfo() rg = netpollunblock(pd, 'r', false) wg = netpollunblock(pd, 'w', false) if pd.rt.f != nil { @@ -389,16 +457,17 @@ func netpollready(toRun *gList, pd *pollDesc, mode int32) { } func netpollcheckerr(pd *pollDesc, mode int32) int { - if pd.closing { + info := pd.info() + if info.closing() { return pollErrClosing } - if (mode == 'r' && pd.rd < 0) || (mode == 'w' && pd.wd < 0) { + if (mode == 'r' && info.expiredReadDeadline()) || (mode == 'w' && info.expiredWriteDeadline()) { return pollErrTimeout } // Report an event scanning error only on a read event. // An error on a write event will be captured in a subsequent // write call that is able to report a more specific error. - if mode == 'r' && pd.everr { + if mode == 'r' && info.eventErr() { return pollErrNotPollable } return pollNoError @@ -449,7 +518,7 @@ func netpollblock(pd *pollDesc, mode int32, waitio bool) bool { // need to recheck error states after setting gpp to pdWait // this is necessary because runtime_pollUnblock/runtime_pollSetDeadline/deadlineimpl - // do the opposite: store to closing/rd/wd, membarrier, load of rg/wg + // do the opposite: store to closing/rd/wd, publishInfo, load of rg/wg if waitio || netpollcheckerr(pd, mode) == 0 { gopark(netpollblockcommit, unsafe.Pointer(gpp), waitReasonIOWait, traceEvGoBlockNet, 5) } @@ -509,7 +578,7 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { throw("runtime: inconsistent read deadline") } pd.rd = -1 - atomic.StorepNoWB(unsafe.Pointer(&pd.rt.f), nil) // full memory barrier between store to rd and load of rg in netpollunblock + pd.publishInfo() rg = netpollunblock(pd, 'r', false) } var wg *g @@ -518,7 +587,7 @@ func netpolldeadlineimpl(pd *pollDesc, seq uintptr, read, write bool) { throw("runtime: inconsistent write deadline") } pd.wd = -1 - atomic.StorepNoWB(unsafe.Pointer(&pd.wt.f), nil) // full memory barrier between store to wd and load of wg in netpollunblock + pd.publishInfo() wg = netpollunblock(pd, 'w', false) } unlock(&pd.lock) |