diff --git a/builder/testdata/binary-size.txt b/builder/testdata/binary-size.txt index a39d05ed1a..c50e27139a 100644 --- a/builder/testdata/binary-size.txt +++ b/builder/testdata/binary-size.txt @@ -1,4 +1,4 @@ target package code rodata data bss hifive1b examples/echo 4542 346 0 2268 -microbit examples/serial 2993 391 8 2264 -wioterminal examples/pininterrupt 8275 1741 148 7496 +microbit examples/serial 2979 389 8 2264 +wioterminal examples/pininterrupt 8307 1741 148 7504 diff --git a/compiler/testdata/channel.ll b/compiler/testdata/channel.ll index 31466ead0f..fb3dd5c349 100644 --- a/compiler/testdata/channel.ll +++ b/compiler/testdata/channel.ll @@ -15,7 +15,7 @@ entry: } ; Function Attrs: nounwind -define hidden void @main.chanIntSend(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanIntSend(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 %chan.value = alloca i32, align 4 @@ -31,13 +31,13 @@ entry: ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.start.p0(ptr nocapture) #2 -declare void @runtime.chanSend(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare void @runtime.chanSend(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.end.p0(ptr nocapture) #2 ; Function Attrs: nounwind -define hidden void @main.chanIntRecv(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanIntRecv(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 %chan.value = alloca i32, align 4 @@ -49,10 +49,10 @@ entry: ret void } -declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nounwind -define hidden void @main.chanZeroSend(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanZeroSend(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 call void @llvm.lifetime.start.p0(ptr nonnull %chan.op) @@ -62,7 +62,7 @@ entry: } ; Function Attrs: nounwind -define hidden void @main.chanZeroRecv(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.chanZeroRecv(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: %chan.op = alloca %runtime.channelOp, align 8 call void @llvm.lifetime.start.p0(ptr nonnull %chan.op) @@ -72,7 +72,7 @@ entry: } ; Function Attrs: nounwind -define hidden void @main.selectZeroRecv(ptr dereferenceable_or_null(36) %ch1, ptr dereferenceable_or_null(36) %ch2, ptr %context) unnamed_addr #1 { +define hidden void @main.selectZeroRecv(ptr dereferenceable_or_null(40) %ch1, ptr dereferenceable_or_null(40) %ch2, ptr %context) unnamed_addr #1 { entry: %select.states.alloca = alloca [2 x %runtime.chanSelectState], align 8 %select.send.value = alloca i32, align 4 diff --git a/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll b/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll index bf117fa975..2479d5e34b 100644 --- a/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll +++ b/compiler/testdata/goroutine-cortex-m-qemu-tasks.ll @@ -146,13 +146,13 @@ declare i32 @llvm.umin.i32(i32, i32) #7 declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) #8 ; Function Attrs: nounwind -define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #0 { +define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #0 { entry: call void @runtime.chanClose(ptr %ch, ptr undef) #11 ret void } -declare void @runtime.chanClose(ptr dereferenceable_or_null(36), ptr) #1 +declare void @runtime.chanClose(ptr dereferenceable_or_null(40), ptr) #1 ; Function Attrs: nounwind define hidden void @main.startInterfaceMethod(ptr %itf.typecode, ptr %itf.value, ptr %context) unnamed_addr #0 { diff --git a/compiler/testdata/goroutine-wasm-asyncify.ll b/compiler/testdata/goroutine-wasm-asyncify.ll index 062ec1a423..f39fbed1aa 100644 --- a/compiler/testdata/goroutine-wasm-asyncify.ll +++ b/compiler/testdata/goroutine-wasm-asyncify.ll @@ -155,13 +155,13 @@ declare i32 @llvm.umin.i32(i32, i32) #7 declare void @llvm.memmove.p0.p0.i32(ptr nocapture writeonly, ptr nocapture readonly, i32, i1 immarg) #8 ; Function Attrs: nounwind -define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(36) %ch, ptr %context) unnamed_addr #1 { +define hidden void @main.closeBuiltinGoroutine(ptr dereferenceable_or_null(40) %ch, ptr %context) unnamed_addr #1 { entry: call void @runtime.chanClose(ptr %ch, ptr undef) #11 ret void } -declare void @runtime.chanClose(ptr dereferenceable_or_null(36), ptr) #0 +declare void @runtime.chanClose(ptr dereferenceable_or_null(40), ptr) #0 ; Function Attrs: nounwind define hidden void @main.startInterfaceMethod(ptr %itf.typecode, ptr %itf.value, ptr %context) unnamed_addr #1 { diff --git a/compiler/testdata/large.ll b/compiler/testdata/large.ll index e792c9d8e9..451cb067fd 100644 --- a/compiler/testdata/large.ll +++ b/compiler/testdata/large.ll @@ -487,7 +487,7 @@ declare void @runtime.hashmapBinarySet(ptr dereferenceable_or_null(52), ptr, ptr declare i1 @runtime.hashmapBinaryGet(ptr dereferenceable_or_null(52), ptr, ptr, i32, ptr) #0 ; Function Attrs: nounwind -define hidden i8 @main.useLargeChannel(ptr dereferenceable_or_null(36) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { +define hidden i8 @main.useLargeChannel(ptr dereferenceable_or_null(40) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { entry: %chan.op1 = alloca %runtime.channelOp, align 8 %chan.op = alloca %runtime.channelOp, align 8 @@ -524,15 +524,15 @@ if.then: ; preds = %entry ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.start.p0(ptr nocapture) #12 -declare void @runtime.chanSend(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare void @runtime.chanSend(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: readwrite) declare void @llvm.lifetime.end.p0(ptr nocapture) #12 -declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(36), ptr, ptr dereferenceable_or_null(16), ptr) #0 +declare i1 @runtime.chanRecv(ptr dereferenceable_or_null(40), ptr, ptr dereferenceable_or_null(16), ptr) #0 ; Function Attrs: nounwind -define hidden i8 @main.selectLargeChannel(ptr dereferenceable_or_null(36) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { +define hidden i8 @main.selectLargeChannel(ptr dereferenceable_or_null(40) %ch, ptr readonly dereferenceable_or_null(1025) %value, ptr %context) unnamed_addr #1 { entry: %select.block.alloca = alloca [2 x %runtime.channelOp], align 8 %select.states.alloca = alloca [2 x %runtime.chanSelectState], align 8 diff --git a/compiler/testdata/pragma.ll b/compiler/testdata/pragma.ll index d8b446b89d..339dfc9a13 100644 --- a/compiler/testdata/pragma.ll +++ b/compiler/testdata/pragma.ll @@ -84,10 +84,10 @@ entry: declare void @main.undefinedFunctionNotInSection(ptr) #0 -declare void @main.doesNotEscapeParam(ptr nocapture dereferenceable_or_null(4), ptr nocapture, i32, i32, ptr nocapture dereferenceable_or_null(36), ptr nocapture, ptr) #0 +declare void @main.doesNotEscapeParam(ptr nocapture dereferenceable_or_null(4), ptr nocapture, i32, i32, ptr nocapture dereferenceable_or_null(40), ptr nocapture, ptr) #0 ; Function Attrs: nounwind -define hidden void @main.stillEscapes(ptr dereferenceable_or_null(4) %a, ptr %b.data, i32 %b.len, i32 %b.cap, ptr dereferenceable_or_null(36) %c, ptr %d, ptr %context) unnamed_addr #1 { +define hidden void @main.stillEscapes(ptr dereferenceable_or_null(4) %a, ptr %b.data, i32 %b.len, i32 %b.cap, ptr dereferenceable_or_null(40) %c, ptr %d, ptr %context) unnamed_addr #1 { entry: ret void } diff --git a/make/test.mk b/make/test.mk index bc4657fbb3..a4ca888902 100644 --- a/make/test.mk +++ b/make/test.mk @@ -223,6 +223,9 @@ tinygo-test: ifneq ($(TEST_PACKAGES_SHORT_HOST),) $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) $(TEST_SKIP_FLAG) -short $(TEST_PACKAGES_SHORT_HOST) endif + $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) -skip='^(TestReflectFuncOf|TestChannelMovedOutOfBubble|TestTimerFromInsideBubble|TestWaitGroupMovedIntoBubble|TestWaitGroupMovedOutOfBubble|TestWaitGroupMovedBetweenBubblesWithNonZeroCount)$$' internal/synctest + $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) -skip='^(TestFatal|TestError|TestVerboseError|TestSkip|TestVerboseSkip|TestHelper|TestHTTPTransport100Continue)$$' testing/synctest + $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) -run='^TestSynctestMarshal$$' encoding/json ifeq ($(TEST_ENCODING_XML),true) $(TINYGO) test $(TEST_ADDITIONAL_FLAGS) $(TEST_SKIP_FLAG) -short -stack-size=16MB encoding/xml endif diff --git a/src/internal/task/futex-cores.go b/src/internal/task/futex-cores.go index 9bf493f25c..70c48f9c2e 100644 --- a/src/internal/task/futex-cores.go +++ b/src/internal/task/futex-cores.go @@ -42,19 +42,21 @@ func (f *Futex) Wait(cmp uint32) (awoken bool) { // Wake a single waiter. func (f *Futex) Wake() { mask := lockFutex() - if t := f.waiters.Pop(); t != nil { + t := f.waiters.Pop() + unlockFutex(mask) + if t != nil { scheduleTask(t) } - unlockFutex(mask) } // Wake all waiters. func (f *Futex) WakeAll() { mask := lockFutex() - for t := f.waiters.Pop(); t != nil; t = f.waiters.Pop() { + waiters := f.waiters.Queue() + unlockFutex(mask) + for t := waiters.Pop(); t != nil; t = waiters.Pop() { scheduleTask(t) } - unlockFutex(mask) } //go:linkname lockFutex runtime.lockFutex diff --git a/src/internal/task/task.go b/src/internal/task/task.go index 8ad04327a9..0421250a86 100644 --- a/src/internal/task/task.go +++ b/src/internal/task/task.go @@ -33,9 +33,15 @@ type Task struct { // Exited is set after a task with a releasable stack has finished. Exited bool + // SynctestBlocked is set while this task is durably blocked. + SynctestBlocked bool + // DeferFrame stores a pointer to the (stack allocated) defer frame of the // goroutine that is used for the recover builtin. DeferFrame unsafe.Pointer + + // SynctestBubble identifies the synctest bubble this task belongs to. + SynctestBubble unsafe.Pointer } const ( @@ -80,3 +86,34 @@ func runtime_freeTaskStack(ptr uintptr) //go:linkname scheduleTask runtime.scheduleTask func scheduleTask(*Task) + +func inheritSynctest(t *Task) { + if !synctestIsEnabled() { + return + } + parent := Current() + if parent == nil || parent.SynctestBubble == nil { + return + } + t.SynctestBubble = parent.SynctestBubble + synctestTaskCreated(t) +} + +func exitSynctest(t *Task) { + if !synctestIsEnabled() { + return + } + if t.SynctestBubble != nil { + synctestTaskExited(t) + t.SynctestBubble = nil + } +} + +//go:linkname synctestTaskCreated runtime.synctestTaskCreated +func synctestTaskCreated(*Task) + +//go:linkname synctestTaskExited runtime.synctestTaskExited +func synctestTaskExited(*Task) + +//go:linkname synctestIsEnabled runtime.synctestIsEnabled +func synctestIsEnabled() bool diff --git a/src/internal/task/task_asyncify.go b/src/internal/task/task_asyncify.go index 955161a494..1a0403d929 100644 --- a/src/internal/task/task_asyncify.go +++ b/src/internal/task/task_asyncify.go @@ -63,6 +63,7 @@ type stackState struct { // The new goroutine is immediately started. func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { t := &Task{} + inheritSynctest(t) addLiveTask(t) t.state.initialize(fn, args, stackSize) scheduleTask(t) diff --git a/src/internal/task/task_exit.go b/src/internal/task/task_exit.go index 2bd008cff1..f844dd6db8 100644 --- a/src/internal/task/task_exit.go +++ b/src/internal/task/task_exit.go @@ -32,6 +32,7 @@ func exit(goexit bool) { if hasReleasableStack { t.Exited = true } + exitSynctest(t) remaining := atomic.AddUint32(&liveTasks, ^uint32(0)) if t == mainTask { if goexit { diff --git a/src/internal/task/task_stack.go b/src/internal/task/task_stack.go index 345b048eb8..8bba8110ef 100644 --- a/src/internal/task/task_stack.go +++ b/src/internal/task/task_stack.go @@ -72,6 +72,7 @@ var startTask [0]uint8 // The new goroutine is scheduled to run later. func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { t := &Task{} + inheritSynctest(t) addLiveTask(t) t.state.initialize(fn, args, stackSize) scheduleTask(t) diff --git a/src/internal/task/task_threads.go b/src/internal/task/task_threads.go index 93277bd9e2..2720d9c344 100644 --- a/src/internal/task/task_threads.go +++ b/src/internal/task/task_threads.go @@ -100,6 +100,7 @@ func (t *Task) Resume() { // Start a new OS thread. func start(fn uintptr, args unsafe.Pointer, stackSize uintptr) { t := &Task{} + inheritSynctest(t) t.state.id = atomic.AddUintptr(&goroutineID, 1) if verbose { println("*** start: ", t.state.id, "from", Current().state.id) @@ -132,6 +133,8 @@ func taskExited(t *Task) { } func exit(t *Task) bool { + exitSynctest(t) + // Remove from the queue. // TODO: this can be made more efficient by using a doubly linked list. activeTaskLock.Lock() diff --git a/src/runtime/chan.go b/src/runtime/chan.go index a961069db8..b7cb7e048d 100644 --- a/src/runtime/chan.go +++ b/src/runtime/chan.go @@ -61,6 +61,7 @@ type channel struct { receivers chanQueue lock task.PMutex buf unsafe.Pointer + synctest unsafe.Pointer } const ( @@ -142,6 +143,23 @@ func chanMake(elementSize uintptr, bufSize uintptr, elementLayout unsafe.Pointer elementSize: elementSize, bufCap: bufSize, buf: alloc(elementSize*bufSize, elementLayout), + synctest: currentTaskSynctestBubble(), + } +} + +func currentTaskSynctestBubble() unsafe.Pointer { + if !synctestIsEnabled() { + return nil + } + if current := task.Current(); current != nil { + return current.SynctestBubble + } + return nil +} + +func (ch *channel) checkSynctest(op string) { + if synctestIsEnabled() && ch.synctest != nil && currentTaskSynctestBubble() != ch.synctest { + runtimeFatal(op + " synctest channel from outside bubble") } } @@ -233,6 +251,7 @@ func chanSend(ch *channel, value unsafe.Pointer, op *channelOp) { // A nil channel blocks forever. Do not schedule this goroutine again. deadlock() } + ch.checkSynctest("send on") mask := interrupt.Disable() ch.lock.Lock() @@ -254,6 +273,9 @@ func chanSend(ch *channel, value unsafe.Pointer, op *channelOp) { op.index = 0 op.value = value ch.senders.push(op) + if synctestIsEnabled() && ch.synctest != nil { + synctestTaskBlock(t) + } ch.lock.Unlock() interrupt.Restore(mask) @@ -313,6 +335,7 @@ func chanRecv(ch *channel, value unsafe.Pointer, op *channelOp) bool { // A nil channel blocks forever. Do not schedule this goroutine again. deadlock() } + ch.checkSynctest("receive on") mask := interrupt.Disable() ch.lock.Lock() @@ -334,6 +357,9 @@ func chanRecv(ch *channel, value unsafe.Pointer, op *channelOp) bool { op.task = t op.index = 0 ch.receivers.push(op) + if synctestIsEnabled() && ch.synctest != nil { + synctestTaskBlock(t) + } ch.lock.Unlock() interrupt.Restore(mask) @@ -351,6 +377,7 @@ func chanClose(ch *channel) { // Not allowed by the language spec. runtimePanic(errCloseNilChannel) } + ch.checkSynctest("close of") mask := interrupt.Disable() ch.lock.Lock() @@ -457,6 +484,12 @@ func unlockAllStates(states []chanSelectState) { // The 'ops' slice must be set if (and only if) this is a blocking select. func chanSelect(recvbuf unsafe.Pointer, states []chanSelectState, ops []channelOp) (uint32, bool) { mask := interrupt.Disable() + var currentBubble unsafe.Pointer + var synctestDurable bool + if synctestIsEnabled() { + currentBubble = currentTaskSynctestBubble() + synctestDurable = currentBubble != nil + } // Lock everything. chanSelectLock.Lock() @@ -475,6 +508,17 @@ func chanSelect(recvbuf unsafe.Pointer, states []chanSelectState, ops []channelO // operation. continue } + if synctestIsEnabled() { + if state.ch.synctest != nil && state.ch.synctest != currentBubble { + unlockAllStates(states) + chanSelectLock.Unlock() + interrupt.Restore(mask) + runtimeFatal("select on synctest channel from outside bubble") + } + if state.ch.synctest == nil { + synctestDurable = false + } + } if state.value == nil { // chan receive if received, ok, sender := state.ch.tryRecv(recvbuf); received { @@ -528,6 +572,9 @@ func chanSelect(recvbuf unsafe.Pointer, states []chanSelectState, ops []channelO state.ch.senders.push(op) } } + if synctestDurable { + synctestTaskBlock(t) + } // Now we wait until one of the send/receive operations can proceed. unlockAllStates(states) diff --git a/src/runtime/runtime_mutex_cores.go b/src/runtime/runtime_mutex_cores.go new file mode 100644 index 0000000000..adaa0b90ec --- /dev/null +++ b/src/runtime/runtime_mutex_cores.go @@ -0,0 +1,21 @@ +//go:build scheduler.cores + +package runtime + +import "internal/task" + +type runtimeSpinLock struct { + state task.Uint32 +} + +func (lock *runtimeSpinLock) Lock() { + for !lock.state.CompareAndSwap(0, 1) { + } +} + +func (lock *runtimeSpinLock) Unlock() { + lock.state.Store(0) +} + +type timerLock = runtimeSpinLock +type synctestLock = runtimeSpinLock diff --git a/src/runtime/runtime_mutex_other.go b/src/runtime/runtime_mutex_other.go new file mode 100644 index 0000000000..c8a26f98ba --- /dev/null +++ b/src/runtime/runtime_mutex_other.go @@ -0,0 +1,8 @@ +//go:build !scheduler.cores + +package runtime + +import "internal/task" + +type timerLock = task.PMutex +type synctestLock = task.PMutex diff --git a/src/runtime/scheduler_cooperative.go b/src/runtime/scheduler_cooperative.go index f4ec3df565..3af507eea0 100644 --- a/src/runtime/scheduler_cooperative.go +++ b/src/runtime/scheduler_cooperative.go @@ -56,7 +56,9 @@ var finalizerIdleGC func() bool func deadlock() { // Keep permanently blocked tasks reachable so their suspended stacks remain // GC roots, but never put them back on the runnable queue. - deadlockedTasks.Push(task.Current()) + current := task.Current() + synctestTaskBlock(current) + deadlockedTasks.Push(current) task.Pause() runtimeFatal("unreachable") } @@ -88,6 +90,7 @@ func goexit() { // Add this task to the end of the run queue. func scheduleTask(t *task.Task) { + synctestTaskWake(t) runqueue.Push(t) } @@ -147,11 +150,12 @@ func addTimer(tim *timerNode) { interrupt.Restore(mask) } -// reAddTimer advances and re-adds a periodic timer (a ticker) after its -// callback has run. The cooperative scheduler runs timer callbacks to -// completion inside the scheduler loop, so a timer can't be stopped or reset -// while its callback is running and the timer can always be re-added directly. +// reAddTimer finishes firing a timer. The cooperative scheduler runs timer +// callbacks to completion, so periodic timers can be re-added directly. func reAddTimer(tn *timerNode) { + if tn.timer.period == 0 { + return + } tn.timer.when += tn.timer.period addTimer(tn) } @@ -291,6 +295,9 @@ func sleep(duration int64) { if duration <= 0 { return } + if synctestSleep(duration) { + return + } addSleepTask(task.Current(), nanosecondsToTicks(duration)) task.Pause() } diff --git a/src/runtime/scheduler_cores.go b/src/runtime/scheduler_cores.go index b26abc7f30..93000c39ec 100644 --- a/src/runtime/scheduler_cores.go +++ b/src/runtime/scheduler_cores.go @@ -28,6 +28,7 @@ var ( func deadlock() { // Call yield without requesting a wakeup. + synctestTaskBlock(task.Current()) task.Pause() trap() } @@ -39,6 +40,7 @@ func goexit() { // Mark the given task as ready to resume. // This is allowed even if the task isn't paused yet, but will pause soon. func scheduleTask(t *task.Task) { + synctestTaskWake(t) schedulerLock.Lock() switch t.RunState { case task.RunStatePaused: @@ -106,9 +108,8 @@ func addTimer(tn *timerNode) { schedulerLock.Unlock() } -// reAddTimer advances and re-adds a periodic timer (a ticker) after its -// callback has run, unless it was stopped or reset while the callback was -// running (in which case it must not be re-added). +// reAddTimer finishes firing a timer. It re-adds periodic timers unless they +// were stopped or reset while the callback was running. func reAddTimer(tn *timerNode) { schedulerLock.Lock() @@ -124,6 +125,10 @@ func reAddTimer(tn *timerNode) { schedulerLock.Unlock() return } + if tn.timer.period == 0 { + schedulerLock.Unlock() + return + } tn.timer.when += tn.timer.period timerQueueAdd(tn) interruptSleepTicksMulticore(tn.whenTicks()) @@ -153,6 +158,9 @@ func sleep(duration int64) { if duration <= 0 { return } + if synctestSleep(duration) { + return + } wakeup := ticks() + nanosecondsToTicks(duration) @@ -231,12 +239,6 @@ func scheduler(_ bool) { schedulerLock.Unlock() tn.callback(tn, delay) schedulerLock.Lock() - // A periodic timer (a ticker) already removed itself from the - // firing list in reAddTimer; a one-shot timer isn't re-added, so - // remove it from the firing list here. - if tn.timer.period == 0 { - firingTimersRemove(tn) - } continue } } diff --git a/src/runtime/scheduler_threads.go b/src/runtime/scheduler_threads.go index 166a342c82..57527e27ee 100644 --- a/src/runtime/scheduler_threads.go +++ b/src/runtime/scheduler_threads.go @@ -35,11 +35,15 @@ func sleep(duration int64) { if duration <= 0 { return } + if synctestSleep(duration) { + return + } sleepTicks(nanosecondsToTicks(duration)) } func deadlock() { + synctestTaskBlock(task.Current()) task.Pause() } @@ -48,6 +52,7 @@ func goexit() { } func scheduleTask(t *task.Task) { + synctestTaskWake(t) t.Resume() } @@ -92,6 +97,7 @@ func timerRunner() { tn := timerQueue timerQueue = tn.next tn.next = nil + delay := ticksToNanoseconds(now - tn.whenTicks()) // Mark the timer as firing, so that a concurrent Stop or Reset (via // removeTimer) can prevent a periodic timer from re-adding itself in its @@ -101,17 +107,7 @@ func timerRunner() { timerQueueLock.Unlock() // Run the callback stored in this timer node. - delay := ticksToNanoseconds(now - tn.whenTicks()) tn.callback(tn, delay) - - // The callback has finished running. A periodic timer (a ticker) already - // removed itself from the firing list in reAddTimer; a one-shot timer - // isn't re-added, so remove it from the firing list here. - timerQueueLock.Lock() - if tn.timer.period == 0 { - firingTimersRemove(tn) - } - timerQueueLock.Unlock() } } @@ -131,9 +127,8 @@ func addTimer(tim *timerNode) { timerQueueLock.Unlock() } -// reAddTimer advances and re-adds a periodic timer (a ticker) after its -// callback has run, unless it was stopped or reset while the callback was -// running (in which case it must not be re-added). +// reAddTimer finishes firing a timer. It re-adds periodic timers unless they +// were stopped or reset while the callback was running. func reAddTimer(tn *timerNode) { timerQueueLock.Lock() @@ -149,6 +144,10 @@ func reAddTimer(tn *timerNode) { timerQueueLock.Unlock() return } + if tn.timer.period == 0 { + timerQueueLock.Unlock() + return + } tn.timer.when += tn.timer.period timerQueueAdd(tn) diff --git a/src/runtime/synctest.go b/src/runtime/synctest.go index fa11c991fc..37a950af7c 100644 --- a/src/runtime/synctest.go +++ b/src/runtime/synctest.go @@ -1,15 +1,573 @@ package runtime -// Dummy implementation of synctest functions (we don't support synctest at the -// moment). +import ( + "internal/task" + "unsafe" +) + +const synctestBaseTime = 946684800000000000 + +var synctestEnabled task.Uint32 + +func synctestIsEnabled() bool { + return synctestEnabled.Load() != 0 +} + +type synctestBubble struct { + lock synctestLock + + timers *timerNode + firingTimers *timerNode + + root *task.Task + main *task.Task + waiter *task.Task + + total int + running int + active int + + rootSleeping bool + waiting bool + done bool + now int64 + timerSeq uint32 +} + +func taskSynctestBubble(t *task.Task) *synctestBubble { + if t == nil || t.SynctestBubble == nil { + return nil + } + return (*synctestBubble)(t.SynctestBubble) +} + +func currentSynctestBubble() *synctestBubble { + if !synctestIsEnabled() { + return nil + } + return taskSynctestBubble(task.Current()) +} + +func (bubble *synctestBubble) wakeLocked() *task.Task { + if bubble.running != 0 || bubble.active != 0 { + return nil + } + bubble.active++ + if bubble.timers != nil && bubble.timers.timer.when <= bubble.now { + if bubble.rootSleeping { + bubble.rootSleeping = false + return bubble.root + } + bubble.active-- + return nil + } + if bubble.waiter != nil { + waiter := bubble.waiter + bubble.waiter = nil + return waiter + } + if bubble.rootSleeping { + bubble.rootSleeping = false + return bubble.root + } + bubble.active-- + return nil +} + +func (bubble *synctestBubble) addFiringTimerLocked(tn *timerNode) { + tn.stopped = false + tn.firingNext = bubble.firingTimers + bubble.firingTimers = tn +} + +func (bubble *synctestBubble) removeFiringTimerLocked(tn *timerNode) { + for queue := &bubble.firingTimers; *queue != nil; queue = &(*queue).firingNext { + if *queue == tn { + *queue = tn.firingNext + tn.firingNext = nil + return + } + } +} + +func (bubble *synctestBubble) stopFiringTimerLocked(tim *timer) { + for tn := bubble.firingTimers; tn != nil; tn = tn.firingNext { + if tn.timer == tim { + tn.stopped = true + return + } + } +} + +func (bubble *synctestBubble) time() int64 { + bubble.lock.Lock() + now := bubble.now + bubble.lock.Unlock() + return now +} + +func (bubble *synctestBubble) addTimer(tn *timerNode) { + if bubble.queueTimer(tn) { + tn.callback(tn, 0) + } +} + +func (bubble *synctestBubble) queueTimer(tn *timerNode) bool { + bubble.lock.Lock() + if tn.timer.when <= bubble.now { + bubble.addFiringTimerLocked(tn) + bubble.lock.Unlock() + return true + } + bubble.addTimerLocked(tn) + bubble.lock.Unlock() + return false +} + +func (bubble *synctestBubble) finishTimer(tn *timerNode) { + bubble.lock.Lock() + bubble.removeFiringTimerLocked(tn) + if tn.stopped { + bubble.lock.Unlock() + return + } + if tn.timer.period == 0 { + bubble.lock.Unlock() + return + } + next := tn.timer.when + tn.timer.period + if next < 0 { + next = 1<<63 - 1 + } + tn.timer.when = next + bubble.addTimerLocked(tn) + bubble.lock.Unlock() +} + +func (bubble *synctestBubble) addTimerLocked(tn *timerNode) { + bubble.timerSeq++ + insertBeforeEqual := (bubble.timerSeq/2)&1 != 0 + queue := &bubble.timers + for *queue != nil { + if (*queue).timer.when > tn.timer.when { + break + } + if insertBeforeEqual && (*queue).timer.when == tn.timer.when { + break + } + queue = &(*queue).next + } + tn.next = *queue + *queue = tn +} + +func (bubble *synctestBubble) removeTimer(tim *timer) *timerNode { + bubble.lock.Lock() + defer bubble.lock.Unlock() + for queue := &bubble.timers; *queue != nil; queue = &(*queue).next { + if (*queue).timer == tim { + node := *queue + *queue = node.next + node.next = nil + return node + } + } + bubble.stopFiringTimerLocked(tim) + return nil +} + +func (bubble *synctestBubble) checkTimerAccess(op string) { + if currentSynctestBubble() != bubble { + runtimeFatal(op + " of synctest timer from outside bubble") + } +} + +func synctestWakeTaskTimer(tn *timerNode, delta int64) { + tn.timer.synctest.finishTimer(tn) + scheduleTask(tn.timer.arg.(*task.Task)) +} + +func synctestSleep(duration int64) bool { + if !synctestIsEnabled() { + return false + } + current := task.Current() + bubble := taskSynctestBubble(current) + if bubble == nil { + return false + } + + bubble.lock.Lock() + bubble.active++ + when := bubble.now + duration + if when < 0 { + when = 1<<63 - 1 + } + bubble.lock.Unlock() + tim := &timer{ + when: when, + arg: current, + synctest: bubble, + } + node := &timerNode{ + timer: tim, + callback: synctestWakeTaskTimer, + } + runNow := bubble.queueTimer(node) + bubble.lock.Lock() + if runNow { + bubble.removeFiringTimerLocked(node) + bubble.active-- + if bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid sleep transition") + } + bubble.lock.Unlock() + return true + } + if !current.SynctestBlocked { + current.SynctestBlocked = true + bubble.running-- + } + bubble.active-- + if bubble.running < 0 || bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid sleep transition") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } + task.Pause() + return true +} + +func synctestTaskCreated(t *task.Task) { + if !synctestIsEnabled() { + return + } + bubble := taskSynctestBubble(t) + bubble.lock.Lock() + bubble.total++ + bubble.running++ + bubble.lock.Unlock() +} + +func synctestTaskExited(t *task.Task) { + if !synctestIsEnabled() { + return + } + bubble := taskSynctestBubble(t) + bubble.lock.Lock() + if t.SynctestBlocked { + t.SynctestBlocked = false + } else { + bubble.running-- + } + bubble.total-- + if t == bubble.main { + bubble.done = true + } + if bubble.running < 0 || bubble.total < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid task count") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +func synctestTaskWake(t *task.Task) { + if !synctestIsEnabled() { + return + } + bubble := taskSynctestBubble(t) + if bubble == nil { + return + } + bubble.lock.Lock() + if t.SynctestBlocked { + t.SynctestBlocked = false + bubble.running++ + } + bubble.lock.Unlock() +} + +func synctestTaskBlock(t *task.Task) { + if !synctestIsEnabled() { + return + } + bubble := taskSynctestBubble(t) + if bubble == nil { + return + } + bubble.lock.Lock() + if !t.SynctestBlocked { + t.SynctestBlocked = true + bubble.running-- + } + if bubble.running < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid running task count") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +func synctestTaskBlockBegin(t *task.Task) bool { + if !synctestIsEnabled() { + return false + } + bubble := taskSynctestBubble(t) + if bubble == nil { + return false + } + bubble.lock.Lock() + bubble.active++ + bubble.lock.Unlock() + return true +} + +func synctestTaskBlockEnd(t *task.Task, blocked bool) { + bubble := taskSynctestBubble(t) + if bubble == nil { + return + } + bubble.lock.Lock() + if blocked && !t.SynctestBlocked { + t.SynctestBlocked = true + bubble.running-- + } + bubble.active-- + if bubble.running < 0 || bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid block transition") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +func synctestTaskBlockCommit(t *task.Task, state *task.Uint32, old, new uint32) bool { + bubble := taskSynctestBubble(t) + if bubble == nil { + return false + } + bubble.lock.Lock() + blocked := state.CompareAndSwap(old, new) + if blocked && !t.SynctestBlocked { + t.SynctestBlocked = true + bubble.running-- + } + bubble.active-- + if bubble.running < 0 || bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid block transition") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } + return blocked +} + +func synctestBlockBegin(t *task.Task) bool { + return synctestTaskBlockBegin(t) +} + +func synctestBlockEnd(t *task.Task, blocked bool) { + synctestTaskBlockEnd(t, blocked) +} + +func synctestBlockCommit(t *task.Task, state *task.Uint32, old, new uint32) bool { + return synctestTaskBlockCommit(t, state, old, new) +} + +func synctestBlock(t *task.Task) { + synctestTaskBlock(t) +} + +//go:linkname synctest_run internal/synctest.Run +func synctest_run(f func()) { + synctestEnabled.Store(1) + root := task.Current() + if root.SynctestBubble != nil { + panic("synctest.Run called from within a synctest bubble") + } + + bubble := &synctestBubble{ + root: root, + now: synctestBaseTime, + } + + // Let the new goroutine inherit the bubble from the root task. + root.SynctestBubble = unsafe.Pointer(bubble) + go func() { + bubble.lock.Lock() + bubble.main = task.Current() + bubble.lock.Unlock() + f() + }() + root.SynctestBubble = nil + + for { + bubble.lock.Lock() + if bubble.rootSleeping { + bubble.lock.Unlock() + runtimeFatal("synctest: root resumed while marked sleeping") + } + if bubble.total == 0 && bubble.active == 0 { + bubble.lock.Unlock() + return + } + if bubble.running == 0 && bubble.active == 0 { + dueTimer := bubble.timers != nil && bubble.timers.timer.when <= bubble.now + if bubble.timers != nil && !bubble.done && (dueTimer || bubble.waiter == nil) { + timer := bubble.timers + bubble.timers = timer.next + timer.next = nil + bubble.addFiringTimerLocked(timer) + bubble.active++ + if timer.timer.when > bubble.now { + bubble.now = timer.timer.when + } + bubble.lock.Unlock() + + // Timer callbacks run on the root task inside the bubble. + root.SynctestBubble = unsafe.Pointer(bubble) + timer.callback(timer, 0) + root.SynctestBubble = nil + bubble.lock.Lock() + bubble.active-- + if bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid active count") + } + bubble.lock.Unlock() + continue + } + if bubble.waiter != nil { + waiter := bubble.waiter + bubble.waiter = nil + bubble.active++ + bubble.lock.Unlock() + scheduleTask(waiter) + continue + } + done := bubble.done + bubble.lock.Unlock() + if done { + panic("deadlock: main bubble goroutine has exited but blocked goroutines remain") + } + panic("deadlock: all goroutines in bubble are blocked") + } + bubble.rootSleeping = true + bubble.lock.Unlock() + task.Pause() + bubble.lock.Lock() + bubble.active-- + if bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid active count") + } + bubble.lock.Unlock() + } +} + +//go:linkname synctest_wait internal/synctest.Wait +func synctest_wait() { + current := task.Current() + bubble := taskSynctestBubble(current) + if bubble == nil { + panic("goroutine is not in a bubble") + } + + bubble.lock.Lock() + if bubble.waiting { + bubble.lock.Unlock() + panic("wait already in progress") + } + bubble.waiting = true + current.SynctestBlocked = true + bubble.running-- + dueTimer := bubble.timers != nil && bubble.timers.timer.when <= bubble.now + if bubble.running == 0 && bubble.active == 0 && !dueTimer { + current.SynctestBlocked = false + bubble.running++ + bubble.waiting = false + bubble.lock.Unlock() + return + } + bubble.waiter = current + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } + + task.Pause() + + bubble.lock.Lock() + bubble.active-- + if bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid active count") + } + bubble.waiting = false + bubble.lock.Unlock() +} + +//go:linkname synctest_isInBubble internal/synctest.IsInBubble +func synctest_isInBubble() bool { + return currentSynctestBubble() != nil +} //go:linkname synctest_acquire internal/synctest.acquire func synctest_acquire() any { - // Dummy: we don't support synctest. - return nil + bubble := currentSynctestBubble() + if bubble == nil { + return nil + } + bubble.lock.Lock() + bubble.active++ + bubble.lock.Unlock() + return bubble } //go:linkname synctest_release internal/synctest.release -func synctest_release(sg any) { - // Dummy: we don't support synctest. +func synctest_release(value any) { + bubble := value.(*synctestBubble) + bubble.lock.Lock() + bubble.active-- + if bubble.active < 0 { + bubble.lock.Unlock() + runtimeFatal("synctest: invalid active count") + } + wake := bubble.wakeLocked() + bubble.lock.Unlock() + if wake != nil { + scheduleTask(wake) + } +} + +//go:linkname synctest_inBubble internal/synctest.inBubble +func synctest_inBubble(value any, f func()) { + current := task.Current() + if current.SynctestBubble != nil { + panic("goroutine is already bubbled") + } + current.SynctestBubble = unsafe.Pointer(value.(*synctestBubble)) + defer func() { + current.SynctestBubble = nil + }() + f() } diff --git a/src/runtime/time.go b/src/runtime/time.go index e0a0346a50..cfd1db3a09 100644 --- a/src/runtime/time.go +++ b/src/runtime/time.go @@ -1,9 +1,13 @@ package runtime -import "unsafe" +import ( + "unsafe" +) // This is the timer that's used internally inside the runtime. type timer struct { + lock timerLock + // When to call the timer, and the interval for the ticker. when int64 period int64 @@ -11,6 +15,8 @@ type timer struct { // Callback from the time package. f func(arg any, seq uintptr, delta int64) arg any + + synctest *synctestBubble } func (tim *timer) callCallback(delta int64) { @@ -21,61 +27,115 @@ func (tim *timer) callCallback(delta int64) { // the same as time.Timer and time.Ticker so it can be used as-is in the time // package. type timeTimer struct { - c unsafe.Pointer // <-chan time.Time - init bool + c unsafe.Pointer // <-chan time.Time + initTimer bool timer } //go:linkname newTimer time.newTimer func newTimer(when, period int64, f func(arg any, seq uintptr, delta int64), arg any, c unsafe.Pointer) *timeTimer { + bubble := currentSynctestBubble() tim := &timeTimer{ - c: c, - init: true, + c: c, + initTimer: true, timer: timer{ - when: when, - period: period, - f: f, - arg: arg, + when: when, + period: period, + f: f, + arg: arg, + synctest: bubble, }, } scheduleLog("new timer") - addTimer(&timerNode{ + node := &timerNode{ timer: &tim.timer, callback: timerCallback, - }) + } + if bubble != nil { + bubble.addTimer(node) + } else { + addTimer(node) + } return tim } //go:linkname stopTimer time.stopTimer func stopTimer(tim *timeTimer) bool { - return removeTimer(&tim.timer) != nil + if tim.timer.synctest != nil { + tim.timer.synctest.checkTimerAccess("stop") + } + tim.timer.lock.Lock() + var removed bool + if tim.timer.synctest != nil { + removed = tim.timer.synctest.removeTimer(&tim.timer) != nil + } else { + removed = removeTimer(&tim.timer) != nil + } + tim.timer.lock.Unlock() + return removed } //go:linkname resetTimer time.resetTimer func resetTimer(t *timeTimer, when, period int64) bool { - n := removeTimer(&t.timer) + if t.timer.synctest != nil { + t.timer.synctest.checkTimerAccess("reset") + } + t.timer.lock.Lock() + var n *timerNode + if t.timer.synctest != nil { + n = t.timer.synctest.removeTimer(&t.timer) + } else { + n = removeTimer(&t.timer) + } removed := n != nil if n == nil { - n = new(timerNode) + // Allocation can start GC, so do not hold the cores spin lock. + t.timer.lock.Unlock() + replacement := new(timerNode) + t.timer.lock.Lock() + // A concurrent reset can queue the timer during allocation. + // Remove it again so this reset takes effect after that operation. + if t.timer.synctest != nil { + n = t.timer.synctest.removeTimer(&t.timer) + } else { + n = removeTimer(&t.timer) + } + removed = n != nil + if n == nil { + n = replacement + } } t.timer.when = when t.timer.period = period n.timer = &t.timer n.callback = timerCallback - addTimer(n) + var runNow bool + if t.timer.synctest != nil { + runNow = t.timer.synctest.queueTimer(n) + } else { + addTimer(n) + } + t.timer.lock.Unlock() + if runNow { + n.callback(n, 0) + } return removed } //go:linkname time_runtimeNano time.runtimeNano func time_runtimeNano() int64 { - // Note: we're ignoring sync groups here (package testing/synctest). - // See: https://github.com/golang/go/issues/67434 + if bubble := currentSynctestBubble(); bubble != nil { + return bubble.time() + } return nanotime() } //go:linkname time_runtimeNow time.runtimeNow func time_runtimeNow() (sec int64, nsec int32, mono int64) { - // Also ignoring the sync group here, like time_runtimeNano above. + if bubble := currentSynctestBubble(); bubble != nil { + now := bubble.time() + return now / 1e9, int32(now % 1e9), 0 + } return now() } @@ -90,7 +150,7 @@ type timerNode struct { // schedulers). They make it possible to stop or reset a periodic timer (a // ticker) while its callback is running, without the callback re-adding the // timer to the queue afterwards. They are protected by the scheduler's - // timer lock. + // timer lock for normal timers and the bubble lock for synctest timers. // // firingNext links nodes whose callback is currently running into the // firingTimers list. stopped is set when the timer was stopped or reset @@ -119,14 +179,17 @@ func timerCallback(tn *timerNode, delta int64) { // package so is left zero. tn.timer.callCallback(delta) - // If this is a periodic timer (a ticker), re-add it to the queue. - if tn.timer.period != 0 { + // Finish firing the timer and re-add it if it is periodic. + tn.timer.lock.Lock() + if tn.timer.synctest != nil { + tn.timer.synctest.finishTimer(tn) + } else { reAddTimer(tn) } + tn.timer.lock.Unlock() } //go:linkname time_runtimeIsBubbled time.runtimeIsBubbled func time_runtimeIsBubbled() bool { - // We don't currently support bubbles. - return false + return currentSynctestBubble() != nil } diff --git a/src/sync/cond.go b/src/sync/cond.go index 139d8e0229..611925951e 100644 --- a/src/sync/cond.go +++ b/src/sync/cond.go @@ -5,17 +5,7 @@ import ( "unsafe" ) -// Condition variable. -// A goroutine that called Wait() can be in one of a few states depending on the -// Task.Data field: -// - When entering Wait, and before going to sleep, the data field is 0. -// - When the goroutine that calls Wait changes its data value from 0 to 1, it -// is going to sleep. It has not been awoken early. -// - When instead a call to Signal or Broadcast can change the data field from 0 -// to 1, it will _not_ go to sleep but be signalled early. -// This can happen when a concurrent call to Signal happens, or the Unlock -// function calls Signal for some reason. - +// Cond is a condition variable. type Cond struct { L Locker @@ -23,6 +13,15 @@ type Cond struct { lock task.PMutex } +// A waiting task stores one of these states in Task.Data. +// Signal schedules only a task that has reached condBlocked. +const ( + condWaiting = iota + condCommitting + condSignaled + condBlocked +) + func NewCond(l Locker) *Cond { return &Cond{L: l} } @@ -31,12 +30,12 @@ func (c *Cond) trySignal() bool { // Pop a blocked task off of the stack, and schedule it if applicable. t := c.blocked.Pop() if t != nil { + if t.SynctestBubble != nil && task.Current().SynctestBubble != t.SynctestBubble { + runtimeFatal("semaphore wake of synctest goroutine from outside bubble") + } dataPtr := (*task.Uint32)(unsafe.Pointer(&t.Data)) - // The data value is 0 when the task is not yet sleeping, and 1 when it is. - if dataPtr.Swap(1) != 0 { - // The value was already 1, so the task went to sleep (or is about to go - // to sleep). Schedule the task to be resumed. + if dataPtr.Swap(condSignaled) == condBlocked { scheduleTask(t) } return true @@ -64,31 +63,47 @@ func (c *Cond) Wait() { // Mark us as not yet signalled or sleeping. t := task.Current() dataPtr := (*task.Uint32)(unsafe.Pointer(&t.Data)) - dataPtr.Store(0) + dataPtr.Store(condWaiting) // Add us to the list of waiting goroutines. c.lock.Lock() c.blocked.Push(t) c.lock.Unlock() + transition := synctestBlockBegin(t) + // Temporarily unlock L. c.L.Unlock() // Re-acquire the lock before returning. defer c.L.Lock() - // If we were signaled while unlocking, immediately complete. - if dataPtr.Swap(1) != 0 { - // The data value was already 1, so we got a signal already (and weren't - // scheduled because trySignal was the first to change the value). + // Commit to blocking unless a signal arrived while unlocking. + if !dataPtr.CompareAndSwap(condWaiting, condCommitting) { + if transition { + synctestBlockEnd(t, false) + } return } - // We were the first to change the value from 0 to 1, meaning we did not get - // a signal during the call to Unlock(). So we wait until we do get a - // signal. + if transition { + if !synctestBlockCommit(t, dataPtr, condCommitting, condBlocked) { + return + } + } else if !dataPtr.CompareAndSwap(condCommitting, condBlocked) { + return + } task.Pause() } //go:linkname scheduleTask runtime.scheduleTask func scheduleTask(*task.Task) + +//go:linkname synctestBlockBegin runtime.synctestBlockBegin +func synctestBlockBegin(*task.Task) bool + +//go:linkname synctestBlockEnd runtime.synctestBlockEnd +func synctestBlockEnd(*task.Task, bool) + +//go:linkname synctestBlockCommit runtime.synctestBlockCommit +func synctestBlockCommit(*task.Task, *task.Uint32, uint32, uint32) bool diff --git a/src/sync/waitgroup.go b/src/sync/waitgroup.go index 9a564c5f7b..f0b27cfb08 100644 --- a/src/sync/waitgroup.go +++ b/src/sync/waitgroup.go @@ -1,19 +1,90 @@ package sync -import "internal/task" +import ( + "internal/task" + "unsafe" +) + +const waitGroupBubbleFlag = uint32(1 << 31) type WaitGroup struct { - futex task.Futex + futex task.Futex + lock task.PMutex + waiters task.Stack + counter int + waiting int + synctest unsafe.Pointer } func (wg *WaitGroup) Add(delta int) { - switch { - case delta > 0: + if !synctestIsEnabled() { + wg.addPlain(delta) + return + } + + currentBubble := task.Current().SynctestBubble + wg.lock.Lock() + if currentBubble == nil && wg.synctest == nil { + wg.lock.Unlock() + wg.addPlain(delta) + return + } + if currentBubble != nil { + if wg.synctest == nil { + if !wg.futex.CompareAndSwap(0, waitGroupBubbleFlag) { + wg.lock.Unlock() + runtimeFatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } + wg.synctest = currentBubble + } else if wg.synctest != currentBubble { + wg.lock.Unlock() + runtimeFatal("sync: WaitGroup.Add called from multiple synctest bubbles") + } + } else if wg.synctest != nil { + wg.lock.Unlock() + runtimeFatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } + + if delta > 0 && wg.counter == 0 && wg.waiting != 0 { + wg.lock.Unlock() + panic("sync: WaitGroup misuse: Add called concurrently with Wait") + } + if delta > 0 && wg.counter > int(^uint32(0)>>1)-delta { + wg.lock.Unlock() + panic("sync: WaitGroup counter overflowed") + } + wg.counter += delta + if wg.counter < 0 { + wg.lock.Unlock() + panic("sync: negative WaitGroup counter") + } + if wg.counter != 0 { + wg.lock.Unlock() + return + } + + waiters := wg.waiters.Queue() + if wg.waiting == 0 { + wg.synctest = nil + wg.futex.Store(0) + } + wg.lock.Unlock() + + for waiter := waiters.Pop(); waiter != nil; waiter = waiters.Pop() { + scheduleTask(waiter) + } +} + +func (wg *WaitGroup) addPlain(delta int) { + if delta > 0 { // Delta is positive. for { // Check for overflow. counter := wg.futex.Load() - if uint32(delta) > (^uint32(0))-counter { + if synctestIsEnabled() && counter&waitGroupBubbleFlag != 0 { + runtimeFatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } + if uint64(delta) > uint64(waitGroupBubbleFlag-1-counter) { panic("sync: WaitGroup counter overflowed") } @@ -23,46 +94,49 @@ func (wg *WaitGroup) Add(delta int) { return } } - default: - // Delta is negative (or zero). - for { - counter := wg.futex.Load() + } - // Check for underflow. - if uint32(-delta) > counter { - panic("sync: negative WaitGroup counter") - } + // Delta is negative (or zero). + for { + counter := wg.futex.Load() + if synctestIsEnabled() && counter&waitGroupBubbleFlag != 0 { + runtimeFatal("sync: WaitGroup.Add called from inside and outside synctest bubble") + } - // Subtract from the counter. - if !wg.futex.CompareAndSwap(counter, counter-uint32(-delta)) { - // Could not swap, trying again. - continue - } + // Check for underflow. + if uint32(-delta) > counter { + panic("sync: negative WaitGroup counter") + } - // If the counter is zero, everything is done and the waiters should - // be resumed. - // When there are multiple thread, there is a chance for the counter - // to go to zero, WakeAll to be called, and then the counter to be - // incremented again before a waiting goroutine has a chance to - // check the new (zero) value. However the last increment is - // explicitly given in the docs as something that should not be - // done: - // - // > Note that calls with a positive delta that occur when the - // > counter is zero must happen before a Wait. - // - // So we're fine here. - if counter-uint32(-delta) == 0 { - // TODO: this is not the most efficient implementation possible - // because we wake up all waiters unconditionally, even if there - // might be none. Though since the common usage is for this to - // be called with at least one waiter, it's probably fine. - wg.futex.WakeAll() - } + // Subtract from the counter. + if !wg.futex.CompareAndSwap(counter, counter-uint32(-delta)) { + // Could not swap, trying again. + continue + } - // Successfully swapped (and woken all waiting tasks if needed). - return + // If the counter is zero, everything is done and the waiters should + // be resumed. + // When there are multiple thread, there is a chance for the counter + // to go to zero, WakeAll to be called, and then the counter to be + // incremented again before a waiting goroutine has a chance to + // check the new (zero) value. However the last increment is + // explicitly given in the docs as something that should not be + // done: + // + // > Note that calls with a positive delta that occur when the + // > counter is zero must happen before a Wait. + // + // So we're fine here. + if counter-uint32(-delta) == 0 { + // TODO: this is not the most efficient implementation possible + // because we wake up all waiters unconditionally, even if there + // might be none. Though since the common usage is for this to + // be called with at least one waiter, it's probably fine. + wg.futex.WakeAll() } + + // Successfully swapped (and woken all waiting tasks if needed). + return } } @@ -71,15 +145,58 @@ func (wg *WaitGroup) Done() { } func (wg *WaitGroup) Wait() { + if !synctestIsEnabled() { + wg.waitPlain() + return + } + + wg.lock.Lock() + if wg.synctest == nil { + wg.lock.Unlock() + wg.waitPlain() + return + } + current := task.Current() + if wg.counter == 0 { + if wg.waiting == 0 { + wg.synctest = nil + wg.futex.Store(0) + } + wg.lock.Unlock() + return + } + wg.waiting++ + wg.waiters.Push(current) + // An unbubbled waiter is allowed, but its wait is not durable. + if wg.synctest != nil && wg.synctest == current.SynctestBubble { + synctestBlock(current) + } + wg.lock.Unlock() + + task.Pause() + + wg.lock.Lock() + wg.waiting-- + if wg.counter != 0 { + wg.lock.Unlock() + panic("sync: WaitGroup is reused before previous Wait has returned") + } + if wg.waiting == 0 { + wg.synctest = nil + wg.futex.Store(0) + } + wg.lock.Unlock() +} + +func (wg *WaitGroup) waitPlain() { for { counter := wg.futex.Load() if counter == 0 { return // everything already finished } - if wg.futex.Wait(counter) { // Successfully woken by WakeAll (in wg.Add). - break + return } } } @@ -91,3 +208,9 @@ func (wg *WaitGroup) Go(f func()) { f() }() } + +//go:linkname synctestBlock runtime.synctestBlock +func synctestBlock(*task.Task) + +//go:linkname synctestIsEnabled runtime.synctestIsEnabled +func synctestIsEnabled() bool diff --git a/src/testing/sub_test.go b/src/testing/sub_test.go index 7ac1ea47d4..afe26c98ab 100644 --- a/src/testing/sub_test.go +++ b/src/testing/sub_test.go @@ -5,9 +5,135 @@ package testing import ( + "internal/synctest" "reflect" + "time" ) +func TestSynctestDuringCleanup(t *T) { + parent := &T{} + parent.cleanupStarted.Store(true) + defer func() { + const want = "testing: synctest.Run called during t.Cleanup" + if got := recover(); got != want { + t.Errorf("panic = %v, want %q", got, want) + } + }() + testingSynctestTest(parent, func(*T) {}) +} + +func TestSynctestAcquireDelaysRun(t *T) { + acquired := make(chan *synctest.Bubble) + release := make(chan struct{}) + runDone := make(chan struct{}) + + go func() { + bubble := <-acquired + <-release + bubble.Release() + }() + + go func() { + synctest.Run(func() { + acquired <- synctest.Acquire() + }) + close(runDone) + }() + + select { + case <-runDone: + t.Fatal("synctest.Run returned before the bubble reference was released") + case <-time.After(time.Millisecond): + } + close(release) + <-runDone +} + +func TestSynctestSleepOverflow(t *T) { + synctest.Run(func() { + start := time.Now() + time.Sleep(time.Duration(1<<63 - 1)) + if elapsed := time.Since(start); elapsed == 0 { + t.Fatal("maximum-duration sleep returned without advancing fake time") + } + time.Sleep(time.Nanosecond) + synctest.Wait() + }) +} + +func TestSynctestTickerConcurrentStopAndReset(t *T) { + for range 100 { + synctest.Run(func() { + ticker := time.NewTicker(time.Nanosecond) + stopped := make(chan struct{}) + go func() { + <-ticker.C + ticker.Stop() + close(stopped) + }() + <-stopped + time.Sleep(time.Nanosecond) + select { + case <-ticker.C: + t.Fatal("stopped ticker fired again") + default: + } + }) + + synctest.Run(func() { + ticker := time.NewTicker(time.Nanosecond) + reset := make(chan struct{}) + go func() { + <-ticker.C + ticker.Reset(10 * time.Nanosecond) + close(reset) + }() + <-reset + start := time.Now() + <-ticker.C + if elapsed := time.Since(start); elapsed != 10*time.Nanosecond { + t.Fatalf("reset ticker fired after %v, want 10ns", elapsed) + } + ticker.Stop() + }) + } +} + +func TestSynctestTimerImmediateReset(t *T) { + synctest.Run(func() { + timer := time.NewTimer(time.Hour) + timer.Reset(0) + <-timer.C + }) +} + +func TestSynctestTickerConcurrentResets(t *T) { + for range 100 { + synctest.Run(func() { + ticker := time.NewTicker(time.Hour) + start := make(chan struct{}) + done := make(chan struct{}, 2) + for _, duration := range []time.Duration{10, 20} { + go func() { + <-start + ticker.Reset(duration) + done <- struct{}{} + }() + } + close(start) + <-done + <-done + ticker.Stop() + time.Sleep(100 * time.Nanosecond) + select { + case <-ticker.C: + t.Fatal("stopped ticker fired after concurrent resets") + default: + } + }) + } +} + func TestCleanup(t *T) { var cleanups []int t.Run("test", func(t *T) { diff --git a/src/testing/testing.go b/src/testing/testing.go index 60d8fab645..67b48f91a0 100644 --- a/src/testing/testing.go +++ b/src/testing/testing.go @@ -22,6 +22,7 @@ import ( "runtime" "strconv" "strings" + "sync/atomic" "time" "unicode" "unicode/utf8" @@ -77,6 +78,8 @@ type common struct { start time.Time // Time test or benchmark started duration time.Duration + cleanupStarted atomic.Bool + tempDir string tempDirErr error tempDirSeq int32 @@ -184,7 +187,8 @@ var _ TB = (*B)(nil) // Logs are accumulated during execution and dumped to standard output when done. type T struct { common - context *testContext // For running tests and subtests. + context *testContext // For running tests and subtests. + isSynctest bool } // Name returns the name of the running test or benchmark. @@ -460,6 +464,11 @@ func (c *common) Chdir(dir string) { // runCleanup is called at the end of the test. func (c *common) runCleanup() { + c.cleanupStarted.Store(true) + if c.cancelCtx != nil { + c.cancelCtx() + c.cancelCtx = nil + } for { var cleanup func() if len(c.cleanups) > 0 { @@ -470,15 +479,15 @@ func (c *common) runCleanup() { if cleanup == nil { return } - if c.cancelCtx != nil { - c.cancelCtx() - } cleanup() } } // Parallel is not implemented, it is only provided for compatibility. func (t *T) Parallel() { + if t.isSynctest { + panic("testing: t.Parallel called inside synctest bubble") + } // Unimplemented. } @@ -504,9 +513,41 @@ func tRunner(t *T, fn func(t *T)) { t.finished = true } +//go:linkname testingSynctestTest testing/synctest.testingSynctestTest +func testingSynctestTest(t *T, f func(*T)) bool { + if t.cleanupStarted.Load() { + panic("testing: synctest.Run called during t.Cleanup") + } + + ctx, cancelCtx := context.WithCancel(context.Background()) + synctestT := T{ + common: common{ + output: &logger{logToStdout: flagVerbose}, + name: t.name, + parent: &t.common, + level: t.level + 1, + ctx: ctx, + cancelCtx: cancelCtx, + }, + context: t.context, + isSynctest: true, + } + + done := make(chan struct{}) + go func() { + defer close(done) + tRunner(&synctestT, f) + }() + <-done + return !synctestT.failed +} + // Run runs f as a subtest of t called name. It waits until the subtest is finished // and returns whether the subtest succeeded. func (t *T) Run(name string, f func(t *T)) bool { + if t.isSynctest { + panic("testing: t.Run called inside synctest bubble") + } t.hasSub = true testName, ok, _ := t.context.match.fullName(&t.common, name) if !ok { @@ -550,6 +591,9 @@ func (t *T) Run(name string, f func(t *T)) bool { // // Not Implemented. func (t *T) Deadline() (deadline time.Time, ok bool) { + if t.isSynctest { + panic("testing: t.Deadline called inside synctest bubble") + } deadline = t.context.deadline return deadline, !deadline.IsZero() }