TLA Line data Source code
1 : //
2 : // Copyright (c) 2026 Steve Gerbino
3 : //
4 : // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 : // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 : //
7 : // Official repository: https://github.com/cppalliance/corosio
8 : //
9 :
10 : #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
11 : #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
12 :
13 : #include <boost/corosio/detail/config.hpp>
14 : #include <boost/corosio/detail/object_ref.hpp>
15 : #include <boost/capy/ex/execution_context.hpp>
16 : #include <boost/capy/ex/abandon.hpp>
17 :
18 : #include <boost/corosio/detail/ready_queue.hpp>
19 : #include <boost/corosio/detail/scheduler.hpp>
20 : #include <boost/corosio/detail/scheduler_op.hpp>
21 : #include <boost/corosio/detail/thread_local_ptr.hpp>
22 : #include <boost/corosio/detail/timer_service.hpp>
23 : #include <boost/corosio/native/detail/reactor/reactor_defer_policy.hpp>
24 :
25 : #include <atomic>
26 : #include <chrono>
27 : #include <coroutine>
28 : #include <cstddef>
29 : #include <cstdint>
30 : #include <limits>
31 : #include <memory>
32 :
33 : #include <boost/corosio/detail/conditionally_enabled_mutex.hpp>
34 : #include <boost/corosio/detail/conditionally_enabled_event.hpp>
35 :
36 : namespace boost::corosio::detail {
37 :
38 : // Forward declarations
39 : class reactor_scheduler;
40 :
41 : /** A deregistered descriptor whose owner must outlive the reactor pass
42 : that may still hold it.
43 :
44 : An event the reactor has already read from the kernel can name a
45 : descriptor that another thread closes before the reactor queues it.
46 : The closing thread hands over a reference instead of dropping it, so
47 : the owner is neither freed nor recycled until that pass ends.
48 : */
49 : struct reactor_retirable
50 : {
51 : // All four fields are guarded by the scheduler's mutex.
52 :
53 : /// Next in the scheduler's retired list.
54 : reactor_retirable* retired_next_ = nullptr;
55 :
56 : /// Owner reference held until the pass ends.
57 : object_ref retired_ref_;
58 :
59 : /// Next in a settling thread's list of references to drop.
60 : reactor_retirable* dropped_next_ = nullptr;
61 :
62 : /// Owner reference waiting to be dropped outside the mutex.
63 : object_ref dropped_ref_;
64 :
65 : /// Move @p keep into a queued completion, if one is pending.
66 : virtual void hand_off(object_ref& keep) noexcept = 0;
67 :
68 : protected:
69 HIT 12645 : ~reactor_retirable() = default;
70 : };
71 :
72 : /** Per-thread state for a reactor scheduler.
73 :
74 : Each thread running a scheduler's event loop has one of these
75 : on a thread-local stack. It holds a private work queue and
76 : inline completion budget for speculative I/O fast paths.
77 : */
78 : struct BOOST_COROSIO_SYMBOL_VISIBLE reactor_scheduler_context
79 : {
80 : /// Scheduler this context belongs to.
81 : reactor_scheduler const* key;
82 :
83 : /// Next context frame on this thread's stack.
84 : reactor_scheduler_context* next;
85 :
86 : /// Private work queue for reduced contention.
87 : ready_queue private_queue;
88 :
89 : /// Unflushed work count for the private queue.
90 : std::int64_t private_outstanding_work;
91 :
92 : /// Remaining inline completions allowed this cycle.
93 : int inline_budget;
94 :
95 : /// Maximum inline budget (adaptive, 2-16).
96 : int inline_budget_max;
97 :
98 : /// True if no other thread absorbed queued work last cycle.
99 : bool unassisted;
100 :
101 : /// Construct a context frame linked to @a n.
102 : reactor_scheduler_context(
103 : reactor_scheduler const* k, reactor_scheduler_context* n);
104 : };
105 :
106 : /// Thread-local context stack for reactor schedulers.
107 : inline thread_local_ptr<reactor_scheduler_context> reactor_context_stack;
108 :
109 : /// Find the context frame for a scheduler on this thread.
110 : inline reactor_scheduler_context*
111 950944 : reactor_find_context(reactor_scheduler const* self) noexcept
112 : {
113 951478 : for (auto* c = reactor_context_stack.get(); c != nullptr; c = c->next)
114 : {
115 888484 : if (c->key == self)
116 887950 : return c;
117 : }
118 62994 : return nullptr;
119 : }
120 :
121 : /** Non-template base for reactor-backed scheduler implementations.
122 :
123 : Provides the complete threading model shared by epoll, kqueue,
124 : and select schedulers: signal state machine, inline completion
125 : budget, work counting, run/poll methods, and the do_one event
126 : loop.
127 :
128 : Derived classes provide platform-specific hooks by overriding:
129 : - `run_task(lock, ctx)` to run the reactor poll
130 : - `interrupt_reactor()` to wake a blocked reactor
131 :
132 : De-templated from the original CRTP design to eliminate
133 : duplicate instantiations when multiple backends are compiled
134 : into the same binary. Virtual dispatch for run_task (called
135 : once per reactor cycle, before a blocking syscall) has
136 : negligible overhead.
137 :
138 : @par Thread Safety
139 : All public member functions are thread-safe.
140 : */
141 : class reactor_scheduler : public scheduler
142 : {
143 : public:
144 : using context_type = reactor_scheduler_context;
145 : using mutex_type = conditionally_enabled_mutex;
146 : using lock_type = mutex_type::scoped_lock;
147 : using event_type = conditionally_enabled_event;
148 :
149 : /// Post a coroutine for deferred execution.
150 : void post(std::coroutine_handle<> h) const override;
151 :
152 : /// Post a scheduler operation for deferred execution.
153 : void post(scheduler_op* h) const override;
154 :
155 : /** Release an owner reference once no reactor pass can hold `d`.
156 :
157 : Called after a descriptor is deregistered. If a reactor pass is
158 : running, `keep` is parked on `d` until that pass ends. Then, or
159 : at once if no pass is running, it moves to `d`'s queued
160 : invocation if `d` is queued, and is dropped otherwise.
161 :
162 : @param d The deregistered descriptor.
163 : @param keep A reference to the descriptor's owner.
164 : */
165 : void retire_descriptor(reactor_retirable& d, object_ref keep) const;
166 :
167 : /// Post a continuation for deferred execution.
168 : void post(capy::continuation&) const override;
169 :
170 : /// Return true if called from a thread running this scheduler.
171 : bool running_in_this_thread() const noexcept override;
172 :
173 : /// Request the scheduler to stop dispatching handlers.
174 : void stop() override;
175 :
176 : /// Return true if the scheduler has been stopped.
177 : bool stopped() const noexcept override;
178 :
179 : /// Reset the stopped state so `run()` can resume.
180 : void restart() override;
181 :
182 : /// Run the event loop until no work remains.
183 : std::size_t run() override;
184 :
185 : /// Run until one handler completes or no work remains.
186 : std::size_t run_one() override;
187 :
188 : /// Run until one handler completes or @a usec elapses.
189 : std::size_t wait_one(long usec) override;
190 :
191 : /// Run ready handlers without blocking.
192 : std::size_t poll() override;
193 :
194 : /// Run at most one ready handler without blocking.
195 : std::size_t poll_one() override;
196 :
197 : /// Increment the outstanding work count.
198 : void work_started() noexcept override;
199 :
200 : /// Decrement the outstanding work count, stopping on zero.
201 : void work_finished() noexcept override;
202 :
203 : /** Reset the thread's inline completion budget.
204 :
205 : Called at the start of each posted completion handler to
206 : grant a fresh budget for speculative inline completions.
207 : */
208 : void reset_inline_budget() const noexcept;
209 :
210 : /** Consume one unit of inline budget if available.
211 :
212 : @return True if budget was available and consumed.
213 : */
214 : bool try_consume_inline_budget() const noexcept;
215 :
216 : /** Offset a forthcoming work_finished from work_cleanup.
217 :
218 : Called by descriptor_state when all I/O returned EAGAIN and
219 : no handler will be executed. Must be called from a scheduler
220 : thread.
221 : */
222 : /** Note that an op was parked outside the usual registration path.
223 :
224 : A reactor that builds its poll set from parked ops must rebuild
225 : it; one the kernel tracks needs nothing.
226 : */
227 411 : virtual void notify_reparked() const {}
228 :
229 : void compensating_work_started() const noexcept;
230 :
231 : /** Post an operation whose work is already counted.
232 :
233 : The posting thread touches nothing of the scheduler after this
234 : returns, unlike a @ref post followed by @ref work_finished: once
235 : posted, the operation can complete and let its context be
236 : destroyed.
237 :
238 : @pre work_started() was called for @p op.
239 :
240 : @param op The operation to post.
241 : */
242 : void post_counted(scheduler_op* op) const;
243 :
244 : /** Post completed operations for deferred invocation.
245 :
246 : If called from a thread running this scheduler, operations
247 : go to the thread's private queue (fast path). Otherwise,
248 : operations are added to the global queue under mutex and a
249 : waiter is signaled.
250 :
251 : @pre work_started() must have been called for each operation.
252 :
253 : @param ops Queue of operations to post.
254 : */
255 : void post_deferred_completions(ready_queue& ops) const;
256 :
257 : /** Apply runtime configuration to the scheduler.
258 :
259 : Called by `io_context` after construction. Values that do
260 : not apply to this backend are silently ignored.
261 :
262 : @pre `max_events` is in [ 1, `INT_MAX` ] and `budget_max` is at
263 : most `INT_MAX`; `io_context` validates both.
264 :
265 : @param max_events Event buffer size for epoll/kqueue.
266 : @param budget_init Starting inline completion budget.
267 : @param budget_max Hard ceiling on adaptive budget ramp-up.
268 : @param unassisted Budget when single-threaded.
269 : */
270 : virtual void configure_reactor(
271 : unsigned max_events,
272 : unsigned budget_init,
273 : unsigned budget_max,
274 : unsigned unassisted);
275 :
276 : /// Return the configured initial inline budget.
277 10907 : unsigned inline_budget_initial() const noexcept
278 : {
279 10907 : return inline_budget_initial_;
280 : }
281 :
282 : /// Return true when scheduler locking is disabled (fully-lockless tier).
283 1460 : bool scheduler_locking_disabled() const noexcept override
284 : {
285 1460 : return scheduler_locking_disabled_;
286 : }
287 :
288 4680 : void configure_threading(threading_config cfg) noexcept override
289 : {
290 4680 : scheduler_locking_disabled_ = !cfg.scheduler_locking;
291 : // reactor_io_locking takes effect at descriptor registration (see the
292 : // register_descriptor overrides), not here.
293 4680 : reactor_io_locking_ = cfg.reactor_io_locking;
294 4680 : one_thread_ = cfg.one_thread;
295 4680 : mutex_.set_enabled(cfg.scheduler_locking);
296 4680 : cond_.set_enabled(cfg.scheduler_locking);
297 4680 : }
298 :
299 : protected:
300 : bool scheduler_locking_disabled_ = false;
301 : bool reactor_io_locking_ = true;
302 : bool one_thread_ = false;
303 :
304 4696 : reactor_scheduler() = default;
305 :
306 : /// Destroy, destroying whatever was posted after the drain.
307 : ~reactor_scheduler() override;
308 :
309 : /** Drain completed_ops during shutdown.
310 :
311 : Pops all operations from the global queue and destroys them,
312 : skipping the task sentinel. Signals all waiting threads.
313 : Derived classes call this from their shutdown() override
314 : before performing platform-specific cleanup.
315 : */
316 : void shutdown_drain();
317 :
318 : /// Destroy every queued operation without running it.
319 : void destroy_queued(lock_type& lock) noexcept;
320 :
321 : /// RAII guard that re-inserts the task sentinel after `run_task`.
322 : struct task_cleanup
323 : {
324 : reactor_scheduler const* sched;
325 : lock_type* lock;
326 : context_type& ctx;
327 : ~task_cleanup();
328 : };
329 :
330 : mutable mutex_type mutex_{true};
331 : mutable event_type cond_{true};
332 : mutable ready_queue completed_ops_;
333 : mutable std::atomic<std::int64_t> outstanding_work_{0};
334 : std::atomic<bool> stopped_{false};
335 : mutable std::atomic<bool> task_running_{false};
336 : mutable bool task_interrupted_ = false;
337 : // Set while the sentinel waits behind queued handlers (guarded by
338 : // mutex_).
339 : bool task_deferred_ = false;
340 : // Whether waiting is paying off (guarded by mutex_).
341 : reactor_defer_policy defer_policy_;
342 :
343 : // Descriptors retired during the running reactor pass (guarded by
344 : // mutex_).
345 : mutable reactor_retirable* retired_ = nullptr;
346 :
347 : /// Settle every descriptor retired before the current pass ended.
348 : /// @pre `lock` holds `mutex_`; it is held again on return.
349 : void settle_retired(lock_type& lock) const;
350 :
351 : /// Move what an enclosing handler on this thread queued privately
352 : /// to the shared queue. @pre `mutex_` is held.
353 : void adopt_enclosing_work(context_type const& ctx);
354 :
355 : // Runtime-configurable reactor tuning parameters.
356 : // Defaults match the library's built-in values.
357 : unsigned max_events_per_poll_ = 128;
358 : unsigned inline_budget_initial_ = 2;
359 : unsigned inline_budget_max_ = 16;
360 : unsigned unassisted_budget_ = 4;
361 :
362 : /// Bit 0 of `state_`: set when the condvar should be signaled.
363 : static constexpr std::size_t signaled_bit = 1;
364 :
365 : /// Increment per waiting thread in `state_`.
366 : static constexpr std::size_t waiter_increment = 2;
367 : mutable std::size_t state_ = 0;
368 :
369 : /// Sentinel op that triggers a reactor poll when dequeued.
370 : struct task_op final : scheduler_op
371 : {
372 : // LCOV_EXCL_START: the sentinel is intercepted by pointer
373 : // identity; its virtuals exist for vtable completeness.
374 : void operator()() override {}
375 : void destroy() override {}
376 : // LCOV_EXCL_STOP
377 : };
378 : task_op task_op_;
379 :
380 : /** Run the platform-specific reactor poll.
381 :
382 : @par Postconditions
383 : `lock` is owned on return, however the poll ended. An
384 : implementation that unlocks around the blocking call owes the
385 : caller a matching re-acquire on every path out, including the
386 : errors it retries rather than reports.
387 : */
388 : virtual void
389 : run_task(lock_type& lock, context_type& ctx, long timeout_us) = 0;
390 :
391 : /// Wake a blocked reactor (e.g. write to eventfd or pipe).
392 : virtual void interrupt_reactor() const = 0;
393 :
394 : /// Wake the reactor whenever the earliest timer changes.
395 : void on_timer_service_attached(timer_service& svc) override;
396 :
397 : private:
398 : struct work_cleanup
399 : {
400 : reactor_scheduler* sched;
401 : lock_type* lock;
402 : context_type& ctx;
403 : ~work_cleanup();
404 : };
405 :
406 : std::size_t do_one(lock_type& lock, long timeout_us, context_type& ctx);
407 :
408 : void signal_all(lock_type& lock) const;
409 : bool maybe_unlock_and_signal_one(lock_type& lock) const;
410 : bool unlock_and_signal_one(lock_type& lock) const;
411 : void clear_signal() const;
412 : void wait_for_signal(lock_type& lock) const;
413 : void wait_for_signal_for(lock_type& lock, long timeout_us) const;
414 : void wake_one_thread_and_unlock(lock_type& lock) const;
415 : };
416 :
417 : /** RAII guard that pushes/pops a scheduler context frame.
418 :
419 : On construction, pushes a new context frame onto the
420 : thread-local stack. On destruction, drains any remaining
421 : private queue items to the global queue and pops the frame.
422 : */
423 : struct reactor_thread_context_guard
424 : {
425 : /// The context frame managed by this guard.
426 : reactor_scheduler_context frame_;
427 :
428 : /// Marks the scheduler innermost for the executor's fast check.
429 : running_scheduler_guard running_;
430 :
431 : /// Construct the guard, pushing a frame for @a sched.
432 10907 : explicit reactor_thread_context_guard(
433 : reactor_scheduler const* sched) noexcept
434 10907 : : frame_(sched, reactor_context_stack.get())
435 10907 : , running_(sched)
436 : {
437 10907 : reactor_context_stack.set(&frame_);
438 10907 : }
439 :
440 : /** Destroy the guard, popping the frame.
441 :
442 : The private queue is empty here by invariant: work_cleanup and
443 : task_cleanup splice it to the global queue after every handler
444 : and every reactor pass.
445 : */
446 10907 : ~reactor_thread_context_guard() noexcept
447 : {
448 10907 : reactor_context_stack.set(frame_.next);
449 10907 : }
450 : };
451 :
452 : // ---- Inline implementations ------------------------------------------------
453 :
454 10907 : inline reactor_scheduler_context::reactor_scheduler_context(
455 10907 : reactor_scheduler const* k, reactor_scheduler_context* n)
456 10907 : : key(k)
457 10907 : , next(n)
458 10907 : , private_outstanding_work(0)
459 10907 : , inline_budget(0)
460 10907 : , inline_budget_max(static_cast<int>(k->inline_budget_initial()))
461 10907 : , unassisted(false)
462 : {
463 10907 : }
464 :
465 : inline void
466 50 : reactor_scheduler::configure_reactor(
467 : unsigned max_events,
468 : unsigned budget_init,
469 : unsigned budget_max,
470 : unsigned unassisted)
471 : {
472 50 : BOOST_COROSIO_ASSERT(
473 : max_events >= 1 &&
474 : max_events <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
475 50 : BOOST_COROSIO_ASSERT(
476 : budget_max <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
477 :
478 : // Clamp initial and unassisted to budget_max.
479 50 : if (budget_init > budget_max)
480 20 : budget_init = budget_max;
481 50 : if (unassisted > budget_max)
482 20 : unassisted = budget_max;
483 :
484 50 : max_events_per_poll_ = max_events;
485 50 : inline_budget_initial_ = budget_init;
486 50 : inline_budget_max_ = budget_max;
487 50 : unassisted_budget_ = unassisted;
488 50 : }
489 :
490 : inline void
491 105223 : reactor_scheduler::reset_inline_budget() const noexcept
492 : {
493 : // When budget is disabled (max==0), all paths below would no-op
494 : // (inline_budget stays 0). Skip the TLS lookup entirely.
495 105223 : if (inline_budget_max_ == 0)
496 58 : return;
497 105165 : if (auto* ctx = reactor_find_context(this))
498 : {
499 : // Cap when no other thread absorbed queued work
500 105165 : if (ctx->unassisted)
501 : {
502 104264 : ctx->inline_budget_max = static_cast<int>(unassisted_budget_);
503 104264 : ctx->inline_budget = static_cast<int>(unassisted_budget_);
504 104264 : return;
505 : }
506 : // Ramp up when previous cycle fully consumed budget.
507 : // max(1, ...) ensures the doubling escapes zero.
508 : // Doubled in a wider type: the cap may be as large as INT_MAX.
509 901 : if (ctx->inline_budget == 0)
510 : {
511 : long long const doubled =
512 290 : 2LL * (std::max)(1, ctx->inline_budget_max);
513 290 : long long const cap = inline_budget_max_;
514 290 : ctx->inline_budget_max = static_cast<int>((std::min)(doubled, cap));
515 : }
516 611 : else if (ctx->inline_budget < ctx->inline_budget_max)
517 : {
518 205 : ctx->inline_budget_max = static_cast<int>(inline_budget_initial_);
519 : }
520 901 : ctx->inline_budget = ctx->inline_budget_max;
521 : }
522 : }
523 :
524 : inline bool
525 443465 : reactor_scheduler::try_consume_inline_budget() const noexcept
526 : {
527 443465 : if (inline_budget_max_ == 0)
528 44 : return false;
529 443421 : if (auto* ctx = reactor_find_context(this))
530 : {
531 443419 : if (ctx->inline_budget > 0)
532 : {
533 354454 : --ctx->inline_budget;
534 354454 : return true;
535 : }
536 : }
537 88967 : return false;
538 : }
539 :
540 : inline void
541 3862 : reactor_scheduler::post(std::coroutine_handle<> h) const
542 : {
543 : struct post_handler final : scheduler_op
544 : {
545 : std::coroutine_handle<> h_;
546 :
547 3862 : explicit post_handler(std::coroutine_handle<> h) : h_(h) {}
548 7724 : ~post_handler() override = default;
549 :
550 3790 : void operator()() override
551 : {
552 3790 : auto saved = h_;
553 3790 : delete this;
554 3790 : saved.resume();
555 3774 : }
556 :
557 72 : void destroy() override
558 : {
559 : // A bare handle carries no owner to unwind from: it is
560 : // destroyed directly.
561 72 : auto saved = h_;
562 72 : delete this;
563 72 : saved.destroy();
564 72 : }
565 : };
566 :
567 3862 : auto ph = std::make_unique<post_handler>(h);
568 :
569 3862 : if (auto* ctx = reactor_find_context(this))
570 : {
571 34 : ++ctx->private_outstanding_work;
572 34 : ctx->private_queue.push(ph.release());
573 34 : return;
574 : }
575 :
576 3828 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
577 :
578 3828 : lock_type lock(mutex_);
579 3828 : completed_ops_.push(ph.release());
580 3828 : wake_one_thread_and_unlock(lock);
581 3862 : }
582 :
583 : inline void
584 113244 : reactor_scheduler::post(scheduler_op* h) const
585 : {
586 113244 : if (auto* ctx = reactor_find_context(this))
587 : {
588 111614 : ++ctx->private_outstanding_work;
589 111614 : ctx->private_queue.push(h);
590 111614 : return;
591 : }
592 :
593 1630 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
594 :
595 1630 : lock_type lock(mutex_);
596 1630 : completed_ops_.push(h);
597 1630 : wake_one_thread_and_unlock(lock);
598 1630 : }
599 :
600 : inline void
601 238558 : reactor_scheduler::post(capy::continuation& c) const
602 : {
603 238558 : if (auto* ctx = reactor_find_context(this))
604 : {
605 206256 : ++ctx->private_outstanding_work;
606 206256 : ctx->private_queue.push(c);
607 206256 : return;
608 : }
609 :
610 32302 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
611 :
612 32302 : lock_type lock(mutex_);
613 32302 : completed_ops_.push(c);
614 32302 : wake_one_thread_and_unlock(lock);
615 32302 : }
616 :
617 : inline bool
618 23846 : reactor_scheduler::running_in_this_thread() const noexcept
619 : {
620 23846 : return reactor_find_context(this) != nullptr;
621 : }
622 :
623 : inline void
624 8461 : reactor_scheduler::stop()
625 : {
626 8461 : lock_type lock(mutex_);
627 8461 : if (!stopped_.load(std::memory_order_acquire))
628 : {
629 7437 : stopped_.store(true, std::memory_order_release);
630 7437 : signal_all(lock);
631 7437 : interrupt_reactor();
632 : }
633 8461 : }
634 :
635 : inline bool
636 1359 : reactor_scheduler::stopped() const noexcept
637 : {
638 1359 : return stopped_.load(std::memory_order_acquire);
639 : }
640 :
641 : inline void
642 7668 : reactor_scheduler::restart()
643 : {
644 7668 : stopped_.store(false, std::memory_order_release);
645 7668 : }
646 :
647 : inline std::size_t
648 6487 : reactor_scheduler::run()
649 : {
650 12974 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
651 : {
652 216 : stop();
653 216 : return 0;
654 : }
655 :
656 6271 : reactor_thread_context_guard ctx(this);
657 6271 : lock_type lock(mutex_);
658 6271 : adopt_enclosing_work(ctx.frame_);
659 :
660 6271 : std::size_t n = 0;
661 : for (;;)
662 : {
663 379478 : if (!do_one(lock, -1, ctx.frame_))
664 6266 : break;
665 373207 : if (n != (std::numeric_limits<std::size_t>::max)())
666 373207 : ++n;
667 373207 : if (!lock.owns_lock())
668 74073 : lock.lock();
669 : }
670 6266 : return n;
671 6276 : }
672 :
673 : inline std::size_t
674 146 : reactor_scheduler::run_one()
675 : {
676 292 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
677 : {
678 5 : stop();
679 5 : return 0;
680 : }
681 :
682 141 : reactor_thread_context_guard ctx(this);
683 141 : lock_type lock(mutex_);
684 141 : adopt_enclosing_work(ctx.frame_);
685 141 : return do_one(lock, -1, ctx.frame_);
686 143 : }
687 :
688 : inline std::size_t
689 3809 : reactor_scheduler::wait_one(long usec)
690 : {
691 7618 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
692 : {
693 834 : stop();
694 834 : return 0;
695 : }
696 :
697 2975 : reactor_thread_context_guard ctx(this);
698 2975 : lock_type lock(mutex_);
699 2975 : adopt_enclosing_work(ctx.frame_);
700 2975 : return do_one(lock, usec, ctx.frame_);
701 2983 : }
702 :
703 : inline std::size_t
704 1479 : reactor_scheduler::poll()
705 : {
706 2958 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
707 : {
708 31 : stop();
709 31 : return 0;
710 : }
711 :
712 1448 : reactor_thread_context_guard ctx(this);
713 1448 : lock_type lock(mutex_);
714 1448 : adopt_enclosing_work(ctx.frame_);
715 :
716 1448 : std::size_t n = 0;
717 : for (;;)
718 : {
719 9628 : if (!do_one(lock, 0, ctx.frame_))
720 1446 : break;
721 8180 : if (n != (std::numeric_limits<std::size_t>::max)())
722 8180 : ++n;
723 8180 : if (!lock.owns_lock())
724 8176 : lock.lock();
725 : }
726 1446 : return n;
727 1450 : }
728 :
729 : inline std::size_t
730 79 : reactor_scheduler::poll_one()
731 : {
732 158 : if (outstanding_work_.load(std::memory_order_acquire) == 0)
733 : {
734 7 : stop();
735 7 : return 0;
736 : }
737 :
738 72 : reactor_thread_context_guard ctx(this);
739 72 : lock_type lock(mutex_);
740 72 : adopt_enclosing_work(ctx.frame_);
741 72 : return do_one(lock, 0, ctx.frame_);
742 74 : }
743 :
744 : inline void
745 66968 : reactor_scheduler::work_started() noexcept
746 : {
747 66968 : outstanding_work_.fetch_add(1, std::memory_order_relaxed);
748 66968 : }
749 :
750 : inline void
751 112059 : reactor_scheduler::work_finished() noexcept
752 : {
753 224118 : if (outstanding_work_.fetch_sub(1, std::memory_order_acq_rel) == 1)
754 7263 : stop();
755 112059 : }
756 :
757 : inline void
758 20858 : reactor_scheduler::compensating_work_started() const noexcept
759 : {
760 20858 : auto* ctx = reactor_find_context(this);
761 20858 : if (ctx)
762 20856 : ++ctx->private_outstanding_work;
763 20858 : }
764 :
765 : inline void
766 1988 : reactor_scheduler::post_counted(scheduler_op* op) const
767 : {
768 1988 : if (auto* ctx = reactor_find_context(this))
769 : {
770 604 : ctx->private_queue.push(op);
771 604 : return;
772 : }
773 :
774 1384 : lock_type lock(mutex_);
775 1384 : completed_ops_.push(op);
776 1384 : wake_one_thread_and_unlock(lock);
777 1384 : }
778 :
779 : inline void
780 15785 : reactor_scheduler::post_deferred_completions(ready_queue& ops) const
781 : {
782 15785 : if (ops.empty())
783 15785 : return;
784 :
785 2 : if (auto* ctx = reactor_find_context(this))
786 : {
787 2 : ctx->private_queue.splice(ops);
788 2 : return;
789 : }
790 :
791 MIS 0 : lock_type lock(mutex_);
792 0 : completed_ops_.splice(ops);
793 0 : wake_one_thread_and_unlock(lock);
794 0 : }
795 :
796 : inline void
797 HIT 22866 : reactor_scheduler::retire_descriptor(
798 : reactor_retirable& d, object_ref keep) const
799 : {
800 22866 : lock_type lock(mutex_);
801 22866 : if (!task_running_.load(std::memory_order_relaxed))
802 : {
803 : // No pass can hold an event, but an earlier one may have
804 : // queued d.
805 17528 : d.hand_off(keep);
806 17528 : return;
807 : }
808 : // Already parked for this pass: the held reference suffices.
809 5338 : if (d.retired_ref_)
810 92 : return;
811 5246 : d.retired_ref_ = std::move(keep);
812 5246 : d.retired_next_ = retired_;
813 5246 : retired_ = &d;
814 22866 : }
815 :
816 : inline void
817 10907 : reactor_scheduler::adopt_enclosing_work(context_type const& ctx)
818 : {
819 : // A run nested in a handler would otherwise never see what that
820 : // handler posted: it stays in the handler's frame until it returns.
821 10913 : for (auto* c = ctx.next; c != nullptr; c = c->next)
822 : {
823 12 : if (c->key != this)
824 6 : continue;
825 6 : if (c->private_outstanding_work > 0)
826 : {
827 6 : outstanding_work_.fetch_add(
828 : c->private_outstanding_work, std::memory_order_relaxed);
829 6 : c->private_outstanding_work = 0;
830 : }
831 6 : completed_ops_.splice(c->private_queue);
832 6 : return;
833 : }
834 : }
835 :
836 : inline void
837 177578 : reactor_scheduler::settle_retired(lock_type& lock) const
838 : {
839 : // Handed off under the lock, since a later pass may retire the same
840 : // descriptor again as soon as it is released. A reference that is
841 : // not handed off may be the owner's last, and its release recycles
842 : // through the pool mutex, so it is dropped with the lock released.
843 177578 : reactor_retirable* dropped = nullptr;
844 182824 : while (auto* d = retired_)
845 : {
846 5246 : retired_ = d->retired_next_;
847 5246 : d->retired_next_ = nullptr;
848 5246 : auto keep = std::move(d->retired_ref_);
849 5246 : d->hand_off(keep);
850 : // A drop already pending keeps the owner alive, so this one
851 : // cannot be the last and is released here.
852 5246 : if (!keep || d->dropped_ref_)
853 344 : continue;
854 4902 : d->dropped_ref_ = std::move(keep);
855 4902 : d->dropped_next_ = dropped;
856 4902 : dropped = d;
857 10492 : }
858 182480 : while (auto* d = dropped)
859 : {
860 4902 : dropped = d->dropped_next_;
861 4902 : d->dropped_next_ = nullptr;
862 4902 : auto keep = std::move(d->dropped_ref_);
863 4902 : lock.unlock();
864 4902 : keep.reset();
865 4902 : lock.lock();
866 4902 : }
867 177578 : }
868 :
869 : inline void
870 410 : reactor_scheduler::on_timer_service_attached(timer_service& svc)
871 : {
872 410 : svc.set_on_earliest_changed(timer_service::callback(this, [](void* p) {
873 3116 : static_cast<reactor_scheduler*>(p)->interrupt_reactor();
874 3116 : }));
875 410 : }
876 :
877 : inline void
878 4680 : reactor_scheduler::shutdown_drain()
879 : {
880 4680 : wait_cancels_posting();
881 :
882 4680 : lock_type lock(mutex_);
883 4680 : settle_retired(lock);
884 4680 : destroy_queued(lock);
885 4680 : signal_all(lock);
886 4680 : }
887 :
888 4696 : inline reactor_scheduler::~reactor_scheduler()
889 : {
890 : // Services give up their work in shutdown(), but anything posted
891 : // after the drain, such as from a destructor, is still destroyed.
892 4696 : lock_type lock(mutex_);
893 4696 : destroy_queued(lock);
894 4696 : }
895 :
896 : inline void
897 9376 : reactor_scheduler::destroy_queued(lock_type& lock) noexcept
898 : {
899 23532 : while (auto e = completed_ops_.pop())
900 : {
901 14156 : if (ready_is_continuation(e))
902 : {
903 7836 : lock.unlock();
904 7836 : if (auto* c = ready_as_cont(e); c->h)
905 7836 : capy::abandon(*c);
906 7836 : lock.lock();
907 : }
908 : else
909 : {
910 6320 : auto* op = ready_as_op(e);
911 6320 : if (op == &task_op_)
912 4677 : continue;
913 1643 : lock.unlock();
914 1643 : op->destroy();
915 1643 : lock.lock();
916 : }
917 14156 : }
918 9376 : }
919 :
920 : inline void
921 12117 : reactor_scheduler::signal_all(lock_type&) const
922 : {
923 12117 : state_ |= signaled_bit;
924 12117 : cond_.notify_all();
925 12117 : }
926 :
927 : inline bool
928 40628 : reactor_scheduler::maybe_unlock_and_signal_one(lock_type& lock) const
929 : {
930 40628 : state_ |= signaled_bit;
931 40628 : if (state_ > signaled_bit)
932 : {
933 : // Signalled before unlocking: a post from another context's
934 : // thread may be the last touch of this scheduler before the
935 : // posted work runs, finishes and lets it be destroyed.
936 198 : cond_.notify_one();
937 198 : lock.unlock();
938 198 : return true;
939 : }
940 40430 : return false;
941 : }
942 :
943 : inline bool
944 510455 : reactor_scheduler::unlock_and_signal_one(lock_type& lock) const
945 : {
946 510455 : state_ |= signaled_bit;
947 510455 : bool have_waiters = state_ > signaled_bit;
948 510455 : lock.unlock();
949 510455 : if (have_waiters)
950 4400 : cond_.notify_one();
951 510455 : return have_waiters;
952 : }
953 :
954 : inline void
955 792 : reactor_scheduler::clear_signal() const
956 : {
957 792 : state_ &= ~signaled_bit;
958 792 : }
959 :
960 : inline void
961 767 : reactor_scheduler::wait_for_signal(lock_type& lock) const
962 : {
963 1598 : while ((state_ & signaled_bit) == 0)
964 : {
965 831 : state_ += waiter_increment;
966 831 : cond_.wait(lock);
967 831 : state_ -= waiter_increment;
968 : }
969 767 : }
970 :
971 : inline void
972 25 : reactor_scheduler::wait_for_signal_for(lock_type& lock, long timeout_us) const
973 : {
974 25 : if ((state_ & signaled_bit) == 0)
975 : {
976 25 : state_ += waiter_increment;
977 25 : cond_.wait_for(lock, std::chrono::microseconds(timeout_us));
978 25 : state_ -= waiter_increment;
979 : }
980 25 : }
981 :
982 : inline void
983 39144 : reactor_scheduler::wake_one_thread_and_unlock(lock_type& lock) const
984 : {
985 39144 : if (maybe_unlock_and_signal_one(lock))
986 178 : return;
987 :
988 38966 : if (task_running_.load(std::memory_order_relaxed) && !task_interrupted_)
989 : {
990 1674 : task_interrupted_ = true;
991 : // Under the lock, for the same reason as the signal above.
992 1674 : interrupt_reactor();
993 1674 : lock.unlock();
994 : }
995 : else
996 : {
997 37292 : lock.unlock();
998 : }
999 : }
1000 :
1001 384288 : inline reactor_scheduler::work_cleanup::~work_cleanup()
1002 : {
1003 384288 : std::int64_t produced = ctx.private_outstanding_work;
1004 384288 : if (produced > 1)
1005 955 : sched->outstanding_work_.fetch_add(
1006 : produced - 1, std::memory_order_relaxed);
1007 383333 : else if (produced < 1)
1008 65272 : sched->work_finished();
1009 384288 : ctx.private_outstanding_work = 0;
1010 :
1011 384288 : if (!ctx.private_queue.empty())
1012 : {
1013 299470 : lock->lock();
1014 299470 : sched->completed_ops_.splice(ctx.private_queue);
1015 : }
1016 384288 : }
1017 :
1018 172901 : inline reactor_scheduler::task_cleanup::~task_cleanup()
1019 : {
1020 172901 : if (ctx.private_outstanding_work > 0)
1021 : {
1022 10229 : sched->outstanding_work_.fetch_add(
1023 10229 : ctx.private_outstanding_work, std::memory_order_relaxed);
1024 10229 : ctx.private_outstanding_work = 0;
1025 : }
1026 :
1027 172901 : if (!ctx.private_queue.empty())
1028 : {
1029 10229 : if (!lock->owns_lock())
1030 MIS 0 : lock->lock();
1031 HIT 10229 : sched->completed_ops_.splice(ctx.private_queue);
1032 : }
1033 172901 : }
1034 :
1035 : inline std::size_t
1036 392294 : reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
1037 : {
1038 : for (;;)
1039 : {
1040 693928 : if (stopped_.load(std::memory_order_acquire))
1041 : {
1042 : // The next run is scored afresh, but a deferral left pending
1043 : // stays marked: clearing the mark would let a loop of short
1044 : // runs defer the sentinel forever.
1045 6504 : defer_policy_.reset();
1046 6504 : return 0;
1047 : }
1048 :
1049 687424 : auto const e = completed_ops_.pop();
1050 687424 : scheduler_op* op = ready_is_continuation(e) ? nullptr : ready_as_op(e);
1051 :
1052 : // Handle reactor sentinel — time to poll for I/O
1053 687424 : if (op == &task_op_)
1054 : {
1055 302335 : bool more_handlers = !completed_ops_.empty();
1056 :
1057 : // Handlers queued here usually start the next I/O
1058 : // themselves, so running them first can save a reactor pass.
1059 : // Defer at most once in a row so reposting work cannot
1060 : // starve I/O. poll() and poll_one() don't defer: the reactor
1061 : // turn they would skip may be their last before returning.
1062 302335 : if (task_deferred_)
1063 : {
1064 128968 : task_deferred_ = false;
1065 128968 : defer_policy_.on_deferred_turn(more_handlers);
1066 : }
1067 173367 : else if (more_handlers && timeout_us != 0)
1068 : {
1069 131431 : if (defer_policy_.may_defer())
1070 : {
1071 129428 : task_deferred_ = true;
1072 129428 : completed_ops_.push(&task_op_);
1073 129428 : continue;
1074 : }
1075 2003 : defer_policy_.on_polled_instead();
1076 : }
1077 :
1078 218472 : if (!more_handlers &&
1079 91130 : outstanding_work_.load(std::memory_order_acquire) == 0)
1080 : {
1081 6 : completed_ops_.push(&task_op_);
1082 6 : return 0;
1083 : }
1084 :
1085 172901 : long task_timeout_us = more_handlers ? 0 : timeout_us;
1086 172901 : task_interrupted_ = task_timeout_us == 0;
1087 172901 : task_running_.store(true, std::memory_order_release);
1088 :
1089 : // Wake a peer to take the pending handlers while this thread
1090 : // polls the reactor; skipped when one_thread_ (no peer exists).
1091 172901 : if (more_handlers && !one_thread_)
1092 127340 : unlock_and_signal_one(lock);
1093 :
1094 : try
1095 : {
1096 172901 : run_task(lock, ctx, task_timeout_us);
1097 : }
1098 3 : catch (...)
1099 : {
1100 3 : task_running_.store(false, std::memory_order_relaxed);
1101 3 : throw;
1102 3 : }
1103 :
1104 172898 : task_running_.store(false, std::memory_order_relaxed);
1105 172898 : bool const idle = completed_ops_.empty();
1106 172898 : completed_ops_.push(&task_op_);
1107 172898 : settle_retired(lock);
1108 : // A bounded call blocks at most once, then finishes as a poll
1109 : // does, running what its pass queued.
1110 172898 : if (timeout_us > 0)
1111 2396 : timeout_us = 0;
1112 : // A poll ends once a pass leaves nothing queued; going round
1113 : // again would only repeat the pass. A peer that found the
1114 : // queue empty during the pass may be asleep, so it is woken
1115 : // to take the sentinel.
1116 172898 : if (timeout_us == 0 && idle)
1117 : {
1118 1484 : maybe_unlock_and_signal_one(lock);
1119 1484 : return 0;
1120 : }
1121 171414 : continue;
1122 171414 : }
1123 :
1124 : // Handle ready entry (op or continuation)
1125 385089 : if (e != nullptr)
1126 : {
1127 384288 : bool more = !completed_ops_.empty();
1128 :
1129 384288 : if (more && !one_thread_)
1130 : {
1131 : // Wake a peer for the remaining work; unassisted if none
1132 : // was parked to take it.
1133 383115 : ctx.unassisted = !unlock_and_signal_one(lock);
1134 : }
1135 : else
1136 : {
1137 : // No peer to wake (one_thread_, or nothing more queued).
1138 1173 : ctx.unassisted = more;
1139 1173 : lock.unlock();
1140 : }
1141 :
1142 384288 : [[maybe_unused]] work_cleanup on_exit{this, &lock, ctx};
1143 :
1144 384288 : if (ready_is_continuation(e))
1145 230722 : ready_as_cont(e)->h.resume();
1146 : else
1147 153566 : (*op)();
1148 384272 : return 1;
1149 384288 : }
1150 :
1151 1602 : if (outstanding_work_.load(std::memory_order_acquire) == 0 ||
1152 : timeout_us == 0)
1153 9 : return 0;
1154 :
1155 792 : clear_signal();
1156 792 : if (timeout_us < 0)
1157 : {
1158 767 : wait_for_signal(lock);
1159 : }
1160 : else
1161 : {
1162 25 : wait_for_signal_for(lock, timeout_us);
1163 : // A bounded call blocks once, so its caller can check its
1164 : // deadline; another thread may hold the reactor for good.
1165 25 : timeout_us = 0;
1166 : }
1167 301634 : }
1168 : }
1169 :
1170 : } // namespace boost::corosio::detail
1171 :
1172 : #endif // BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
|