include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp

98.8% Lines (409 / 414, 2 excl) 100.0% Functions (52 / 52, 2 excl)
reactor_scheduler.hpp
f(x) Functions (54)
Function Calls Lines Blocks
boost::corosio::detail::reactor_retirable::~reactor_retirable() :69 12645x 100.0% 100.0% boost::corosio::detail::reactor_find_context(boost::corosio::detail::reactor_scheduler const*) :111 950944x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::notify_reparked() const :227 411x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::inline_budget_initial() const :277 10907x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::scheduler_locking_disabled() const :283 1460x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::configure_threading(boost::corosio::detail::scheduler::threading_config) :288 4680x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::reactor_scheduler() :304 4696x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::task_op::operator()() :374 – – – boost::corosio::detail::reactor_scheduler::task_op::destroy() :375 – – – boost::corosio::detail::reactor_thread_context_guard::reactor_thread_context_guard(boost::corosio::detail::reactor_scheduler const*) :432 10907x 100.0% 100.0% boost::corosio::detail::reactor_thread_context_guard::~reactor_thread_context_guard() :446 10907x 100.0% 100.0% boost::corosio::detail::reactor_scheduler_context::reactor_scheduler_context(boost::corosio::detail::reactor_scheduler const*, boost::corosio::detail::reactor_scheduler_context*) :454 10907x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::configure_reactor(unsigned int, unsigned int, unsigned int, unsigned int) :466 50x 100.0% 85.0% boost::corosio::detail::reactor_scheduler::reset_inline_budget() const :491 105223x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::try_consume_inline_budget() const :525 443465x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const :541 3862x 100.0% 84.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::post_handler(std::__n4861::coroutine_handle<void>) :547 3862x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::~post_handler() :548 7724x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::operator()() :550 3790x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::destroy() :557 72x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(boost::corosio::detail::scheduler_op*) const :584 113244x 100.0% 87.0% boost::corosio::detail::reactor_scheduler::post(boost::capy::continuation&) const :601 238558x 100.0% 87.0% boost::corosio::detail::reactor_scheduler::running_in_this_thread() const :618 23846x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::stop() :624 8461x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::stopped() const :636 1359x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::restart() :642 7668x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::run() :648 6487x 100.0% 86.0% boost::corosio::detail::reactor_scheduler::run_one() :674 146x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::wait_one(long) :689 3809x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::poll() :704 1479x 100.0% 86.0% boost::corosio::detail::reactor_scheduler::poll_one() :730 79x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::work_started() :745 66968x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::work_finished() :751 112059x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::compensating_work_started() const :758 20858x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post_counted(boost::corosio::detail::scheduler_op*) const :766 1988x 100.0% 85.0% boost::corosio::detail::reactor_scheduler::post_deferred_completions(boost::corosio::detail::ready_queue&) const :780 15785x 60.0% 59.0% boost::corosio::detail::reactor_scheduler::retire_descriptor(boost::corosio::detail::reactor_retirable&, boost::corosio::detail::object_ref) const :797 22866x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::adopt_enclosing_work(boost::corosio::detail::reactor_scheduler_context const&) :817 10907x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::settle_retired(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :837 177578x 100.0% 94.0% boost::corosio::detail::reactor_scheduler::on_timer_service_attached(boost::corosio::detail::timer_service&) :870 410x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::on_timer_service_attached(boost::corosio::detail::timer_service&)::{lambda(void*)#1}::operator()(void*) const :872 3116x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::shutdown_drain() :878 4680x 100.0% 80.0% boost::corosio::detail::reactor_scheduler::~reactor_scheduler() :888 4696x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::destroy_queued(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) :897 9376x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::signal_all(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :921 12117x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::maybe_unlock_and_signal_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :928 40628x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::unlock_and_signal_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :944 510455x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::clear_signal() const :955 792x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wait_for_signal(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :961 767x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wait_for_signal_for(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&, long) const :972 25x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wake_one_thread_and_unlock(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :983 39144x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::work_cleanup::~work_cleanup() :1001 384288x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::task_cleanup::~task_cleanup() :1018 172901x 90.0% 91.0% boost::corosio::detail::reactor_scheduler::do_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&, long, boost::corosio::detail::reactor_scheduler_context&) :1036 392294x 100.0% 90.0%
Line TLA Hits Source Code
1 //
2 // Copyright (c) 2026 Steve Gerbino
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 //
7 // Official repository: https://github.com/cppalliance/corosio
8 //
9
10 #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
11 #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
12
13 #include <boost/corosio/detail/config.hpp>
14 #include <boost/corosio/detail/object_ref.hpp>
15 #include <boost/capy/ex/execution_context.hpp>
16 #include <boost/capy/ex/abandon.hpp>
17
18 #include <boost/corosio/detail/ready_queue.hpp>
19 #include <boost/corosio/detail/scheduler.hpp>
20 #include <boost/corosio/detail/scheduler_op.hpp>
21 #include <boost/corosio/detail/thread_local_ptr.hpp>
22 #include <boost/corosio/detail/timer_service.hpp>
23 #include <boost/corosio/native/detail/reactor/reactor_defer_policy.hpp>
24
25 #include <atomic>
26 #include <chrono>
27 #include <coroutine>
28 #include <cstddef>
29 #include <cstdint>
30 #include <limits>
31 #include <memory>
32
33 #include <boost/corosio/detail/conditionally_enabled_mutex.hpp>
34 #include <boost/corosio/detail/conditionally_enabled_event.hpp>
35
36 namespace boost::corosio::detail {
37
38 // Forward declarations
39 class reactor_scheduler;
40
41 /** A deregistered descriptor whose owner must outlive the reactor pass
42 that may still hold it.
43
44 An event the reactor has already read from the kernel can name a
45 descriptor that another thread closes before the reactor queues it.
46 The closing thread hands over a reference instead of dropping it, so
47 the owner is neither freed nor recycled until that pass ends.
48 */
49 struct reactor_retirable
50 {
51 // All four fields are guarded by the scheduler's mutex.
52
53 /// Next in the scheduler's retired list.
54 reactor_retirable* retired_next_ = nullptr;
55
56 /// Owner reference held until the pass ends.
57 object_ref retired_ref_;
58
59 /// Next in a settling thread's list of references to drop.
60 reactor_retirable* dropped_next_ = nullptr;
61
62 /// Owner reference waiting to be dropped outside the mutex.
63 object_ref dropped_ref_;
64
65 /// Move @p keep into a queued completion, if one is pending.
66 virtual void hand_off(object_ref& keep) noexcept = 0;
67
68 protected:
69 12645x ~reactor_retirable() = default;
70 };
71
72 /** Per-thread state for a reactor scheduler.
73
74 Each thread running a scheduler's event loop has one of these
75 on a thread-local stack. It holds a private work queue and
76 inline completion budget for speculative I/O fast paths.
77 */
78 struct BOOST_COROSIO_SYMBOL_VISIBLE reactor_scheduler_context
79 {
80 /// Scheduler this context belongs to.
81 reactor_scheduler const* key;
82
83 /// Next context frame on this thread's stack.
84 reactor_scheduler_context* next;
85
86 /// Private work queue for reduced contention.
87 ready_queue private_queue;
88
89 /// Unflushed work count for the private queue.
90 std::int64_t private_outstanding_work;
91
92 /// Remaining inline completions allowed this cycle.
93 int inline_budget;
94
95 /// Maximum inline budget (adaptive, 2-16).
96 int inline_budget_max;
97
98 /// True if no other thread absorbed queued work last cycle.
99 bool unassisted;
100
101 /// Construct a context frame linked to @a n.
102 reactor_scheduler_context(
103 reactor_scheduler const* k, reactor_scheduler_context* n);
104 };
105
106 /// Thread-local context stack for reactor schedulers.
107 inline thread_local_ptr<reactor_scheduler_context> reactor_context_stack;
108
109 /// Find the context frame for a scheduler on this thread.
110 inline reactor_scheduler_context*
111 950944x reactor_find_context(reactor_scheduler const* self) noexcept
112 {
113 951478x for (auto* c = reactor_context_stack.get(); c != nullptr; c = c->next)
114 {
115 888484x if (c->key == self)
116 887950x return c;
117 }
118 62994x return nullptr;
119 }
120
121 /** Non-template base for reactor-backed scheduler implementations.
122
123 Provides the complete threading model shared by epoll, kqueue,
124 and select schedulers: signal state machine, inline completion
125 budget, work counting, run/poll methods, and the do_one event
126 loop.
127
128 Derived classes provide platform-specific hooks by overriding:
129 - `run_task(lock, ctx)` to run the reactor poll
130 - `interrupt_reactor()` to wake a blocked reactor
131
132 De-templated from the original CRTP design to eliminate
133 duplicate instantiations when multiple backends are compiled
134 into the same binary. Virtual dispatch for run_task (called
135 once per reactor cycle, before a blocking syscall) has
136 negligible overhead.
137
138 @par Thread Safety
139 All public member functions are thread-safe.
140 */
141 class reactor_scheduler : public scheduler
142 {
143 public:
144 using context_type = reactor_scheduler_context;
145 using mutex_type = conditionally_enabled_mutex;
146 using lock_type = mutex_type::scoped_lock;
147 using event_type = conditionally_enabled_event;
148
149 /// Post a coroutine for deferred execution.
150 void post(std::coroutine_handle<> h) const override;
151
152 /// Post a scheduler operation for deferred execution.
153 void post(scheduler_op* h) const override;
154
155 /** Release an owner reference once no reactor pass can hold `d`.
156
157 Called after a descriptor is deregistered. If a reactor pass is
158 running, `keep` is parked on `d` until that pass ends. Then, or
159 at once if no pass is running, it moves to `d`'s queued
160 invocation if `d` is queued, and is dropped otherwise.
161
162 @param d The deregistered descriptor.
163 @param keep A reference to the descriptor's owner.
164 */
165 void retire_descriptor(reactor_retirable& d, object_ref keep) const;
166
167 /// Post a continuation for deferred execution.
168 void post(capy::continuation&) const override;
169
170 /// Return true if called from a thread running this scheduler.
171 bool running_in_this_thread() const noexcept override;
172
173 /// Request the scheduler to stop dispatching handlers.
174 void stop() override;
175
176 /// Return true if the scheduler has been stopped.
177 bool stopped() const noexcept override;
178
179 /// Reset the stopped state so `run()` can resume.
180 void restart() override;
181
182 /// Run the event loop until no work remains.
183 std::size_t run() override;
184
185 /// Run until one handler completes or no work remains.
186 std::size_t run_one() override;
187
188 /// Run until one handler completes or @a usec elapses.
189 std::size_t wait_one(long usec) override;
190
191 /// Run ready handlers without blocking.
192 std::size_t poll() override;
193
194 /// Run at most one ready handler without blocking.
195 std::size_t poll_one() override;
196
197 /// Increment the outstanding work count.
198 void work_started() noexcept override;
199
200 /// Decrement the outstanding work count, stopping on zero.
201 void work_finished() noexcept override;
202
203 /** Reset the thread's inline completion budget.
204
205 Called at the start of each posted completion handler to
206 grant a fresh budget for speculative inline completions.
207 */
208 void reset_inline_budget() const noexcept;
209
210 /** Consume one unit of inline budget if available.
211
212 @return True if budget was available and consumed.
213 */
214 bool try_consume_inline_budget() const noexcept;
215
216 /** Offset a forthcoming work_finished from work_cleanup.
217
218 Called by descriptor_state when all I/O returned EAGAIN and
219 no handler will be executed. Must be called from a scheduler
220 thread.
221 */
222 /** Note that an op was parked outside the usual registration path.
223
224 A reactor that builds its poll set from parked ops must rebuild
225 it; one the kernel tracks needs nothing.
226 */
227 411x virtual void notify_reparked() const {}
228
229 void compensating_work_started() const noexcept;
230
231 /** Post an operation whose work is already counted.
232
233 The posting thread touches nothing of the scheduler after this
234 returns, unlike a @ref post followed by @ref work_finished: once
235 posted, the operation can complete and let its context be
236 destroyed.
237
238 @pre work_started() was called for @p op.
239
240 @param op The operation to post.
241 */
242 void post_counted(scheduler_op* op) const;
243
244 /** Post completed operations for deferred invocation.
245
246 If called from a thread running this scheduler, operations
247 go to the thread's private queue (fast path). Otherwise,
248 operations are added to the global queue under mutex and a
249 waiter is signaled.
250
251 @pre work_started() must have been called for each operation.
252
253 @param ops Queue of operations to post.
254 */
255 void post_deferred_completions(ready_queue& ops) const;
256
257 /** Apply runtime configuration to the scheduler.
258
259 Called by `io_context` after construction. Values that do
260 not apply to this backend are silently ignored.
261
262 @pre `max_events` is in [ 1, `INT_MAX` ] and `budget_max` is at
263 most `INT_MAX`; `io_context` validates both.
264
265 @param max_events Event buffer size for epoll/kqueue.
266 @param budget_init Starting inline completion budget.
267 @param budget_max Hard ceiling on adaptive budget ramp-up.
268 @param unassisted Budget when single-threaded.
269 */
270 virtual void configure_reactor(
271 unsigned max_events,
272 unsigned budget_init,
273 unsigned budget_max,
274 unsigned unassisted);
275
276 /// Return the configured initial inline budget.
277 10907x unsigned inline_budget_initial() const noexcept
278 {
279 10907x return inline_budget_initial_;
280 }
281
282 /// Return true when scheduler locking is disabled (fully-lockless tier).
283 1460x bool scheduler_locking_disabled() const noexcept override
284 {
285 1460x return scheduler_locking_disabled_;
286 }
287
288 4680x void configure_threading(threading_config cfg) noexcept override
289 {
290 4680x scheduler_locking_disabled_ = !cfg.scheduler_locking;
291 // reactor_io_locking takes effect at descriptor registration (see the
292 // register_descriptor overrides), not here.
293 4680x reactor_io_locking_ = cfg.reactor_io_locking;
294 4680x one_thread_ = cfg.one_thread;
295 4680x mutex_.set_enabled(cfg.scheduler_locking);
296 4680x cond_.set_enabled(cfg.scheduler_locking);
297 4680x }
298
299 protected:
300 bool scheduler_locking_disabled_ = false;
301 bool reactor_io_locking_ = true;
302 bool one_thread_ = false;
303
304 4696x reactor_scheduler() = default;
305
306 /// Destroy, destroying whatever was posted after the drain.
307 ~reactor_scheduler() override;
308
309 /** Drain completed_ops during shutdown.
310
311 Pops all operations from the global queue and destroys them,
312 skipping the task sentinel. Signals all waiting threads.
313 Derived classes call this from their shutdown() override
314 before performing platform-specific cleanup.
315 */
316 void shutdown_drain();
317
318 /// Destroy every queued operation without running it.
319 void destroy_queued(lock_type& lock) noexcept;
320
321 /// RAII guard that re-inserts the task sentinel after `run_task`.
322 struct task_cleanup
323 {
324 reactor_scheduler const* sched;
325 lock_type* lock;
326 context_type& ctx;
327 ~task_cleanup();
328 };
329
330 mutable mutex_type mutex_{true};
331 mutable event_type cond_{true};
332 mutable ready_queue completed_ops_;
333 mutable std::atomic<std::int64_t> outstanding_work_{0};
334 std::atomic<bool> stopped_{false};
335 mutable std::atomic<bool> task_running_{false};
336 mutable bool task_interrupted_ = false;
337 // Set while the sentinel waits behind queued handlers (guarded by
338 // mutex_).
339 bool task_deferred_ = false;
340 // Whether waiting is paying off (guarded by mutex_).
341 reactor_defer_policy defer_policy_;
342
343 // Descriptors retired during the running reactor pass (guarded by
344 // mutex_).
345 mutable reactor_retirable* retired_ = nullptr;
346
347 /// Settle every descriptor retired before the current pass ended.
348 /// @pre `lock` holds `mutex_`; it is held again on return.
349 void settle_retired(lock_type& lock) const;
350
351 /// Move what an enclosing handler on this thread queued privately
352 /// to the shared queue. @pre `mutex_` is held.
353 void adopt_enclosing_work(context_type const& ctx);
354
355 // Runtime-configurable reactor tuning parameters.
356 // Defaults match the library's built-in values.
357 unsigned max_events_per_poll_ = 128;
358 unsigned inline_budget_initial_ = 2;
359 unsigned inline_budget_max_ = 16;
360 unsigned unassisted_budget_ = 4;
361
362 /// Bit 0 of `state_`: set when the condvar should be signaled.
363 static constexpr std::size_t signaled_bit = 1;
364
365 /// Increment per waiting thread in `state_`.
366 static constexpr std::size_t waiter_increment = 2;
367 mutable std::size_t state_ = 0;
368
369 /// Sentinel op that triggers a reactor poll when dequeued.
370 struct task_op final : scheduler_op
371 {
372 // LCOV_EXCL_START: the sentinel is intercepted by pointer
373 // identity; its virtuals exist for vtable completeness.
374 − void operator()() override {}
375 − void destroy() override {}
376 // LCOV_EXCL_STOP
377 };
378 task_op task_op_;
379
380 /** Run the platform-specific reactor poll.
381
382 @par Postconditions
383 `lock` is owned on return, however the poll ended. An
384 implementation that unlocks around the blocking call owes the
385 caller a matching re-acquire on every path out, including the
386 errors it retries rather than reports.
387 */
388 virtual void
389 run_task(lock_type& lock, context_type& ctx, long timeout_us) = 0;
390
391 /// Wake a blocked reactor (e.g. write to eventfd or pipe).
392 virtual void interrupt_reactor() const = 0;
393
394 /// Wake the reactor whenever the earliest timer changes.
395 void on_timer_service_attached(timer_service& svc) override;
396
397 private:
398 struct work_cleanup
399 {
400 reactor_scheduler* sched;
401 lock_type* lock;
402 context_type& ctx;
403 ~work_cleanup();
404 };
405
406 std::size_t do_one(lock_type& lock, long timeout_us, context_type& ctx);
407
408 void signal_all(lock_type& lock) const;
409 bool maybe_unlock_and_signal_one(lock_type& lock) const;
410 bool unlock_and_signal_one(lock_type& lock) const;
411 void clear_signal() const;
412 void wait_for_signal(lock_type& lock) const;
413 void wait_for_signal_for(lock_type& lock, long timeout_us) const;
414 void wake_one_thread_and_unlock(lock_type& lock) const;
415 };
416
417 /** RAII guard that pushes/pops a scheduler context frame.
418
419 On construction, pushes a new context frame onto the
420 thread-local stack. On destruction, drains any remaining
421 private queue items to the global queue and pops the frame.
422 */
423 struct reactor_thread_context_guard
424 {
425 /// The context frame managed by this guard.
426 reactor_scheduler_context frame_;
427
428 /// Marks the scheduler innermost for the executor's fast check.
429 running_scheduler_guard running_;
430
431 /// Construct the guard, pushing a frame for @a sched.
432 10907x explicit reactor_thread_context_guard(
433 reactor_scheduler const* sched) noexcept
434 10907x : frame_(sched, reactor_context_stack.get())
435 10907x , running_(sched)
436 {
437 10907x reactor_context_stack.set(&frame_);
438 10907x }
439
440 /** Destroy the guard, popping the frame.
441
442 The private queue is empty here by invariant: work_cleanup and
443 task_cleanup splice it to the global queue after every handler
444 and every reactor pass.
445 */
446 10907x ~reactor_thread_context_guard() noexcept
447 {
448 10907x reactor_context_stack.set(frame_.next);
449 10907x }
450 };
451
452 // ---- Inline implementations ------------------------------------------------
453
454 10907x inline reactor_scheduler_context::reactor_scheduler_context(
455 10907x reactor_scheduler const* k, reactor_scheduler_context* n)
456 10907x : key(k)
457 10907x , next(n)
458 10907x , private_outstanding_work(0)
459 10907x , inline_budget(0)
460 10907x , inline_budget_max(static_cast<int>(k->inline_budget_initial()))
461 10907x , unassisted(false)
462 {
463 10907x }
464
465 inline void
466 50x reactor_scheduler::configure_reactor(
467 unsigned max_events,
468 unsigned budget_init,
469 unsigned budget_max,
470 unsigned unassisted)
471 {
472 50x BOOST_COROSIO_ASSERT(
473 max_events >= 1 &&
474 max_events <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
475 50x BOOST_COROSIO_ASSERT(
476 budget_max <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
477
478 // Clamp initial and unassisted to budget_max.
479 50x if (budget_init > budget_max)
480 20x budget_init = budget_max;
481 50x if (unassisted > budget_max)
482 20x unassisted = budget_max;
483
484 50x max_events_per_poll_ = max_events;
485 50x inline_budget_initial_ = budget_init;
486 50x inline_budget_max_ = budget_max;
487 50x unassisted_budget_ = unassisted;
488 50x }
489
490 inline void
491 105223x reactor_scheduler::reset_inline_budget() const noexcept
492 {
493 // When budget is disabled (max==0), all paths below would no-op
494 // (inline_budget stays 0). Skip the TLS lookup entirely.
495 105223x if (inline_budget_max_ == 0)
496 58x return;
497 105165x if (auto* ctx = reactor_find_context(this))
498 {
499 // Cap when no other thread absorbed queued work
500 105165x if (ctx->unassisted)
501 {
502 104264x ctx->inline_budget_max = static_cast<int>(unassisted_budget_);
503 104264x ctx->inline_budget = static_cast<int>(unassisted_budget_);
504 104264x return;
505 }
506 // Ramp up when previous cycle fully consumed budget.
507 // max(1, ...) ensures the doubling escapes zero.
508 // Doubled in a wider type: the cap may be as large as INT_MAX.
509 901x if (ctx->inline_budget == 0)
510 {
511 long long const doubled =
512 290x 2LL * (std::max)(1, ctx->inline_budget_max);
513 290x long long const cap = inline_budget_max_;
514 290x ctx->inline_budget_max = static_cast<int>((std::min)(doubled, cap));
515 }
516 611x else if (ctx->inline_budget < ctx->inline_budget_max)
517 {
518 205x ctx->inline_budget_max = static_cast<int>(inline_budget_initial_);
519 }
520 901x ctx->inline_budget = ctx->inline_budget_max;
521 }
522 }
523
524 inline bool
525 443465x reactor_scheduler::try_consume_inline_budget() const noexcept
526 {
527 443465x if (inline_budget_max_ == 0)
528 44x return false;
529 443421x if (auto* ctx = reactor_find_context(this))
530 {
531 443419x if (ctx->inline_budget > 0)
532 {
533 354454x --ctx->inline_budget;
534 354454x return true;
535 }
536 }
537 88967x return false;
538 }
539
540 inline void
541 3862x reactor_scheduler::post(std::coroutine_handle<> h) const
542 {
543 struct post_handler final : scheduler_op
544 {
545 std::coroutine_handle<> h_;
546
547 3862x explicit post_handler(std::coroutine_handle<> h) : h_(h) {}
548 7724x ~post_handler() override = default;
549
550 3790x void operator()() override
551 {
552 3790x auto saved = h_;
553 3790x delete this;
554 3790x saved.resume();
555 3774x }
556
557 72x void destroy() override
558 {
559 // A bare handle carries no owner to unwind from: it is
560 // destroyed directly.
561 72x auto saved = h_;
562 72x delete this;
563 72x saved.destroy();
564 72x }
565 };
566
567 3862x auto ph = std::make_unique<post_handler>(h);
568
569 3862x if (auto* ctx = reactor_find_context(this))
570 {
571 34x ++ctx->private_outstanding_work;
572 34x ctx->private_queue.push(ph.release());
573 34x return;
574 }
575
576 3828x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
577
578 3828x lock_type lock(mutex_);
579 3828x completed_ops_.push(ph.release());
580 3828x wake_one_thread_and_unlock(lock);
581 3862x }
582
583 inline void
584 113244x reactor_scheduler::post(scheduler_op* h) const
585 {
586 113244x if (auto* ctx = reactor_find_context(this))
587 {
588 111614x ++ctx->private_outstanding_work;
589 111614x ctx->private_queue.push(h);
590 111614x return;
591 }
592
593 1630x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
594
595 1630x lock_type lock(mutex_);
596 1630x completed_ops_.push(h);
597 1630x wake_one_thread_and_unlock(lock);
598 1630x }
599
600 inline void
601 238558x reactor_scheduler::post(capy::continuation& c) const
602 {
603 238558x if (auto* ctx = reactor_find_context(this))
604 {
605 206256x ++ctx->private_outstanding_work;
606 206256x ctx->private_queue.push(c);
607 206256x return;
608 }
609
610 32302x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
611
612 32302x lock_type lock(mutex_);
613 32302x completed_ops_.push(c);
614 32302x wake_one_thread_and_unlock(lock);
615 32302x }
616
617 inline bool
618 23846x reactor_scheduler::running_in_this_thread() const noexcept
619 {
620 23846x return reactor_find_context(this) != nullptr;
621 }
622
623 inline void
624 8461x reactor_scheduler::stop()
625 {
626 8461x lock_type lock(mutex_);
627 8461x if (!stopped_.load(std::memory_order_acquire))
628 {
629 7437x stopped_.store(true, std::memory_order_release);
630 7437x signal_all(lock);
631 7437x interrupt_reactor();
632 }
633 8461x }
634
635 inline bool
636 1359x reactor_scheduler::stopped() const noexcept
637 {
638 1359x return stopped_.load(std::memory_order_acquire);
639 }
640
641 inline void
642 7668x reactor_scheduler::restart()
643 {
644 7668x stopped_.store(false, std::memory_order_release);
645 7668x }
646
647 inline std::size_t
648 6487x reactor_scheduler::run()
649 {
650 12974x if (outstanding_work_.load(std::memory_order_acquire) == 0)
651 {
652 216x stop();
653 216x return 0;
654 }
655
656 6271x reactor_thread_context_guard ctx(this);
657 6271x lock_type lock(mutex_);
658 6271x adopt_enclosing_work(ctx.frame_);
659
660 6271x std::size_t n = 0;
661 for (;;)
662 {
663 379478x if (!do_one(lock, -1, ctx.frame_))
664 6266x break;
665 373207x if (n != (std::numeric_limits<std::size_t>::max)())
666 373207x ++n;
667 373207x if (!lock.owns_lock())
668 74073x lock.lock();
669 }
670 6266x return n;
671 6276x }
672
673 inline std::size_t
674 146x reactor_scheduler::run_one()
675 {
676 292x if (outstanding_work_.load(std::memory_order_acquire) == 0)
677 {
678 5x stop();
679 5x return 0;
680 }
681
682 141x reactor_thread_context_guard ctx(this);
683 141x lock_type lock(mutex_);
684 141x adopt_enclosing_work(ctx.frame_);
685 141x return do_one(lock, -1, ctx.frame_);
686 143x }
687
688 inline std::size_t
689 3809x reactor_scheduler::wait_one(long usec)
690 {
691 7618x if (outstanding_work_.load(std::memory_order_acquire) == 0)
692 {
693 834x stop();
694 834x return 0;
695 }
696
697 2975x reactor_thread_context_guard ctx(this);
698 2975x lock_type lock(mutex_);
699 2975x adopt_enclosing_work(ctx.frame_);
700 2975x return do_one(lock, usec, ctx.frame_);
701 2983x }
702
703 inline std::size_t
704 1479x reactor_scheduler::poll()
705 {
706 2958x if (outstanding_work_.load(std::memory_order_acquire) == 0)
707 {
708 31x stop();
709 31x return 0;
710 }
711
712 1448x reactor_thread_context_guard ctx(this);
713 1448x lock_type lock(mutex_);
714 1448x adopt_enclosing_work(ctx.frame_);
715
716 1448x std::size_t n = 0;
717 for (;;)
718 {
719 9628x if (!do_one(lock, 0, ctx.frame_))
720 1446x break;
721 8180x if (n != (std::numeric_limits<std::size_t>::max)())
722 8180x ++n;
723 8180x if (!lock.owns_lock())
724 8176x lock.lock();
725 }
726 1446x return n;
727 1450x }
728
729 inline std::size_t
730 79x reactor_scheduler::poll_one()
731 {
732 158x if (outstanding_work_.load(std::memory_order_acquire) == 0)
733 {
734 7x stop();
735 7x return 0;
736 }
737
738 72x reactor_thread_context_guard ctx(this);
739 72x lock_type lock(mutex_);
740 72x adopt_enclosing_work(ctx.frame_);
741 72x return do_one(lock, 0, ctx.frame_);
742 74x }
743
744 inline void
745 66968x reactor_scheduler::work_started() noexcept
746 {
747 66968x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
748 66968x }
749
750 inline void
751 112059x reactor_scheduler::work_finished() noexcept
752 {
753 224118x if (outstanding_work_.fetch_sub(1, std::memory_order_acq_rel) == 1)
754 7263x stop();
755 112059x }
756
757 inline void
758 20858x reactor_scheduler::compensating_work_started() const noexcept
759 {
760 20858x auto* ctx = reactor_find_context(this);
761 20858x if (ctx)
762 20856x ++ctx->private_outstanding_work;
763 20858x }
764
765 inline void
766 1988x reactor_scheduler::post_counted(scheduler_op* op) const
767 {
768 1988x if (auto* ctx = reactor_find_context(this))
769 {
770 604x ctx->private_queue.push(op);
771 604x return;
772 }
773
774 1384x lock_type lock(mutex_);
775 1384x completed_ops_.push(op);
776 1384x wake_one_thread_and_unlock(lock);
777 1384x }
778
779 inline void
780 15785x reactor_scheduler::post_deferred_completions(ready_queue& ops) const
781 {
782 15785x if (ops.empty())
783 15785x return;
784
785 2x if (auto* ctx = reactor_find_context(this))
786 {
787 2x ctx->private_queue.splice(ops);
788 2x return;
789 }
790
791 ✗ lock_type lock(mutex_);
792 ✗ completed_ops_.splice(ops);
793 ✗ wake_one_thread_and_unlock(lock);
794 ✗ }
795
796 inline void
797 22866x reactor_scheduler::retire_descriptor(
798 reactor_retirable& d, object_ref keep) const
799 {
800 22866x lock_type lock(mutex_);
801 22866x if (!task_running_.load(std::memory_order_relaxed))
802 {
803 // No pass can hold an event, but an earlier one may have
804 // queued d.
805 17528x d.hand_off(keep);
806 17528x return;
807 }
808 // Already parked for this pass: the held reference suffices.
809 5338x if (d.retired_ref_)
810 92x return;
811 5246x d.retired_ref_ = std::move(keep);
812 5246x d.retired_next_ = retired_;
813 5246x retired_ = &d;
814 22866x }
815
816 inline void
817 10907x reactor_scheduler::adopt_enclosing_work(context_type const& ctx)
818 {
819 // A run nested in a handler would otherwise never see what that
820 // handler posted: it stays in the handler's frame until it returns.
821 10913x for (auto* c = ctx.next; c != nullptr; c = c->next)
822 {
823 12x if (c->key != this)
824 6x continue;
825 6x if (c->private_outstanding_work > 0)
826 {
827 6x outstanding_work_.fetch_add(
828 c->private_outstanding_work, std::memory_order_relaxed);
829 6x c->private_outstanding_work = 0;
830 }
831 6x completed_ops_.splice(c->private_queue);
832 6x return;
833 }
834 }
835
836 inline void
837 177578x reactor_scheduler::settle_retired(lock_type& lock) const
838 {
839 // Handed off under the lock, since a later pass may retire the same
840 // descriptor again as soon as it is released. A reference that is
841 // not handed off may be the owner's last, and its release recycles
842 // through the pool mutex, so it is dropped with the lock released.
843 177578x reactor_retirable* dropped = nullptr;
844 182824x while (auto* d = retired_)
845 {
846 5246x retired_ = d->retired_next_;
847 5246x d->retired_next_ = nullptr;
848 5246x auto keep = std::move(d->retired_ref_);
849 5246x d->hand_off(keep);
850 // A drop already pending keeps the owner alive, so this one
851 // cannot be the last and is released here.
852 5246x if (!keep || d->dropped_ref_)
853 344x continue;
854 4902x d->dropped_ref_ = std::move(keep);
855 4902x d->dropped_next_ = dropped;
856 4902x dropped = d;
857 10492x }
858 182480x while (auto* d = dropped)
859 {
860 4902x dropped = d->dropped_next_;
861 4902x d->dropped_next_ = nullptr;
862 4902x auto keep = std::move(d->dropped_ref_);
863 4902x lock.unlock();
864 4902x keep.reset();
865 4902x lock.lock();
866 4902x }
867 177578x }
868
869 inline void
870 410x reactor_scheduler::on_timer_service_attached(timer_service& svc)
871 {
872 410x svc.set_on_earliest_changed(timer_service::callback(this, [](void* p) {
873 3116x static_cast<reactor_scheduler*>(p)->interrupt_reactor();
874 3116x }));
875 410x }
876
877 inline void
878 4680x reactor_scheduler::shutdown_drain()
879 {
880 4680x wait_cancels_posting();
881
882 4680x lock_type lock(mutex_);
883 4680x settle_retired(lock);
884 4680x destroy_queued(lock);
885 4680x signal_all(lock);
886 4680x }
887
888 4696x inline reactor_scheduler::~reactor_scheduler()
889 {
890 // Services give up their work in shutdown(), but anything posted
891 // after the drain, such as from a destructor, is still destroyed.
892 4696x lock_type lock(mutex_);
893 4696x destroy_queued(lock);
894 4696x }
895
896 inline void
897 9376x reactor_scheduler::destroy_queued(lock_type& lock) noexcept
898 {
899 23532x while (auto e = completed_ops_.pop())
900 {
901 14156x if (ready_is_continuation(e))
902 {
903 7836x lock.unlock();
904 7836x if (auto* c = ready_as_cont(e); c->h)
905 7836x capy::abandon(*c);
906 7836x lock.lock();
907 }
908 else
909 {
910 6320x auto* op = ready_as_op(e);
911 6320x if (op == &task_op_)
912 4677x continue;
913 1643x lock.unlock();
914 1643x op->destroy();
915 1643x lock.lock();
916 }
917 14156x }
918 9376x }
919
920 inline void
921 12117x reactor_scheduler::signal_all(lock_type&) const
922 {
923 12117x state_ |= signaled_bit;
924 12117x cond_.notify_all();
925 12117x }
926
927 inline bool
928 40628x reactor_scheduler::maybe_unlock_and_signal_one(lock_type& lock) const
929 {
930 40628x state_ |= signaled_bit;
931 40628x if (state_ > signaled_bit)
932 {
933 // Signalled before unlocking: a post from another context's
934 // thread may be the last touch of this scheduler before the
935 // posted work runs, finishes and lets it be destroyed.
936 198x cond_.notify_one();
937 198x lock.unlock();
938 198x return true;
939 }
940 40430x return false;
941 }
942
943 inline bool
944 510455x reactor_scheduler::unlock_and_signal_one(lock_type& lock) const
945 {
946 510455x state_ |= signaled_bit;
947 510455x bool have_waiters = state_ > signaled_bit;
948 510455x lock.unlock();
949 510455x if (have_waiters)
950 4400x cond_.notify_one();
951 510455x return have_waiters;
952 }
953
954 inline void
955 792x reactor_scheduler::clear_signal() const
956 {
957 792x state_ &= ~signaled_bit;
958 792x }
959
960 inline void
961 767x reactor_scheduler::wait_for_signal(lock_type& lock) const
962 {
963 1598x while ((state_ & signaled_bit) == 0)
964 {
965 831x state_ += waiter_increment;
966 831x cond_.wait(lock);
967 831x state_ -= waiter_increment;
968 }
969 767x }
970
971 inline void
972 25x reactor_scheduler::wait_for_signal_for(lock_type& lock, long timeout_us) const
973 {
974 25x if ((state_ & signaled_bit) == 0)
975 {
976 25x state_ += waiter_increment;
977 25x cond_.wait_for(lock, std::chrono::microseconds(timeout_us));
978 25x state_ -= waiter_increment;
979 }
980 25x }
981
982 inline void
983 39144x reactor_scheduler::wake_one_thread_and_unlock(lock_type& lock) const
984 {
985 39144x if (maybe_unlock_and_signal_one(lock))
986 178x return;
987
988 38966x if (task_running_.load(std::memory_order_relaxed) && !task_interrupted_)
989 {
990 1674x task_interrupted_ = true;
991 // Under the lock, for the same reason as the signal above.
992 1674x interrupt_reactor();
993 1674x lock.unlock();
994 }
995 else
996 {
997 37292x lock.unlock();
998 }
999 }
1000
1001 384288x inline reactor_scheduler::work_cleanup::~work_cleanup()
1002 {
1003 384288x std::int64_t produced = ctx.private_outstanding_work;
1004 384288x if (produced > 1)
1005 955x sched->outstanding_work_.fetch_add(
1006 produced - 1, std::memory_order_relaxed);
1007 383333x else if (produced < 1)
1008 65272x sched->work_finished();
1009 384288x ctx.private_outstanding_work = 0;
1010
1011 384288x if (!ctx.private_queue.empty())
1012 {
1013 299470x lock->lock();
1014 299470x sched->completed_ops_.splice(ctx.private_queue);
1015 }
1016 384288x }
1017
1018 172901x inline reactor_scheduler::task_cleanup::~task_cleanup()
1019 {
1020 172901x if (ctx.private_outstanding_work > 0)
1021 {
1022 10229x sched->outstanding_work_.fetch_add(
1023 10229x ctx.private_outstanding_work, std::memory_order_relaxed);
1024 10229x ctx.private_outstanding_work = 0;
1025 }
1026
1027 172901x if (!ctx.private_queue.empty())
1028 {
1029 10229x if (!lock->owns_lock())
1030 ✗ lock->lock();
1031 10229x sched->completed_ops_.splice(ctx.private_queue);
1032 }
1033 172901x }
1034
1035 inline std::size_t
1036 392294x reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
1037 {
1038 for (;;)
1039 {
1040 693928x if (stopped_.load(std::memory_order_acquire))
1041 {
1042 // The next run is scored afresh, but a deferral left pending
1043 // stays marked: clearing the mark would let a loop of short
1044 // runs defer the sentinel forever.
1045 6504x defer_policy_.reset();
1046 6504x return 0;
1047 }
1048
1049 687424x auto const e = completed_ops_.pop();
1050 687424x scheduler_op* op = ready_is_continuation(e) ? nullptr : ready_as_op(e);
1051
1052 // Handle reactor sentinel — time to poll for I/O
1053 687424x if (op == &task_op_)
1054 {
1055 302335x bool more_handlers = !completed_ops_.empty();
1056
1057 // Handlers queued here usually start the next I/O
1058 // themselves, so running them first can save a reactor pass.
1059 // Defer at most once in a row so reposting work cannot
1060 // starve I/O. poll() and poll_one() don't defer: the reactor
1061 // turn they would skip may be their last before returning.
1062 302335x if (task_deferred_)
1063 {
1064 128968x task_deferred_ = false;
1065 128968x defer_policy_.on_deferred_turn(more_handlers);
1066 }
1067 173367x else if (more_handlers && timeout_us != 0)
1068 {
1069 131431x if (defer_policy_.may_defer())
1070 {
1071 129428x task_deferred_ = true;
1072 129428x completed_ops_.push(&task_op_);
1073 129428x continue;
1074 }
1075 2003x defer_policy_.on_polled_instead();
1076 }
1077
1078 218472x if (!more_handlers &&
1079 91130x outstanding_work_.load(std::memory_order_acquire) == 0)
1080 {
1081 6x completed_ops_.push(&task_op_);
1082 6x return 0;
1083 }
1084
1085 172901x long task_timeout_us = more_handlers ? 0 : timeout_us;
1086 172901x task_interrupted_ = task_timeout_us == 0;
1087 172901x task_running_.store(true, std::memory_order_release);
1088
1089 // Wake a peer to take the pending handlers while this thread
1090 // polls the reactor; skipped when one_thread_ (no peer exists).
1091 172901x if (more_handlers && !one_thread_)
1092 127340x unlock_and_signal_one(lock);
1093
1094 try
1095 {
1096 172901x run_task(lock, ctx, task_timeout_us);
1097 }
1098 3x catch (...)
1099 {
1100 3x task_running_.store(false, std::memory_order_relaxed);
1101 3x throw;
1102 3x }
1103
1104 172898x task_running_.store(false, std::memory_order_relaxed);
1105 172898x bool const idle = completed_ops_.empty();
1106 172898x completed_ops_.push(&task_op_);
1107 172898x settle_retired(lock);
1108 // A bounded call blocks at most once, then finishes as a poll
1109 // does, running what its pass queued.
1110 172898x if (timeout_us > 0)
1111 2396x timeout_us = 0;
1112 // A poll ends once a pass leaves nothing queued; going round
1113 // again would only repeat the pass. A peer that found the
1114 // queue empty during the pass may be asleep, so it is woken
1115 // to take the sentinel.
1116 172898x if (timeout_us == 0 && idle)
1117 {
1118 1484x maybe_unlock_and_signal_one(lock);
1119 1484x return 0;
1120 }
1121 171414x continue;
1122 171414x }
1123
1124 // Handle ready entry (op or continuation)
1125 385089x if (e != nullptr)
1126 {
1127 384288x bool more = !completed_ops_.empty();
1128
1129 384288x if (more && !one_thread_)
1130 {
1131 // Wake a peer for the remaining work; unassisted if none
1132 // was parked to take it.
1133 383115x ctx.unassisted = !unlock_and_signal_one(lock);
1134 }
1135 else
1136 {
1137 // No peer to wake (one_thread_, or nothing more queued).
1138 1173x ctx.unassisted = more;
1139 1173x lock.unlock();
1140 }
1141
1142 384288x [[maybe_unused]] work_cleanup on_exit{this, &lock, ctx};
1143
1144 384288x if (ready_is_continuation(e))
1145 230722x ready_as_cont(e)->h.resume();
1146 else
1147 153566x (*op)();
1148 384272x return 1;
1149 384288x }
1150
1151 1602x if (outstanding_work_.load(std::memory_order_acquire) == 0 ||
1152 timeout_us == 0)
1153 9x return 0;
1154
1155 792x clear_signal();
1156 792x if (timeout_us < 0)
1157 {
1158 767x wait_for_signal(lock);
1159 }
1160 else
1161 {
1162 25x wait_for_signal_for(lock, timeout_us);
1163 // A bounded call blocks once, so its caller can check its
1164 // deadline; another thread may hold the reactor for good.
1165 25x timeout_us = 0;
1166 }
1167 301634x }
1168 }
1169
1170 } // namespace boost::corosio::detail
1171
1172 #endif // BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
1173