include/boost/corosio/native/detail/reactor/reactor_scheduler.hpp

98.4% Lines (379 / 385, 2 excl) 100.0% Functions (51 / 51, 2 excl)
reactor_scheduler.hpp
f(x) Functions (53)
Function Calls Lines Blocks
boost::corosio::detail::reactor_retirable::~reactor_retirable() :68 12310x 100.0% 100.0% boost::corosio::detail::reactor_find_context(boost::corosio::detail::reactor_scheduler const*) :110 889995x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::notify_reparked() const :226 278x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::inline_budget_initial() const :276 13116x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::scheduler_locking_disabled() const :282 1460x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::configure_threading(boost::corosio::detail::scheduler::threading_config) :287 4504x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::reactor_scheduler() :303 4520x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::task_op::operator()() :364 – – – boost::corosio::detail::reactor_scheduler::task_op::destroy() :365 – – – boost::corosio::detail::reactor_thread_context_guard::reactor_thread_context_guard(boost::corosio::detail::reactor_scheduler const*) :422 13116x 100.0% 100.0% boost::corosio::detail::reactor_thread_context_guard::~reactor_thread_context_guard() :436 13116x 100.0% 100.0% boost::corosio::detail::reactor_scheduler_context::reactor_scheduler_context(boost::corosio::detail::reactor_scheduler const*, boost::corosio::detail::reactor_scheduler_context*) :444 13116x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::configure_reactor(unsigned int, unsigned int, unsigned int, unsigned int) :456 50x 100.0% 85.0% boost::corosio::detail::reactor_scheduler::reset_inline_budget() const :481 95526x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::try_consume_inline_budget() const :515 394722x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const :531 3678x 100.0% 84.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::post_handler(std::__n4861::coroutine_handle<void>) :537 3678x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::~post_handler() :538 7356x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::operator()() :540 3662x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(std::__n4861::coroutine_handle<void>) const::post_handler::destroy() :547 16x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post(boost::corosio::detail::scheduler_op*) const :574 104016x 100.0% 87.0% boost::corosio::detail::reactor_scheduler::post(boost::capy::continuation&) const :591 236668x 100.0% 87.0% boost::corosio::detail::reactor_scheduler::running_in_this_thread() const :608 23787x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::stop() :614 8327x 100.0% 82.0% boost::corosio::detail::reactor_scheduler::stopped() const :626 3539x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::restart() :632 5718x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::run() :638 6508x 100.0% 86.0% boost::corosio::detail::reactor_scheduler::run_one() :663 128x 100.0% 70.0% boost::corosio::detail::reactor_scheduler::wait_one(long) :677 6132x 100.0% 70.0% boost::corosio::detail::reactor_scheduler::poll() :691 1339x 100.0% 76.0% boost::corosio::detail::reactor_scheduler::poll_one() :716 60x 100.0% 70.0% boost::corosio::detail::reactor_scheduler::work_started() :730 67361x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::work_finished() :736 112736x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::compensating_work_started() const :743 29714x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::post_counted(boost::corosio::detail::scheduler_op*) const :751 1984x 100.0% 85.0% boost::corosio::detail::reactor_scheduler::post_deferred_completions(boost::corosio::detail::ready_queue&) const :765 15663x 60.0% 59.0% boost::corosio::detail::reactor_scheduler::retire_descriptor(boost::corosio::detail::reactor_retirable&, boost::corosio::detail::object_ref) const :782 22916x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::settle_retired(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :802 335286x 100.0% 94.0% boost::corosio::detail::reactor_scheduler::on_timer_service_attached(boost::corosio::detail::timer_service&) :835 396x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::on_timer_service_attached(boost::corosio::detail::timer_service&)::{lambda(void*)#1}::operator()(void*) const :837 3100x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::shutdown_drain() :843 4504x 100.0% 80.0% boost::corosio::detail::reactor_scheduler::~reactor_scheduler() :853 4520x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::destroy_queued(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) :862 9024x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::signal_all(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :886 11842x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::maybe_unlock_and_signal_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :893 39674x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::unlock_and_signal_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :909 629211x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::clear_signal() const :920 769x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wait_for_signal(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :926 758x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wait_for_signal_for(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&, long) const :937 11x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::wake_one_thread_and_unlock(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&) const :948 39674x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::work_cleanup::~work_cleanup() :966 381857x 100.0% 100.0% boost::corosio::detail::reactor_scheduler::task_cleanup::~task_cleanup() :983 330785x 90.0% 91.0% boost::corosio::detail::reactor_scheduler::do_one(boost::corosio::detail::conditionally_enabled_mutex::scoped_lock&, long, boost::corosio::detail::reactor_scheduler_context&) :1001 392128x 97.9% 84.0%
Line TLA Hits Source Code
1 //
2 // Copyright (c) 2026 Steve Gerbino
3 //
4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6 //
7 // Official repository: https://github.com/cppalliance/corosio
8 //
9
10 #ifndef BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
11 #define BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
12
13 #include <boost/corosio/detail/config.hpp>
14 #include <boost/corosio/detail/object_ref.hpp>
15 #include <boost/capy/ex/execution_context.hpp>
16 #include <boost/capy/ex/abandon.hpp>
17
18 #include <boost/corosio/detail/ready_queue.hpp>
19 #include <boost/corosio/detail/scheduler.hpp>
20 #include <boost/corosio/detail/scheduler_op.hpp>
21 #include <boost/corosio/detail/thread_local_ptr.hpp>
22 #include <boost/corosio/detail/timer_service.hpp>
23
24 #include <atomic>
25 #include <chrono>
26 #include <coroutine>
27 #include <cstddef>
28 #include <cstdint>
29 #include <limits>
30 #include <memory>
31
32 #include <boost/corosio/detail/conditionally_enabled_mutex.hpp>
33 #include <boost/corosio/detail/conditionally_enabled_event.hpp>
34
35 namespace boost::corosio::detail {
36
37 // Forward declarations
38 class reactor_scheduler;
39
40 /** A deregistered descriptor whose owner must outlive the reactor pass
41 that may still hold it.
42
43 An event the reactor has already read from the kernel can name a
44 descriptor that another thread closes before the reactor queues it.
45 The closing thread hands over a reference instead of dropping it, so
46 the owner is neither freed nor recycled until that pass ends.
47 */
48 struct reactor_retirable
49 {
50 // All four fields are guarded by the scheduler's mutex.
51
52 /// Next in the scheduler's retired list.
53 reactor_retirable* retired_next_ = nullptr;
54
55 /// Owner reference held until the pass ends.
56 object_ref retired_ref_;
57
58 /// Next in a settling thread's list of references to drop.
59 reactor_retirable* dropped_next_ = nullptr;
60
61 /// Owner reference waiting to be dropped outside the mutex.
62 object_ref dropped_ref_;
63
64 /// Move @p keep into a queued completion, if one is pending.
65 virtual void hand_off(object_ref& keep) noexcept = 0;
66
67 protected:
68 12310x ~reactor_retirable() = default;
69 };
70
71 /** Per-thread state for a reactor scheduler.
72
73 Each thread running a scheduler's event loop has one of these
74 on a thread-local stack. It holds a private work queue and
75 inline completion budget for speculative I/O fast paths.
76 */
77 struct BOOST_COROSIO_SYMBOL_VISIBLE reactor_scheduler_context
78 {
79 /// Scheduler this context belongs to.
80 reactor_scheduler const* key;
81
82 /// Next context frame on this thread's stack.
83 reactor_scheduler_context* next;
84
85 /// Private work queue for reduced contention.
86 ready_queue private_queue;
87
88 /// Unflushed work count for the private queue.
89 std::int64_t private_outstanding_work;
90
91 /// Remaining inline completions allowed this cycle.
92 int inline_budget;
93
94 /// Maximum inline budget (adaptive, 2-16).
95 int inline_budget_max;
96
97 /// True if no other thread absorbed queued work last cycle.
98 bool unassisted;
99
100 /// Construct a context frame linked to @a n.
101 reactor_scheduler_context(
102 reactor_scheduler const* k, reactor_scheduler_context* n);
103 };
104
105 /// Thread-local context stack for reactor schedulers.
106 inline thread_local_ptr<reactor_scheduler_context> reactor_context_stack;
107
108 /// Find the context frame for a scheduler on this thread.
109 inline reactor_scheduler_context*
110 889995x reactor_find_context(reactor_scheduler const* self) noexcept
111 {
112 890564x for (auto* c = reactor_context_stack.get(); c != nullptr; c = c->next)
113 {
114 827090x if (c->key == self)
115 826521x return c;
116 }
117 63474x return nullptr;
118 }
119
120 /** Non-template base for reactor-backed scheduler implementations.
121
122 Provides the complete threading model shared by epoll, kqueue,
123 and select schedulers: signal state machine, inline completion
124 budget, work counting, run/poll methods, and the do_one event
125 loop.
126
127 Derived classes provide platform-specific hooks by overriding:
128 - `run_task(lock, ctx)` to run the reactor poll
129 - `interrupt_reactor()` to wake a blocked reactor
130
131 De-templated from the original CRTP design to eliminate
132 duplicate instantiations when multiple backends are compiled
133 into the same binary. Virtual dispatch for run_task (called
134 once per reactor cycle, before a blocking syscall) has
135 negligible overhead.
136
137 @par Thread Safety
138 All public member functions are thread-safe.
139 */
140 class reactor_scheduler : public scheduler
141 {
142 public:
143 using context_type = reactor_scheduler_context;
144 using mutex_type = conditionally_enabled_mutex;
145 using lock_type = mutex_type::scoped_lock;
146 using event_type = conditionally_enabled_event;
147
148 /// Post a coroutine for deferred execution.
149 void post(std::coroutine_handle<> h) const override;
150
151 /// Post a scheduler operation for deferred execution.
152 void post(scheduler_op* h) const override;
153
154 /** Release an owner reference once no reactor pass can hold `d`.
155
156 Called after a descriptor is deregistered. If a reactor pass is
157 running, `keep` is parked on `d` until that pass ends. Then, or
158 at once if no pass is running, it moves to `d`'s queued
159 invocation if `d` is queued, and is dropped otherwise.
160
161 @param d The deregistered descriptor.
162 @param keep A reference to the descriptor's owner.
163 */
164 void retire_descriptor(reactor_retirable& d, object_ref keep) const;
165
166 /// Post a continuation for deferred execution.
167 void post(capy::continuation&) const override;
168
169 /// Return true if called from a thread running this scheduler.
170 bool running_in_this_thread() const noexcept override;
171
172 /// Request the scheduler to stop dispatching handlers.
173 void stop() override;
174
175 /// Return true if the scheduler has been stopped.
176 bool stopped() const noexcept override;
177
178 /// Reset the stopped state so `run()` can resume.
179 void restart() override;
180
181 /// Run the event loop until no work remains.
182 std::size_t run() override;
183
184 /// Run until one handler completes or no work remains.
185 std::size_t run_one() override;
186
187 /// Run until one handler completes or @a usec elapses.
188 std::size_t wait_one(long usec) override;
189
190 /// Run ready handlers without blocking.
191 std::size_t poll() override;
192
193 /// Run at most one ready handler without blocking.
194 std::size_t poll_one() override;
195
196 /// Increment the outstanding work count.
197 void work_started() noexcept override;
198
199 /// Decrement the outstanding work count, stopping on zero.
200 void work_finished() noexcept override;
201
202 /** Reset the thread's inline completion budget.
203
204 Called at the start of each posted completion handler to
205 grant a fresh budget for speculative inline completions.
206 */
207 void reset_inline_budget() const noexcept;
208
209 /** Consume one unit of inline budget if available.
210
211 @return True if budget was available and consumed.
212 */
213 bool try_consume_inline_budget() const noexcept;
214
215 /** Offset a forthcoming work_finished from work_cleanup.
216
217 Called by descriptor_state when all I/O returned EAGAIN and
218 no handler will be executed. Must be called from a scheduler
219 thread.
220 */
221 /** Note that an op was parked outside the usual registration path.
222
223 A reactor that builds its poll set from parked ops must rebuild
224 it; one the kernel tracks needs nothing.
225 */
226 278x virtual void notify_reparked() const {}
227
228 void compensating_work_started() const noexcept;
229
230 /** Post an operation whose work is already counted.
231
232 The posting thread touches nothing of the scheduler after this
233 returns, unlike a @ref post followed by @ref work_finished: once
234 posted, the operation can complete and let its context be
235 destroyed.
236
237 @pre work_started() was called for @p op.
238
239 @param op The operation to post.
240 */
241 void post_counted(scheduler_op* op) const;
242
243 /** Post completed operations for deferred invocation.
244
245 If called from a thread running this scheduler, operations
246 go to the thread's private queue (fast path). Otherwise,
247 operations are added to the global queue under mutex and a
248 waiter is signaled.
249
250 @pre work_started() must have been called for each operation.
251
252 @param ops Queue of operations to post.
253 */
254 void post_deferred_completions(ready_queue& ops) const;
255
256 /** Apply runtime configuration to the scheduler.
257
258 Called by `io_context` after construction. Values that do
259 not apply to this backend are silently ignored.
260
261 @pre `max_events` is in [ 1, `INT_MAX` ] and `budget_max` is at
262 most `INT_MAX`; `io_context` validates both.
263
264 @param max_events Event buffer size for epoll/kqueue.
265 @param budget_init Starting inline completion budget.
266 @param budget_max Hard ceiling on adaptive budget ramp-up.
267 @param unassisted Budget when single-threaded.
268 */
269 virtual void configure_reactor(
270 unsigned max_events,
271 unsigned budget_init,
272 unsigned budget_max,
273 unsigned unassisted);
274
275 /// Return the configured initial inline budget.
276 13116x unsigned inline_budget_initial() const noexcept
277 {
278 13116x return inline_budget_initial_;
279 }
280
281 /// Return true when scheduler locking is disabled (fully-lockless tier).
282 1460x bool scheduler_locking_disabled() const noexcept override
283 {
284 1460x return scheduler_locking_disabled_;
285 }
286
287 4504x void configure_threading(threading_config cfg) noexcept override
288 {
289 4504x scheduler_locking_disabled_ = !cfg.scheduler_locking;
290 // reactor_io_locking takes effect at descriptor registration (see the
291 // register_descriptor overrides), not here.
292 4504x reactor_io_locking_ = cfg.reactor_io_locking;
293 4504x one_thread_ = cfg.one_thread;
294 4504x mutex_.set_enabled(cfg.scheduler_locking);
295 4504x cond_.set_enabled(cfg.scheduler_locking);
296 4504x }
297
298 protected:
299 bool scheduler_locking_disabled_ = false;
300 bool reactor_io_locking_ = true;
301 bool one_thread_ = false;
302
303 4520x reactor_scheduler() = default;
304
305 /// Destroy, destroying whatever was posted after the drain.
306 ~reactor_scheduler() override;
307
308 /** Drain completed_ops during shutdown.
309
310 Pops all operations from the global queue and destroys them,
311 skipping the task sentinel. Signals all waiting threads.
312 Derived classes call this from their shutdown() override
313 before performing platform-specific cleanup.
314 */
315 void shutdown_drain();
316
317 /// Destroy every queued operation without running it.
318 void destroy_queued(lock_type& lock) noexcept;
319
320 /// RAII guard that re-inserts the task sentinel after `run_task`.
321 struct task_cleanup
322 {
323 reactor_scheduler const* sched;
324 lock_type* lock;
325 context_type& ctx;
326 ~task_cleanup();
327 };
328
329 mutable mutex_type mutex_{true};
330 mutable event_type cond_{true};
331 mutable ready_queue completed_ops_;
332 mutable std::atomic<std::int64_t> outstanding_work_{0};
333 std::atomic<bool> stopped_{false};
334 mutable std::atomic<bool> task_running_{false};
335 mutable bool task_interrupted_ = false;
336
337 // Descriptors retired during the running reactor pass (guarded by
338 // mutex_).
339 mutable reactor_retirable* retired_ = nullptr;
340
341 /// Settle every descriptor retired before the current pass ended.
342 /// @pre `lock` holds `mutex_`; it is held again on return.
343 void settle_retired(lock_type& lock) const;
344
345 // Runtime-configurable reactor tuning parameters.
346 // Defaults match the library's built-in values.
347 unsigned max_events_per_poll_ = 128;
348 unsigned inline_budget_initial_ = 2;
349 unsigned inline_budget_max_ = 16;
350 unsigned unassisted_budget_ = 4;
351
352 /// Bit 0 of `state_`: set when the condvar should be signaled.
353 static constexpr std::size_t signaled_bit = 1;
354
355 /// Increment per waiting thread in `state_`.
356 static constexpr std::size_t waiter_increment = 2;
357 mutable std::size_t state_ = 0;
358
359 /// Sentinel op that triggers a reactor poll when dequeued.
360 struct task_op final : scheduler_op
361 {
362 // LCOV_EXCL_START: the sentinel is intercepted by pointer
363 // identity; its virtuals exist for vtable completeness.
364 − void operator()() override {}
365 − void destroy() override {}
366 // LCOV_EXCL_STOP
367 };
368 task_op task_op_;
369
370 /** Run the platform-specific reactor poll.
371
372 @par Postconditions
373 `lock` is owned on return, however the poll ended. An
374 implementation that unlocks around the blocking call owes the
375 caller a matching re-acquire on every path out, including the
376 errors it retries rather than reports.
377 */
378 virtual void
379 run_task(lock_type& lock, context_type& ctx, long timeout_us) = 0;
380
381 /// Wake a blocked reactor (e.g. write to eventfd or pipe).
382 virtual void interrupt_reactor() const = 0;
383
384 /// Wake the reactor whenever the earliest timer changes.
385 void on_timer_service_attached(timer_service& svc) override;
386
387 private:
388 struct work_cleanup
389 {
390 reactor_scheduler* sched;
391 lock_type* lock;
392 context_type& ctx;
393 ~work_cleanup();
394 };
395
396 std::size_t do_one(lock_type& lock, long timeout_us, context_type& ctx);
397
398 void signal_all(lock_type& lock) const;
399 bool maybe_unlock_and_signal_one(lock_type& lock) const;
400 bool unlock_and_signal_one(lock_type& lock) const;
401 void clear_signal() const;
402 void wait_for_signal(lock_type& lock) const;
403 void wait_for_signal_for(lock_type& lock, long timeout_us) const;
404 void wake_one_thread_and_unlock(lock_type& lock) const;
405 };
406
407 /** RAII guard that pushes/pops a scheduler context frame.
408
409 On construction, pushes a new context frame onto the
410 thread-local stack. On destruction, drains any remaining
411 private queue items to the global queue and pops the frame.
412 */
413 struct reactor_thread_context_guard
414 {
415 /// The context frame managed by this guard.
416 reactor_scheduler_context frame_;
417
418 /// Marks the scheduler innermost for the executor's fast check.
419 running_scheduler_guard running_;
420
421 /// Construct the guard, pushing a frame for @a sched.
422 13116x explicit reactor_thread_context_guard(
423 reactor_scheduler const* sched) noexcept
424 13116x : frame_(sched, reactor_context_stack.get())
425 13116x , running_(sched)
426 {
427 13116x reactor_context_stack.set(&frame_);
428 13116x }
429
430 /** Destroy the guard, popping the frame.
431
432 The private queue is empty here by invariant: work_cleanup and
433 task_cleanup splice it to the global queue after every handler
434 and every reactor pass.
435 */
436 13116x ~reactor_thread_context_guard() noexcept
437 {
438 13116x reactor_context_stack.set(frame_.next);
439 13116x }
440 };
441
442 // ---- Inline implementations ------------------------------------------------
443
444 13116x inline reactor_scheduler_context::reactor_scheduler_context(
445 13116x reactor_scheduler const* k, reactor_scheduler_context* n)
446 13116x : key(k)
447 13116x , next(n)
448 13116x , private_outstanding_work(0)
449 13116x , inline_budget(0)
450 13116x , inline_budget_max(static_cast<int>(k->inline_budget_initial()))
451 13116x , unassisted(false)
452 {
453 13116x }
454
455 inline void
456 50x reactor_scheduler::configure_reactor(
457 unsigned max_events,
458 unsigned budget_init,
459 unsigned budget_max,
460 unsigned unassisted)
461 {
462 50x BOOST_COROSIO_ASSERT(
463 max_events >= 1 &&
464 max_events <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
465 50x BOOST_COROSIO_ASSERT(
466 budget_max <= static_cast<unsigned>((std::numeric_limits<int>::max)()));
467
468 // Clamp initial and unassisted to budget_max.
469 50x if (budget_init > budget_max)
470 20x budget_init = budget_max;
471 50x if (unassisted > budget_max)
472 20x unassisted = budget_max;
473
474 50x max_events_per_poll_ = max_events;
475 50x inline_budget_initial_ = budget_init;
476 50x inline_budget_max_ = budget_max;
477 50x unassisted_budget_ = unassisted;
478 50x }
479
480 inline void
481 95526x reactor_scheduler::reset_inline_budget() const noexcept
482 {
483 // When budget is disabled (max==0), all paths below would no-op
484 // (inline_budget stays 0). Skip the TLS lookup entirely.
485 95526x if (inline_budget_max_ == 0)
486 58x return;
487 95468x if (auto* ctx = reactor_find_context(this))
488 {
489 // Cap when no other thread absorbed queued work
490 95468x if (ctx->unassisted)
491 {
492 94360x ctx->inline_budget_max = static_cast<int>(unassisted_budget_);
493 94360x ctx->inline_budget = static_cast<int>(unassisted_budget_);
494 94360x return;
495 }
496 // Ramp up when previous cycle fully consumed budget.
497 // max(1, ...) ensures the doubling escapes zero.
498 // Doubled in a wider type: the cap may be as large as INT_MAX.
499 1108x if (ctx->inline_budget == 0)
500 {
501 long long const doubled =
502 321x 2LL * (std::max)(1, ctx->inline_budget_max);
503 321x long long const cap = inline_budget_max_;
504 321x ctx->inline_budget_max = static_cast<int>((std::min)(doubled, cap));
505 }
506 787x else if (ctx->inline_budget < ctx->inline_budget_max)
507 {
508 254x ctx->inline_budget_max = static_cast<int>(inline_budget_initial_);
509 }
510 1108x ctx->inline_budget = ctx->inline_budget_max;
511 }
512 }
513
514 inline bool
515 394722x reactor_scheduler::try_consume_inline_budget() const noexcept
516 {
517 394722x if (inline_budget_max_ == 0)
518 44x return false;
519 394678x if (auto* ctx = reactor_find_context(this))
520 {
521 394667x if (ctx->inline_budget > 0)
522 {
523 315489x --ctx->inline_budget;
524 315489x return true;
525 }
526 }
527 79189x return false;
528 }
529
530 inline void
531 3678x reactor_scheduler::post(std::coroutine_handle<> h) const
532 {
533 struct post_handler final : scheduler_op
534 {
535 std::coroutine_handle<> h_;
536
537 3678x explicit post_handler(std::coroutine_handle<> h) : h_(h) {}
538 7356x ~post_handler() override = default;
539
540 3662x void operator()() override
541 {
542 3662x auto saved = h_;
543 3662x delete this;
544 3662x saved.resume();
545 3662x }
546
547 16x void destroy() override
548 {
549 // A bare handle carries no owner to unwind from: it is
550 // destroyed directly.
551 16x auto saved = h_;
552 16x delete this;
553 16x saved.destroy();
554 16x }
555 };
556
557 3678x auto ph = std::make_unique<post_handler>(h);
558
559 3678x if (auto* ctx = reactor_find_context(this))
560 {
561 12x ++ctx->private_outstanding_work;
562 12x ctx->private_queue.push(ph.release());
563 12x return;
564 }
565
566 3666x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
567
568 3666x lock_type lock(mutex_);
569 3666x completed_ops_.push(ph.release());
570 3666x wake_one_thread_and_unlock(lock);
571 3678x }
572
573 inline void
574 104016x reactor_scheduler::post(scheduler_op* h) const
575 {
576 104016x if (auto* ctx = reactor_find_context(this))
577 {
578 101633x ++ctx->private_outstanding_work;
579 101633x ctx->private_queue.push(h);
580 101633x return;
581 }
582
583 2383x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
584
585 2383x lock_type lock(mutex_);
586 2383x completed_ops_.push(h);
587 2383x wake_one_thread_and_unlock(lock);
588 2383x }
589
590 inline void
591 236668x reactor_scheduler::post(capy::continuation& c) const
592 {
593 236668x if (auto* ctx = reactor_find_context(this))
594 {
595 204419x ++ctx->private_outstanding_work;
596 204419x ctx->private_queue.push(c);
597 204419x return;
598 }
599
600 32249x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
601
602 32249x lock_type lock(mutex_);
603 32249x completed_ops_.push(c);
604 32249x wake_one_thread_and_unlock(lock);
605 32249x }
606
607 inline bool
608 23787x reactor_scheduler::running_in_this_thread() const noexcept
609 {
610 23787x return reactor_find_context(this) != nullptr;
611 }
612
613 inline void
614 8327x reactor_scheduler::stop()
615 {
616 8327x lock_type lock(mutex_);
617 8327x if (!stopped_.load(std::memory_order_acquire))
618 {
619 7338x stopped_.store(true, std::memory_order_release);
620 7338x signal_all(lock);
621 7338x interrupt_reactor();
622 }
623 8327x }
624
625 inline bool
626 3539x reactor_scheduler::stopped() const noexcept
627 {
628 3539x return stopped_.load(std::memory_order_acquire);
629 }
630
631 inline void
632 5718x reactor_scheduler::restart()
633 {
634 5718x stopped_.store(false, std::memory_order_release);
635 5718x }
636
637 inline std::size_t
638 6508x reactor_scheduler::run()
639 {
640 13016x if (outstanding_work_.load(std::memory_order_acquire) == 0)
641 {
642 205x stop();
643 205x return 0;
644 }
645
646 6303x reactor_thread_context_guard ctx(this);
647 6303x lock_type lock(mutex_);
648
649 6303x std::size_t n = 0;
650 for (;;)
651 {
652 377309x if (!do_one(lock, -1, ctx.frame_))
653 6300x break;
654 371006x if (n != (std::numeric_limits<std::size_t>::max)())
655 371006x ++n;
656 371006x if (!lock.owns_lock())
657 83599x lock.lock();
658 }
659 6300x return n;
660 6306x }
661
662 inline std::size_t
663 128x reactor_scheduler::run_one()
664 {
665 256x if (outstanding_work_.load(std::memory_order_acquire) == 0)
666 {
667 3x stop();
668 3x return 0;
669 }
670
671 125x reactor_thread_context_guard ctx(this);
672 125x lock_type lock(mutex_);
673 125x return do_one(lock, -1, ctx.frame_);
674 125x }
675
676 inline std::size_t
677 6132x reactor_scheduler::wait_one(long usec)
678 {
679 12264x if (outstanding_work_.load(std::memory_order_acquire) == 0)
680 {
681 809x stop();
682 809x return 0;
683 }
684
685 5323x reactor_thread_context_guard ctx(this);
686 5323x lock_type lock(mutex_);
687 5323x return do_one(lock, usec, ctx.frame_);
688 5323x }
689
690 inline std::size_t
691 1339x reactor_scheduler::poll()
692 {
693 2678x if (outstanding_work_.load(std::memory_order_acquire) == 0)
694 {
695 29x stop();
696 29x return 0;
697 }
698
699 1310x reactor_thread_context_guard ctx(this);
700 1310x lock_type lock(mutex_);
701
702 1310x std::size_t n = 0;
703 for (;;)
704 {
705 9316x if (!do_one(lock, 0, ctx.frame_))
706 1310x break;
707 8006x if (n != (std::numeric_limits<std::size_t>::max)())
708 8006x ++n;
709 8006x if (!lock.owns_lock())
710 8004x lock.lock();
711 }
712 1310x return n;
713 1310x }
714
715 inline std::size_t
716 60x reactor_scheduler::poll_one()
717 {
718 120x if (outstanding_work_.load(std::memory_order_acquire) == 0)
719 {
720 5x stop();
721 5x return 0;
722 }
723
724 55x reactor_thread_context_guard ctx(this);
725 55x lock_type lock(mutex_);
726 55x return do_one(lock, 0, ctx.frame_);
727 55x }
728
729 inline void
730 67361x reactor_scheduler::work_started() noexcept
731 {
732 67361x outstanding_work_.fetch_add(1, std::memory_order_relaxed);
733 67361x }
734
735 inline void
736 112736x reactor_scheduler::work_finished() noexcept
737 {
738 225472x if (outstanding_work_.fetch_sub(1, std::memory_order_acq_rel) == 1)
739 7211x stop();
740 112736x }
741
742 inline void
743 29714x reactor_scheduler::compensating_work_started() const noexcept
744 {
745 29714x auto* ctx = reactor_find_context(this);
746 29714x if (ctx)
747 29712x ++ctx->private_outstanding_work;
748 29714x }
749
750 inline void
751 1984x reactor_scheduler::post_counted(scheduler_op* op) const
752 {
753 1984x if (auto* ctx = reactor_find_context(this))
754 {
755 608x ctx->private_queue.push(op);
756 608x return;
757 }
758
759 1376x lock_type lock(mutex_);
760 1376x completed_ops_.push(op);
761 1376x wake_one_thread_and_unlock(lock);
762 1376x }
763
764 inline void
765 15663x reactor_scheduler::post_deferred_completions(ready_queue& ops) const
766 {
767 15663x if (ops.empty())
768 15663x return;
769
770 2x if (auto* ctx = reactor_find_context(this))
771 {
772 2x ctx->private_queue.splice(ops);
773 2x return;
774 }
775
776 ✗ lock_type lock(mutex_);
777 ✗ completed_ops_.splice(ops);
778 ✗ wake_one_thread_and_unlock(lock);
779 ✗ }
780
781 inline void
782 22916x reactor_scheduler::retire_descriptor(
783 reactor_retirable& d, object_ref keep) const
784 {
785 22916x lock_type lock(mutex_);
786 22916x if (!task_running_.load(std::memory_order_relaxed))
787 {
788 // No pass can hold an event, but an earlier one may have
789 // queued d.
790 17623x d.hand_off(keep);
791 17623x return;
792 }
793 // Already parked for this pass: the held reference suffices.
794 5293x if (d.retired_ref_)
795 8x return;
796 5285x d.retired_ref_ = std::move(keep);
797 5285x d.retired_next_ = retired_;
798 5285x retired_ = &d;
799 22916x }
800
801 inline void
802 335286x reactor_scheduler::settle_retired(lock_type& lock) const
803 {
804 // Handed off under the lock, since a later pass may retire the same
805 // descriptor again as soon as it is released. A reference that is
806 // not handed off may be the owner's last, and its release recycles
807 // through the pool mutex, so it is dropped with the lock released.
808 335286x reactor_retirable* dropped = nullptr;
809 340571x while (auto* d = retired_)
810 {
811 5285x retired_ = d->retired_next_;
812 5285x d->retired_next_ = nullptr;
813 5285x auto keep = std::move(d->retired_ref_);
814 5285x d->hand_off(keep);
815 // A drop already pending keeps the owner alive, so this one
816 // cannot be the last and is released here.
817 5285x if (!keep || d->dropped_ref_)
818 464x continue;
819 4821x d->dropped_ref_ = std::move(keep);
820 4821x d->dropped_next_ = dropped;
821 4821x dropped = d;
822 10570x }
823 340107x while (auto* d = dropped)
824 {
825 4821x dropped = d->dropped_next_;
826 4821x d->dropped_next_ = nullptr;
827 4821x auto keep = std::move(d->dropped_ref_);
828 4821x lock.unlock();
829 4821x keep.reset();
830 4821x lock.lock();
831 4821x }
832 335286x }
833
834 inline void
835 396x reactor_scheduler::on_timer_service_attached(timer_service& svc)
836 {
837 396x svc.set_on_earliest_changed(timer_service::callback(this, [](void* p) {
838 3100x static_cast<reactor_scheduler*>(p)->interrupt_reactor();
839 3100x }));
840 396x }
841
842 inline void
843 4504x reactor_scheduler::shutdown_drain()
844 {
845 4504x wait_cancels_posting();
846
847 4504x lock_type lock(mutex_);
848 4504x settle_retired(lock);
849 4504x destroy_queued(lock);
850 4504x signal_all(lock);
851 4504x }
852
853 4520x inline reactor_scheduler::~reactor_scheduler()
854 {
855 // Services give up their work in shutdown(), but anything posted
856 // after the drain, such as from a destructor, is still destroyed.
857 4520x lock_type lock(mutex_);
858 4520x destroy_queued(lock);
859 4520x }
860
861 inline void
862 9024x reactor_scheduler::destroy_queued(lock_type& lock) noexcept
863 {
864 22917x while (auto e = completed_ops_.pop())
865 {
866 13893x if (ready_is_continuation(e))
867 {
868 7843x lock.unlock();
869 7843x if (auto* c = ready_as_cont(e); c->h)
870 7843x capy::abandon(*c);
871 7843x lock.lock();
872 }
873 else
874 {
875 6050x auto* op = ready_as_op(e);
876 6050x if (op == &task_op_)
877 4501x continue;
878 1549x lock.unlock();
879 1549x op->destroy();
880 1549x lock.lock();
881 }
882 13893x }
883 9024x }
884
885 inline void
886 11842x reactor_scheduler::signal_all(lock_type&) const
887 {
888 11842x state_ |= signaled_bit;
889 11842x cond_.notify_all();
890 11842x }
891
892 inline bool
893 39674x reactor_scheduler::maybe_unlock_and_signal_one(lock_type& lock) const
894 {
895 39674x state_ |= signaled_bit;
896 39674x if (state_ > signaled_bit)
897 {
898 // Signalled before unlocking: a post from another context's
899 // thread may be the last touch of this scheduler before the
900 // posted work runs, finishes and lets it be destroyed.
901 242x cond_.notify_one();
902 242x lock.unlock();
903 242x return true;
904 }
905 39432x return false;
906 }
907
908 inline bool
909 629211x reactor_scheduler::unlock_and_signal_one(lock_type& lock) const
910 {
911 629211x state_ |= signaled_bit;
912 629211x bool have_waiters = state_ > signaled_bit;
913 629211x lock.unlock();
914 629211x if (have_waiters)
915 3701x cond_.notify_one();
916 629211x return have_waiters;
917 }
918
919 inline void
920 769x reactor_scheduler::clear_signal() const
921 {
922 769x state_ &= ~signaled_bit;
923 769x }
924
925 inline void
926 758x reactor_scheduler::wait_for_signal(lock_type& lock) const
927 {
928 1603x while ((state_ & signaled_bit) == 0)
929 {
930 845x state_ += waiter_increment;
931 845x cond_.wait(lock);
932 845x state_ -= waiter_increment;
933 }
934 758x }
935
936 inline void
937 11x reactor_scheduler::wait_for_signal_for(lock_type& lock, long timeout_us) const
938 {
939 11x if ((state_ & signaled_bit) == 0)
940 {
941 11x state_ += waiter_increment;
942 11x cond_.wait_for(lock, std::chrono::microseconds(timeout_us));
943 11x state_ -= waiter_increment;
944 }
945 11x }
946
947 inline void
948 39674x reactor_scheduler::wake_one_thread_and_unlock(lock_type& lock) const
949 {
950 39674x if (maybe_unlock_and_signal_one(lock))
951 242x return;
952
953 39432x if (task_running_.load(std::memory_order_relaxed) && !task_interrupted_)
954 {
955 1596x task_interrupted_ = true;
956 // Under the lock, for the same reason as the signal above.
957 1596x interrupt_reactor();
958 1596x lock.unlock();
959 }
960 else
961 {
962 37836x lock.unlock();
963 }
964 }
965
966 381857x inline reactor_scheduler::work_cleanup::~work_cleanup()
967 {
968 381857x std::int64_t produced = ctx.private_outstanding_work;
969 381857x if (produced > 1)
970 792x sched->outstanding_work_.fetch_add(
971 produced - 1, std::memory_order_relaxed);
972 381065x else if (produced < 1)
973 65529x sched->work_finished();
974 381857x ctx.private_outstanding_work = 0;
975
976 381857x if (!ctx.private_queue.empty())
977 {
978 287733x lock->lock();
979 287733x sched->completed_ops_.splice(ctx.private_queue);
980 }
981 381857x }
982
983 330785x inline reactor_scheduler::task_cleanup::~task_cleanup()
984 {
985 330785x if (ctx.private_outstanding_work > 0)
986 {
987 9677x sched->outstanding_work_.fetch_add(
988 9677x ctx.private_outstanding_work, std::memory_order_relaxed);
989 9677x ctx.private_outstanding_work = 0;
990 }
991
992 330785x if (!ctx.private_queue.empty())
993 {
994 9677x if (!lock->owns_lock())
995 ✗ lock->lock();
996 9677x sched->completed_ops_.splice(ctx.private_queue);
997 }
998 330785x }
999
1000 inline std::size_t
1001 392128x reactor_scheduler::do_one(lock_type& lock, long timeout_us, context_type& ctx)
1002 {
1003 for (;;)
1004 {
1005 721023x if (stopped_.load(std::memory_order_acquire))
1006 6476x return 0;
1007
1008 714547x auto const e = completed_ops_.pop();
1009 714547x scheduler_op* op = ready_is_continuation(e) ? nullptr : ready_as_op(e);
1010
1011 // Handle reactor sentinel — time to poll for I/O
1012 714547x if (op == &task_op_)
1013 {
1014 331921x bool more_handlers = !completed_ops_.empty();
1015
1016 414432x if (!more_handlers &&
1017 165022x (outstanding_work_.load(std::memory_order_acquire) == 0 ||
1018 timeout_us == 0))
1019 {
1020 1136x completed_ops_.push(&task_op_);
1021 1136x return 0;
1022 }
1023
1024 330785x long task_timeout_us = more_handlers ? 0 : timeout_us;
1025 330785x task_interrupted_ = task_timeout_us == 0;
1026 330785x task_running_.store(true, std::memory_order_release);
1027
1028 // Wake a peer to take the pending handlers while this thread
1029 // polls the reactor; skipped when one_thread_ (no peer exists).
1030 330785x if (more_handlers && !one_thread_)
1031 249399x unlock_and_signal_one(lock);
1032
1033 try
1034 {
1035 330785x run_task(lock, ctx, task_timeout_us);
1036 }
1037 3x catch (...)
1038 {
1039 3x task_running_.store(false, std::memory_order_relaxed);
1040 3x throw;
1041 3x }
1042
1043 330782x task_running_.store(false, std::memory_order_relaxed);
1044 330782x completed_ops_.push(&task_op_);
1045 330782x settle_retired(lock);
1046 330782x if (timeout_us > 0)
1047 2656x return 0;
1048 328126x continue;
1049 328126x }
1050
1051 // Handle ready entry (op or continuation)
1052 382626x if (e != nullptr)
1053 {
1054 381857x bool more = !completed_ops_.empty();
1055
1056 381857x if (more && !one_thread_)
1057 {
1058 // Wake a peer for the remaining work; unassisted if none
1059 // was parked to take it.
1060 379812x ctx.unassisted = !unlock_and_signal_one(lock);
1061 }
1062 else
1063 {
1064 // No peer to wake (one_thread_, or nothing more queued).
1065 2045x ctx.unassisted = more;
1066 2045x lock.unlock();
1067 }
1068
1069 381857x [[maybe_unused]] work_cleanup on_exit{this, &lock, ctx};
1070
1071 381857x if (ready_is_continuation(e))
1072 228825x ready_as_cont(e)->h.resume();
1073 else
1074 153032x (*op)();
1075 381857x return 1;
1076 381857x }
1077
1078 1538x if (outstanding_work_.load(std::memory_order_acquire) == 0 ||
1079 timeout_us == 0)
1080 ✗ return 0;
1081
1082 769x clear_signal();
1083 769x if (timeout_us < 0)
1084 758x wait_for_signal(lock);
1085 else
1086 11x wait_for_signal_for(lock, timeout_us);
1087 328895x }
1088 }
1089
1090 } // namespace boost::corosio::detail
1091
1092 #endif // BOOST_COROSIO_NATIVE_DETAIL_REACTOR_REACTOR_SCHEDULER_HPP
1093