src/ex/thread_pool.cpp

99.3% Lines (150 / 151) 100.0% Functions (28 / 28)
thread_pool.cpp
f(x) Functions (28)
Function Calls Lines Blocks
boost::capy::thread_pool::impl::push(boost::capy::continuation*) :72 22158x 100.0% 100.0% boost::capy::thread_pool::impl::pop() :82 25050x 100.0% 100.0% boost::capy::thread_pool::impl::empty() const :93 42311x 100.0% 100.0% boost::capy::thread_pool::impl::~impl() :110 1446x 100.0% 100.0% boost::capy::thread_pool::impl::running_in_this_thread() const :113 3224x 100.0% 100.0% boost::capy::thread_pool::impl::drain_abandoned() :122 2892x 100.0% 100.0% boost::capy::thread_pool::impl::impl(unsigned long, std::basic_string_view<char, std::char_traits<char> >) :138 1446x 100.0% 72.0% boost::capy::thread_pool::impl::post(boost::capy::continuation&) :151 22160x 100.0% 95.0% boost::capy::thread_pool::impl::on_work_started() :175 4104x 100.0% 100.0% boost::capy::thread_pool::impl::on_work_finished() :181 3804x 100.0% 75.0% boost::capy::thread_pool::impl::join() :203 2753x 100.0% 85.0% boost::capy::thread_pool::impl::join()::{lambda()#1}::operator()() const :219 2067x 100.0% 100.0% boost::capy::thread_pool::impl::stop() :231 1749x 100.0% 100.0% boost::capy::thread_pool::impl::ensure_started() :243 391x 83.3% 88.0% boost::capy::thread_pool::impl::ensure_started()::{lambda()#1}::operator()() const :255 513x 100.0% 100.0% boost::capy::thread_pool::impl::run(unsigned long) :260 513x 100.0% 78.0% boost::capy::thread_pool::impl::run(unsigned long)::scoped_pool::scoped_pool(boost::capy::thread_pool::impl const*) :271 513x 100.0% 100.0% boost::capy::thread_pool::impl::run(unsigned long)::scoped_pool::~scoped_pool() :272 513x 100.0% 100.0% boost::capy::thread_pool::impl::run(unsigned long)::{lambda()#1}::operator()() const :280 42311x 100.0% 100.0% boost::capy::thread_pool::~thread_pool() :296 1446x 100.0% 100.0% boost::capy::thread_pool::thread_pool(unsigned long, std::basic_string_view<char, std::char_traits<char> >) :312 1446x 100.0% 55.0% boost::capy::thread_pool::join() :320 1307x 100.0% 100.0% boost::capy::thread_pool::stop() :327 303x 100.0% 100.0% boost::capy::thread_pool::get_executor() const :336 15365x 100.0% 100.0% boost::capy::thread_pool::executor_type::on_work_started() const :344 4104x 100.0% 100.0% boost::capy::thread_pool::executor_type::on_work_finished() const :351 3804x 100.0% 100.0% boost::capy::thread_pool::executor_type::post(boost::capy::continuation&) const :358 18940x 100.0% 100.0% boost::capy::thread_pool::executor_type::dispatch(boost::capy::continuation&) const :365 3224x 100.0% 100.0%
Line TLA Hits Source Code
1 //
2 // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3 // Copyright (c) 2026 Michael Vandeberg
4 //
5 // Distributed under the Boost Software License, Version 1.0. (See accompanying
6 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
7 //
8 // Official repository: https://github.com/boostorg/capy
9 //
10
11 #include <boost/capy/ex/thread_pool.hpp>
12 #include <boost/capy/continuation.hpp>
13 #include <boost/capy/detail/thread_local_ptr.hpp>
14 #include <boost/capy/ex/abandon.hpp>
15 #include <boost/capy/ex/frame_allocator.hpp>
16 #include <boost/capy/test/thread_name.hpp>
17 #include <algorithm>
18 #include <atomic>
19 #include <condition_variable>
20 #include <cstdio>
21 #include <mutex>
22 #include <thread>
23 #include <vector>
24
25 /*
26 Thread pool implementation using a shared work queue.
27
28 Work items are continuations linked via their intrusive next pointer,
29 stored in a single queue protected by a mutex. No per-post heap
30 allocation: the continuation is owned by the caller and linked
31 directly. Worker threads wait on a condition_variable until work
32 is available or stop is requested.
33
34 Threads are started lazily on first post(), under mutex_, to avoid
35 spawning threads for pools that are constructed but never used. A
36 stopped pool starts none. Each thread is named
37 with a configurable prefix plus index for debugger visibility.
38
39 Work tracking: on_work_started/on_work_finished maintain the atomic
40 outstanding_work_ counter. on_work_started is lock-free, and so is
41 on_work_finished while the count stays above one. The release that
42 may take it to zero is made under mutex_, so join() can't see zero,
43 return and let the pool be destroyed while that call still uses it.
44 join() blocks until this counter reaches zero, then signals workers
45 to stop and joins threads.
46
47 Two shutdown paths:
48 - join(): waits for outstanding work to drain, then stops workers.
49 - stop(): immediately signals workers to exit; queued work stays queued.
50 - Destructor: stop() then join(), then queued work is abandoned.
51 */
52
53 namespace boost {
54 namespace capy {
55
56 //------------------------------------------------------------------------------
57
58 class thread_pool::impl
59 {
60 // Identifies the pool owning the current worker thread, or
61 // nullptr if the calling thread is not a pool worker. Checked
62 // by dispatch() to decide between symmetric transfer (inline
63 // resume) and post.
64 static inline detail::thread_local_ptr<impl const> current_;
65
66 // Intrusive queue of continuations: the next link is stored in
67 // continuation::reserved (typed continuation* round-tripped through
68 // void*). No per-post allocation: the continuation is owned by the caller.
69 continuation* head_ = nullptr;
70 continuation* tail_ = nullptr;
71
72 22158x void push(continuation* c) noexcept
73 {
74 22158x c->reserved = nullptr;
75 22158x if(tail_)
76 2083x tail_->reserved = c;
77 else
78 20075x head_ = c;
79 22158x tail_ = c;
80 22158x }
81
82 25050x continuation* pop() noexcept
83 {
84 25050x if(!head_)
85 2892x return nullptr;
86 22158x continuation* c = head_;
87 22158x head_ = static_cast<continuation*>(head_->reserved);
88 22158x if(!head_)
89 20075x tail_ = nullptr;
90 22158x return c;
91 }
92
93 42311x bool empty() const noexcept
94 {
95 42311x return head_ == nullptr;
96 }
97
98 std::mutex mutex_;
99 std::condition_variable work_cv_;
100 std::condition_variable done_cv_;
101 std::vector<std::thread> threads_;
102 std::atomic<std::size_t> outstanding_work_{0};
103 bool stop_{false};
104 bool joined_{false};
105 std::size_t num_threads_;
106 char thread_name_prefix_[13]{}; // 12 chars max + null terminator
107 std::atomic<bool> started_{false};
108
109 public:
110 1446x ~impl() = default;
111
112 bool
113 3224x running_in_this_thread() const noexcept
114 {
115 3224x return current_.get() == this;
116 }
117
118 // Abandon queued coroutines. Each is taken under the lock, as other
119 // threads may still post, and abandoned outside it, as abandoning
120 // can post.
121 void
122 2892x drain_abandoned() noexcept
123 {
124 for(;;)
125 {
126 continuation* c;
127 {
128 3415x std::lock_guard<std::mutex> lock(mutex_);
129 3415x c = pop();
130 3415x }
131 3415x if(!c)
132 2892x return;
133 523x if(c->h && c->h != std::noop_coroutine())
134 270x capy::abandon(*c);
135 523x }
136 }
137
138 1446x impl(std::size_t num_threads, std::string_view thread_name_prefix)
139 1446x : num_threads_(num_threads)
140 {
141 1446x if(num_threads_ == 0)
142 8x num_threads_ = std::max(
143 4x std::thread::hardware_concurrency(), 1u);
144
145 // Truncate prefix to 12 chars, leaving room for up to 3-digit index.
146 1446x auto n = thread_name_prefix.copy(thread_name_prefix_, 12);
147 1446x thread_name_prefix_[n] = '\0';
148 1446x }
149
150 void
151 22160x post(continuation& c)
152 {
153 22160x if(!started_.load(std::memory_order_acquire))
154 {
155 {
156 611x std::lock_guard<std::mutex> lock(mutex_);
157 611x if(stop_)
158 {
159 // Stopped before any thread started: nothing will
160 // run it, and the destructor's drain abandons it.
161 220x push(&c);
162 220x return;
163 }
164 611x }
165 391x ensure_started();
166 }
167 21938x std::lock_guard<std::mutex> lock(mutex_);
168 21938x push(&c);
169 // Under the lock so the pool cannot drain, join, and
170 // destroy the condition variable mid-signal.
171 21938x work_cv_.notify_one();
172 21938x }
173
174 void
175 4104x on_work_started() noexcept
176 {
177 4104x outstanding_work_.fetch_add(1, std::memory_order_acq_rel);
178 4104x }
179
180 void
181 3804x on_work_finished() noexcept
182 {
183 3804x auto n = outstanding_work_.load(std::memory_order_relaxed);
184 3813x while(n > 1)
185 {
186 2342x if(outstanding_work_.compare_exchange_weak(
187 n, n - 1, std::memory_order_acq_rel))
188 1162x return;
189 }
190 2642x std::lock_guard<std::mutex> lock(mutex_);
191 2642x outstanding_work_.fetch_sub(1, std::memory_order_acq_rel);
192 // Re-read: a concurrent on_work_started() may have raised it.
193 2642x if(outstanding_work_.load(
194 2642x std::memory_order_acquire) == 0 && joined_ && !stop_)
195 {
196 705x stop_ = true;
197 705x done_cv_.notify_all();
198 705x work_cv_.notify_all();
199 }
200 2642x }
201
202 void
203 2753x join() noexcept
204 {
205 {
206 2753x std::unique_lock<std::mutex> lock(mutex_);
207 2753x if(joined_)
208 1307x return;
209 1446x joined_ = true;
210
211 1446x if(outstanding_work_.load(
212 1446x std::memory_order_acquire) == 0)
213 {
214 385x stop_ = true;
215 385x work_cv_.notify_all();
216 }
217 else
218 {
219 1061x done_cv_.wait(lock, [this]{
220 2067x return stop_;
221 });
222 }
223 2753x }
224
225 1959x for(auto& t : threads_)
226 513x if(t.joinable())
227 513x t.join();
228 }
229
230 void
231 1749x stop() noexcept
232 {
233 // Under the lock: the join() this releases may return and let
234 // the pool be destroyed as soon as the lock is free.
235 1749x std::lock_guard<std::mutex> lock(mutex_);
236 1749x stop_ = true;
237 1749x work_cv_.notify_all();
238 1749x done_cv_.notify_all();
239 1749x }
240
241 private:
242 void
243 391x ensure_started()
244 {
245 // Under the lock, so join() either sees every thread started
246 // here or makes this start none. A joined pool still starts them:
247 // join() may be waiting on work that needs them.
248 391x std::lock_guard<std::mutex> lock(mutex_);
249 391x if(started_.load(std::memory_order_relaxed) || stop_)
250 ✗ return;
251 // Start at the threads already running: if an earlier call
252 // threw partway, this one adds only the missing ones.
253 391x threads_.reserve(num_threads_);
254 904x for(std::size_t i = threads_.size(); i < num_threads_; ++i)
255 1028x threads_.emplace_back([this, i]{ run(i); });
256 389x started_.store(true, std::memory_order_release);
257 391x }
258
259 void
260 513x run(std::size_t index)
261 {
262 // Build name; set_current_thread_name truncates to platform limits.
263 char name[16];
264 513x std::snprintf(name, sizeof(name), "%s%zu", thread_name_prefix_, index);
265 513x set_current_thread_name(name);
266
267 // Mark this thread as a worker of this pool so dispatch()
268 // can symmetric-transfer when called from within pool work.
269 struct scoped_pool
270 {
271 513x scoped_pool(impl const* p) noexcept { current_.set(p); }
272 513x ~scoped_pool() noexcept { current_.set(nullptr); }
273 513x } guard(this);
274
275 for(;;)
276 {
277 22148x continuation* c = nullptr;
278 {
279 22148x std::unique_lock<std::mutex> lock(mutex_);
280 22148x work_cv_.wait(lock, [this]{
281 62890x return !empty() ||
282 62890x stop_;
283 });
284 22148x if(stop_)
285 1026x return;
286 21635x c = pop();
287 22148x }
288 21635x if(c)
289 21635x safe_resume(c->h);
290 21635x }
291 513x }
292 };
293
294 //------------------------------------------------------------------------------
295
296 1446x thread_pool::
297 ~thread_pool()
298 {
299 1446x impl_->stop();
300 1446x impl_->join();
301 1446x shutdown();
302 // Takes what the unwinds post too, and the work of any service they
303 // create, which is shut down as it is created.
304 1446x impl_->drain_abandoned();
305 1446x destroy();
306 // Services give up their work in shutdown(), but anything a
307 // destructor posts anyway is still destroyed.
308 1446x impl_->drain_abandoned();
309 1446x delete impl_;
310 1446x }
311
312 1446x thread_pool::
313 1446x thread_pool(std::size_t num_threads, std::string_view thread_name_prefix)
314 1446x : impl_(new impl(num_threads, thread_name_prefix))
315 {
316 1446x this->set_frame_allocator(std::allocator<void>{});
317 1446x }
318
319 void
320 1307x thread_pool::
321 join() noexcept
322 {
323 1307x impl_->join();
324 1307x }
325
326 void
327 303x thread_pool::
328 stop() noexcept
329 {
330 303x impl_->stop();
331 303x }
332
333 //------------------------------------------------------------------------------
334
335 thread_pool::executor_type
336 15365x thread_pool::
337 get_executor() const noexcept
338 {
339 15365x return executor_type(
340 15365x const_cast<thread_pool&>(*this));
341 }
342
343 void
344 4104x thread_pool::executor_type::
345 on_work_started() const noexcept
346 {
347 4104x pool_->impl_->on_work_started();
348 4104x }
349
350 void
351 3804x thread_pool::executor_type::
352 on_work_finished() const noexcept
353 {
354 3804x pool_->impl_->on_work_finished();
355 3804x }
356
357 void
358 18940x thread_pool::executor_type::
359 post(continuation& c) const
360 {
361 18940x pool_->impl_->post(c);
362 18938x }
363
364 std::coroutine_handle<>
365 3224x thread_pool::executor_type::
366 dispatch(continuation& c) const
367 {
368 3224x if(pool_->impl_->running_in_this_thread())
369 4x return c.h;
370 3220x pool_->impl_->post(c);
371 3220x return std::noop_coroutine();
372 }
373
374 } // capy
375 } // boost
376