100.00% Lines (101/101)
100.00% Functions (14/14)
| TLA | Baseline | Branch | ||||||
|---|---|---|---|---|---|---|---|---|
| Line | Hits | Code | Line | Hits | Code | |||
| 1 | // | 1 | // | |||||
| 2 | // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) | 2 | // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) | |||||
| 3 | // | 3 | // | |||||
| 4 | // Distributed under the Boost Software License, Version 1.0. (See accompanying | 4 | // Distributed under the Boost Software License, Version 1.0. (See accompanying | |||||
| 5 | // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) | 5 | // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) | |||||
| 6 | // | 6 | // | |||||
| 7 | // Official repository: https://github.com/cppalliance/capy | 7 | // Official repository: https://github.com/cppalliance/capy | |||||
| 8 | // | 8 | // | |||||
| 9 | 9 | |||||||
| 10 | #include <boost/capy/ex/recycling_memory_resource.hpp> | 10 | #include <boost/capy/ex/recycling_memory_resource.hpp> | |||||
| 11 | 11 | |||||||
| 12 | #include <new> | 12 | #include <new> | |||||
| 13 | #include <tuple> | 13 | #include <tuple> | |||||
| 14 | 14 | |||||||
| 15 | namespace boost { | 15 | namespace boost { | |||||
| 16 | namespace capy { | 16 | namespace capy { | |||||
| 17 | 17 | |||||||
| 18 | // Instance destruction does nothing: the resource is stateless (all pools | 18 | // Instance destruction does nothing: the resource is stateless (all pools | |||||
| 19 | // are static). Global-pool cleanup is owned by global()'s holder, exactly | 19 | // are static). Global-pool cleanup is owned by global()'s holder, exactly | |||||
| 20 | // as in the original where the global pool's own destructor drained it, | 20 | // as in the original where the global pool's own destructor drained it, | |||||
| 21 | // independent of any instance lifetime. | 21 | // independent of any instance lifetime. | |||||
| HITCBC | 22 | 327 | recycling_memory_resource::~recycling_memory_resource() = default; | 22 | 327 | recycling_memory_resource::~recycling_memory_resource() = default; | ||
| 23 | 23 | |||||||
| 24 | #if (defined(BOOST_CAPY_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)) && \ | 24 | #if (defined(BOOST_CAPY_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)) && \ | |||||
| 25 | !defined(BOOST_CAPY_STATIC_LINK) | 25 | !defined(BOOST_CAPY_STATIC_LINK) | |||||
| 26 | 26 | |||||||
| 27 | recycling_memory_resource::pool& | 27 | recycling_memory_resource::pool& | |||||
| HITCBC | 28 | 2590777 | recycling_memory_resource::local() noexcept | 28 | 2590777 | recycling_memory_resource::local() noexcept | ||
| 29 | { | 29 | { | |||||
| 30 | static thread_local pool p; | 30 | static thread_local pool p; | |||||
| HITCBC | 31 | 2590777 | return p; | 31 | 2590777 | return p; | ||
| 32 | } | 32 | } | |||||
| 33 | 33 | |||||||
| 34 | bool& | 34 | bool& | |||||
| HITCBC | 35 | 1144341 | recycling_memory_resource::thread_armed() noexcept | 35 | 1144341 | recycling_memory_resource::thread_armed() noexcept | ||
| 36 | { | 36 | { | |||||
| 37 | static thread_local bool armed = false; | 37 | static thread_local bool armed = false; | |||||
| HITCBC | 38 | 1144341 | return armed; | 38 | 1144341 | return armed; | ||
| 39 | } | 39 | } | |||||
| 40 | 40 | |||||||
| 41 | #endif | 41 | #endif | |||||
| 42 | 42 | |||||||
| 43 | namespace { | 43 | namespace { | |||||
| 44 | 44 | |||||||
| 45 | // Set once this thread's pool was drained at exit. Another thread_local | 45 | // Set once this thread's pool was drained at exit. Another thread_local | |||||
| 46 | // destroyed later can still free frames; they must not be cached. | 46 | // destroyed later can still free frames; they must not be cached. | |||||
| 47 | thread_local bool pool_drained = false; | 47 | thread_local bool pool_drained = false; | |||||
| 48 | 48 | |||||||
| 49 | } // namespace | 49 | } // namespace | |||||
| 50 | 50 | |||||||
| 51 | bool | 51 | bool | |||||
| HITCBC | 52 | 302819 | recycling_memory_resource::arm_thread_cleanup() noexcept | 52 | 302819 | recycling_memory_resource::arm_thread_cleanup() noexcept | ||
| 53 | { | 53 | { | |||||
| 54 | struct janitor | 54 | struct janitor | |||||
| 55 | { | 55 | { | |||||
| HITCBC | 56 | 373 | ~janitor() | 56 | 373 | ~janitor() | ||
| 57 | { | 57 | { | |||||
| 58 | // Return this thread's cached blocks to the OS so nothing | 58 | // Return this thread's cached blocks to the OS so nothing | |||||
| 59 | // leaks at thread exit. The pool has a trivial dtor, so its | 59 | // leaks at thread exit. The pool has a trivial dtor, so its | |||||
| 60 | // thread-local storage is still valid here. | 60 | // thread-local storage is still valid here. | |||||
| HITCBC | 61 | 373 | auto& lp = local(); | 61 | 373 | auto& lp = local(); | ||
| HITCBC | 62 | 2611 | for(auto& b : lp.buckets) | 62 | 2611 | for(auto& b : lp.buckets) | ||
| HITCBC | 63 | 52545 | while(b.count > 0) | 63 | 52545 | while(b.count > 0) | ||
| HITCBC | 64 | 50307 | ::operator delete(b.pop()); | 64 | 50307 | ::operator delete(b.pop()); | ||
| HITCBC | 65 | 373 | pool_drained = true; | 65 | 373 | pool_drained = true; | ||
| HITCBC | 66 | 373 | thread_armed() = false; | 66 | 373 | thread_armed() = false; | ||
| HITCBC | 67 | 373 | } | 67 | 373 | } | ||
| 68 | }; | 68 | }; | |||||
| HITCBC | 69 | 302819 | if(pool_drained) | 69 | 302819 | if(pool_drained) | ||
| HITCBC | 70 | 5 | return false; | 70 | 5 | return false; | ||
| HITCBC | 71 | 302814 | static thread_local janitor j; | 71 | 302814 | static thread_local janitor j; | ||
| HITCBC | 72 | 302814 | std::ignore = j; | 72 | 302814 | std::ignore = j; | ||
| HITCBC | 73 | 302814 | return true; | 73 | 302814 | return true; | ||
| 74 | } | 74 | } | |||||
| 75 | 75 | |||||||
| 76 | recycling_memory_resource::pool& | 76 | recycling_memory_resource::pool& | |||||
| HITCBC | 77 | 569107 | recycling_memory_resource::global() noexcept | 77 | 569107 | recycling_memory_resource::global() noexcept | ||
| 78 | { | 78 | { | |||||
| 79 | // Holder gives the global pool a destructor (the trivial-dtor pool type | 79 | // Holder gives the global pool a destructor (the trivial-dtor pool type | |||||
| 80 | // itself must stay guard-free for local()). Runs unconditionally at | 80 | // itself must stay guard-free for local()). Runs unconditionally at | |||||
| 81 | // process exit, mirroring the original global pool destructor, and is | 81 | // process exit, mirroring the original global pool destructor, and is | |||||
| 82 | // locked because a worker thread may still be in a slow path. This path | 82 | // locked because a worker thread may still be in a slow path. This path | |||||
| 83 | // is cold (slow paths only), so the holder's guard costs nothing hot. | 83 | // is cold (slow paths only), so the holder's guard costs nothing hot. | |||||
| 84 | struct holder | 84 | struct holder | |||||
| 85 | { | 85 | { | |||||
| 86 | pool p; | 86 | pool p; | |||||
| 87 | 87 | |||||||
| HITCBC | 88 | 33 | ~holder() | 88 | 33 | ~holder() | ||
| 89 | { | 89 | { | |||||
| HITCBC | 90 | 33 | std::lock_guard<std::mutex> lock(global_mutex()); | 90 | 33 | std::lock_guard<std::mutex> lock(global_mutex()); | ||
| HITCBC | 91 | 231 | for(auto& b : p.buckets) | 91 | 231 | for(auto& b : p.buckets) | ||
| HITCBC | 92 | 5320 | while(b.count > 0) | 92 | 5320 | while(b.count > 0) | ||
| HITCBC | 93 | 5122 | ::operator delete(b.pop()); | 93 | 5122 | ::operator delete(b.pop()); | ||
| HITCBC | 94 | 33 | } | 94 | 33 | } | ||
| 95 | }; | 95 | }; | |||||
| HITCBC | 96 | 569107 | static holder h; | 96 | 569107 | static holder h; | ||
| HITCBC | 97 | 569107 | return h.p; | 97 | 569107 | return h.p; | ||
| 98 | } | 98 | } | |||||
| 99 | 99 | |||||||
| 100 | std::mutex& | 100 | std::mutex& | |||||
| HITCBC | 101 | 569140 | recycling_memory_resource::global_mutex() noexcept | 101 | 569140 | recycling_memory_resource::global_mutex() noexcept | ||
| 102 | { | 102 | { | |||||
| 103 | // Never destroyed: it is locked during process-exit teardown (in | 103 | // Never destroyed: it is locked during process-exit teardown (in | |||||
| 104 | // global()'s holder destructor), after a function-local `static | 104 | // global()'s holder destructor), after a function-local `static | |||||
| 105 | // std::mutex` could itself have been destroyed. Placement-new into | 105 | // std::mutex` could itself have been destroyed. Placement-new into | |||||
| 106 | // static storage owns no heap allocation, so there is nothing to leak. | 106 | // static storage owns no heap allocation, so there is nothing to leak. | |||||
| 107 | alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)]; | 107 | alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)]; | |||||
| 108 | static std::mutex* const mtx = | 108 | static std::mutex* const mtx = | |||||
| HITCBC | 109 | 569140 | ::new(static_cast<void*>(storage)) std::mutex(); | 109 | 569140 | ::new(static_cast<void*>(storage)) std::mutex(); | ||
| HITCBC | 110 | 569140 | return *mtx; | 110 | 569140 | return *mtx; | ||
| 111 | } | 111 | } | |||||
| 112 | 112 | |||||||
| 113 | void* | 113 | void* | |||||
| HITCBC | 114 | 302444 | recycling_memory_resource::allocate_slow( | 114 | 302444 | recycling_memory_resource::allocate_slow( | ||
| 115 | std::size_t rounded, std::size_t idx) | 115 | std::size_t rounded, std::size_t idx) | |||||
| 116 | { | 116 | { | |||||
| 117 | // Once this thread's pool was drained at exit, take one block and | 117 | // Once this thread's pool was drained at exit, take one block and | |||||
| 118 | // leave the pool empty: nothing would release a refill. | 118 | // leave the pool empty: nothing would release a refill. | |||||
| HITCBC | 119 | 302444 | bool const cache = arm_thread_cleanup(); | 119 | 302444 | bool const cache = arm_thread_cleanup(); | ||
| HITCBC | 120 | 302444 | void* p = nullptr; | 120 | 302444 | void* p = nullptr; | ||
| HITCBC | 121 | 302444 | std::size_t refilled = 0; | 121 | 302444 | std::size_t refilled = 0; | ||
| 122 | { | 122 | { | |||||
| HITCBC | 123 | 302444 | std::lock_guard<std::mutex> lock(global_mutex()); | 123 | 302444 | std::lock_guard<std::mutex> lock(global_mutex()); | ||
| HITCBC | 124 | 302444 | auto& b = global().buckets[idx]; | 124 | 302444 | auto& b = global().buckets[idx]; | ||
| HITCBC | 125 | 302444 | if(cache) | 125 | 302444 | if(cache) | ||
| 126 | { | 126 | { | |||||
| HITCBC | 127 | 302443 | auto& lb = local().buckets[idx]; | 127 | 302443 | auto& lb = local().buckets[idx]; | ||
| HITCBC | 128 | 302443 | p = b.pop(lb); | 128 | 302443 | p = b.pop(lb); | ||
| HITCBC | 129 | 302443 | refilled = lb.count; | 129 | 302443 | refilled = lb.count; | ||
| 130 | } | 130 | } | |||||
| 131 | else | 131 | else | |||||
| 132 | { | 132 | { | |||||
| HITCBC | 133 | 1 | p = b.pop(); | 133 | 1 | p = b.pop(); | ||
| 134 | } | 134 | } | |||||
| HITCBC | 135 | 302444 | } | 135 | 302444 | } | ||
| HITCBC | 136 | 302444 | if(p == nullptr) | 136 | 302444 | if(p == nullptr) | ||
| HITCBC | 137 | 302415 | return ::operator new(rounded); | 137 | 302415 | return ::operator new(rounded); | ||
| 138 | 138 | |||||||
| 139 | // A thread that only allocates never frees into its pool, so its | 139 | // A thread that only allocates never frees into its pool, so its | |||||
| 140 | // refills are what drive its decay passes. | 140 | // refills are what drive its decay passes. | |||||
| HITCBC | 141 | 29 | auto& lp = local(); | 141 | 29 | auto& lp = local(); | ||
| HITCBC | 142 | 29 | lp.arrivals += refilled; | 142 | 29 | lp.arrivals += refilled; | ||
| HITCBC | 143 | 29 | if(lp.arrivals >= decay_period) | 143 | 29 | if(lp.arrivals >= decay_period) | ||
| HITCBC | 144 | 7 | decay(lp); | 144 | 7 | decay(lp); | ||
| HITCBC | 145 | 29 | return p; | 145 | 29 | return p; | ||
| 146 | } | 146 | } | |||||
| 147 | 147 | |||||||
| 148 | void | 148 | void | |||||
| HITCBC | 149 | 266384 | recycling_memory_resource::deallocate_slow( | 149 | 266384 | recycling_memory_resource::deallocate_slow( | ||
| 150 | void* p, std::size_t idx) | 150 | void* p, std::size_t idx) | |||||
| 151 | { | 151 | { | |||||
| 152 | { | 152 | { | |||||
| HITCBC | 153 | 266384 | std::lock_guard<std::mutex> lock(global_mutex()); | 153 | 266384 | std::lock_guard<std::mutex> lock(global_mutex()); | ||
| HITCBC | 154 | 266384 | if(global().buckets[idx].push(p, bucket_capacity(idx))) | 154 | 266384 | if(global().buckets[idx].push(p, bucket_capacity(idx))) | ||
| HITCBC | 155 | 59059 | return; | 155 | 59059 | return; | ||
| HITCBC | 156 | 266384 | } | 156 | 266384 | } | ||
| HITCBC | 157 | 207325 | ::operator delete(p); | 157 | 207325 | ::operator delete(p); | ||
| 158 | } | 158 | } | |||||
| 159 | 159 | |||||||
| 160 | void | 160 | void | |||||
| HITCBC | 161 | 279 | recycling_memory_resource::decay(pool& lp) noexcept | 161 | 279 | recycling_memory_resource::decay(pool& lp) noexcept | ||
| 162 | { | 162 | { | |||||
| HITCBC | 163 | 279 | lp.arrivals = 0; | 163 | 279 | lp.arrivals = 0; | ||
| 164 | 164 | |||||||
| 165 | // Unlink each bucket's share before taking the lock, so the walk | 165 | // Unlink each bucket's share before taking the lock, so the walk | |||||
| 166 | // through the blocks happens outside it. | 166 | // through the blocks happens outside it. | |||||
| HITCBC | 167 | 279 | node* heads[num_classes] = {}; | 167 | 279 | node* heads[num_classes] = {}; | ||
| HITCBC | 168 | 279 | node* tails[num_classes] = {}; | 168 | 279 | node* tails[num_classes] = {}; | ||
| HITCBC | 169 | 279 | std::size_t counts[num_classes] = {}; | 169 | 279 | std::size_t counts[num_classes] = {}; | ||
| HITCBC | 170 | 1953 | for(std::size_t i = 0; i < num_classes; ++i) | 170 | 1953 | for(std::size_t i = 0; i < num_classes; ++i) | ||
| 171 | { | 171 | { | |||||
| HITCBC | 172 | 1674 | auto& b = lp.buckets[i]; | 172 | 1674 | auto& b = lp.buckets[i]; | ||
| HITCBC | 173 | 1674 | std::size_t const share = (b.low + 1) / 2; | 173 | 1674 | std::size_t const share = (b.low + 1) / 2; | ||
| HITCBC | 174 | 1674 | if(share > 0) | 174 | 1674 | if(share > 0) | ||
| 175 | { | 175 | { | |||||
| HITCBC | 176 | 122 | heads[i] = b.head; | 176 | 122 | heads[i] = b.head; | ||
| HITCBC | 177 | 122 | node* t = b.head; | 177 | 122 | node* t = b.head; | ||
| HITCBC | 178 | 48938 | for(std::size_t k = 1; k < share; ++k) | 178 | 48938 | for(std::size_t k = 1; k < share; ++k) | ||
| HITCBC | 179 | 48816 | t = t->next; | 179 | 48816 | t = t->next; | ||
| HITCBC | 180 | 122 | b.head = t->next; | 180 | 122 | b.head = t->next; | ||
| HITCBC | 181 | 122 | b.count -= share; | 181 | 122 | b.count -= share; | ||
| HITCBC | 182 | 122 | tails[i] = t; | 182 | 122 | tails[i] = t; | ||
| HITCBC | 183 | 122 | counts[i] = share; | 183 | 122 | counts[i] = share; | ||
| 184 | } | 184 | } | |||||
| HITCBC | 185 | 1674 | b.low = b.count; | 185 | 1674 | b.low = b.count; | ||
| 186 | } | 186 | } | |||||
| 187 | 187 | |||||||
| HITCBC | 188 | 279 | node* spill = nullptr; | 188 | 279 | node* spill = nullptr; | ||
| 189 | { | 189 | { | |||||
| HITCBC | 190 | 279 | std::lock_guard<std::mutex> lock(global_mutex()); | 190 | 279 | std::lock_guard<std::mutex> lock(global_mutex()); | ||
| HITCBC | 191 | 279 | auto& gp = global(); | 191 | 279 | auto& gp = global(); | ||
| HITCBC | 192 | 1953 | for(std::size_t i = 0; i < num_classes; ++i) | 192 | 1953 | for(std::size_t i = 0; i < num_classes; ++i) | ||
| 193 | { | 193 | { | |||||
| HITCBC | 194 | 1674 | if(counts[i] == 0) | 194 | 1674 | if(counts[i] == 0) | ||
| HITCBC | 195 | 1552 | continue; | 195 | 1552 | continue; | ||
| HITCBC | 196 | 122 | auto& g = gp.buckets[i]; | 196 | 122 | auto& g = gp.buckets[i]; | ||
| 197 | // A share that doesn't fit whole is released whole: topping | 197 | // A share that doesn't fit whole is released whole: topping | |||||
| 198 | // off a bucket already at least half full would mean walking | 198 | // off a bucket already at least half full would mean walking | |||||
| 199 | // the chain under the lock. | 199 | // the chain under the lock. | |||||
| HITCBC | 200 | 122 | if(g.count + counts[i] <= bucket_capacity(i)) | 200 | 122 | if(g.count + counts[i] <= bucket_capacity(i)) | ||
| 201 | { | 201 | { | |||||
| HITCBC | 202 | 43 | tails[i]->next = g.head; | 202 | 43 | tails[i]->next = g.head; | ||
| HITCBC | 203 | 43 | g.head = heads[i]; | 203 | 43 | g.head = heads[i]; | ||
| HITCBC | 204 | 43 | g.count += counts[i]; | 204 | 43 | g.count += counts[i]; | ||
| 205 | } | 205 | } | |||||
| 206 | else | 206 | else | |||||
| 207 | { | 207 | { | |||||
| HITCBC | 208 | 79 | tails[i]->next = spill; | 208 | 79 | tails[i]->next = spill; | ||
| HITCBC | 209 | 79 | spill = heads[i]; | 209 | 79 | spill = heads[i]; | ||
| 210 | } | 210 | } | |||||
| 211 | } | 211 | } | |||||
| HITCBC | 212 | 279 | } | 212 | 279 | } | ||
| HITCBC | 213 | 39940 | while(spill != nullptr) | 213 | 39940 | while(spill != nullptr) | ||
| 214 | { | 214 | { | |||||
| HITCBC | 215 | 39661 | node* n = spill; | 215 | 39661 | node* n = spill; | ||
| HITCBC | 216 | 39661 | spill = n->next; | 216 | 39661 | spill = n->next; | ||
| HITCBC | 217 | 39661 | ::operator delete(n); | 217 | 39661 | ::operator delete(n); | ||
| 218 | } | 218 | } | |||||
| HITCBC | 219 | 279 | } | 219 | 279 | } | ||
| 220 | 220 | |||||||
| 221 | void* | 221 | void* | |||||
| HITCBC | 222 | 102458 | recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment) | 222 | 102458 | recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment) | ||
| 223 | { | 223 | { | |||||
| HITCBC | 224 | 102458 | return allocate_fast(bytes, alignment); | 224 | 102458 | return allocate_fast(bytes, alignment); | ||
| 225 | } | 225 | } | |||||
| 226 | 226 | |||||||
| 227 | void | 227 | void | |||||
| HITCBC | 228 | 102458 | recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment) | 228 | 102458 | recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment) | ||
| 229 | { | 229 | { | |||||
| HITCBC | 230 | 102458 | deallocate_fast(p, bytes, alignment); | 230 | 102458 | deallocate_fast(p, bytes, alignment); | ||
| HITCBC | 231 | 102458 | } | 231 | 102458 | } | ||
| 232 | 232 | |||||||
| 233 | std::pmr::memory_resource* | 233 | std::pmr::memory_resource* | |||||
| HITCBC | 234 | 4535 | get_recycling_memory_resource() noexcept | 234 | 4549 | get_recycling_memory_resource() noexcept | ||
| 235 | { | 235 | { | |||||
| HITCBC | 236 | 4535 | static recycling_memory_resource instance; | 236 | 4549 | static recycling_memory_resource instance; | ||
| HITCBC | 237 | 4535 | return &instance; | 237 | 4549 | return &instance; | ||
| 238 | } | 238 | } | |||||
| 239 | 239 | |||||||
| 240 | } // namespace capy | 240 | } // namespace capy | |||||
| 241 | } // namespace boost | 241 | } // namespace boost | |||||