100.00% Lines (101/101) 100.00% Functions (14/14)
TLA Baseline Branch
Line Hits Code Line Hits Code
1   // 1   //
2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com) 2   // Copyright (c) 2025 Vinnie Falco (vinnie.falco@gmail.com)
3   // 3   //
4   // Distributed under the Boost Software License, Version 1.0. (See accompanying 4   // Distributed under the Boost Software License, Version 1.0. (See accompanying
5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) 5   // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
6   // 6   //
7   // Official repository: https://github.com/cppalliance/capy 7   // Official repository: https://github.com/cppalliance/capy
8   // 8   //
9   9  
10   #include <boost/capy/ex/recycling_memory_resource.hpp> 10   #include <boost/capy/ex/recycling_memory_resource.hpp>
11   11  
12   #include <new> 12   #include <new>
13   #include <tuple> 13   #include <tuple>
14   14  
15   namespace boost { 15   namespace boost {
16   namespace capy { 16   namespace capy {
17   17  
18   // Instance destruction does nothing: the resource is stateless (all pools 18   // Instance destruction does nothing: the resource is stateless (all pools
19   // are static). Global-pool cleanup is owned by global()'s holder, exactly 19   // are static). Global-pool cleanup is owned by global()'s holder, exactly
20   // as in the original where the global pool's own destructor drained it, 20   // as in the original where the global pool's own destructor drained it,
21   // independent of any instance lifetime. 21   // independent of any instance lifetime.
HITCBC 22   327 recycling_memory_resource::~recycling_memory_resource() = default; 22   327 recycling_memory_resource::~recycling_memory_resource() = default;
23   23  
24   #if (defined(BOOST_CAPY_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)) && \ 24   #if (defined(BOOST_CAPY_DYN_LINK) || defined(BOOST_ALL_DYN_LINK)) && \
25   !defined(BOOST_CAPY_STATIC_LINK) 25   !defined(BOOST_CAPY_STATIC_LINK)
26   26  
27   recycling_memory_resource::pool& 27   recycling_memory_resource::pool&
HITCBC 28   2590777 recycling_memory_resource::local() noexcept 28   2590777 recycling_memory_resource::local() noexcept
29   { 29   {
30   static thread_local pool p; 30   static thread_local pool p;
HITCBC 31   2590777 return p; 31   2590777 return p;
32   } 32   }
33   33  
34   bool& 34   bool&
HITCBC 35   1144341 recycling_memory_resource::thread_armed() noexcept 35   1144341 recycling_memory_resource::thread_armed() noexcept
36   { 36   {
37   static thread_local bool armed = false; 37   static thread_local bool armed = false;
HITCBC 38   1144341 return armed; 38   1144341 return armed;
39   } 39   }
40   40  
41   #endif 41   #endif
42   42  
43   namespace { 43   namespace {
44   44  
45   // Set once this thread's pool was drained at exit. Another thread_local 45   // Set once this thread's pool was drained at exit. Another thread_local
46   // destroyed later can still free frames; they must not be cached. 46   // destroyed later can still free frames; they must not be cached.
47   thread_local bool pool_drained = false; 47   thread_local bool pool_drained = false;
48   48  
49   } // namespace 49   } // namespace
50   50  
51   bool 51   bool
HITCBC 52   302819 recycling_memory_resource::arm_thread_cleanup() noexcept 52   302819 recycling_memory_resource::arm_thread_cleanup() noexcept
53   { 53   {
54   struct janitor 54   struct janitor
55   { 55   {
HITCBC 56   373 ~janitor() 56   373 ~janitor()
57   { 57   {
58   // Return this thread's cached blocks to the OS so nothing 58   // Return this thread's cached blocks to the OS so nothing
59   // leaks at thread exit. The pool has a trivial dtor, so its 59   // leaks at thread exit. The pool has a trivial dtor, so its
60   // thread-local storage is still valid here. 60   // thread-local storage is still valid here.
HITCBC 61   373 auto& lp = local(); 61   373 auto& lp = local();
HITCBC 62   2611 for(auto& b : lp.buckets) 62   2611 for(auto& b : lp.buckets)
HITCBC 63   52545 while(b.count > 0) 63   52545 while(b.count > 0)
HITCBC 64   50307 ::operator delete(b.pop()); 64   50307 ::operator delete(b.pop());
HITCBC 65   373 pool_drained = true; 65   373 pool_drained = true;
HITCBC 66   373 thread_armed() = false; 66   373 thread_armed() = false;
HITCBC 67   373 } 67   373 }
68   }; 68   };
HITCBC 69   302819 if(pool_drained) 69   302819 if(pool_drained)
HITCBC 70   5 return false; 70   5 return false;
HITCBC 71   302814 static thread_local janitor j; 71   302814 static thread_local janitor j;
HITCBC 72   302814 std::ignore = j; 72   302814 std::ignore = j;
HITCBC 73   302814 return true; 73   302814 return true;
74   } 74   }
75   75  
76   recycling_memory_resource::pool& 76   recycling_memory_resource::pool&
HITCBC 77   569107 recycling_memory_resource::global() noexcept 77   569107 recycling_memory_resource::global() noexcept
78   { 78   {
79   // Holder gives the global pool a destructor (the trivial-dtor pool type 79   // Holder gives the global pool a destructor (the trivial-dtor pool type
80   // itself must stay guard-free for local()). Runs unconditionally at 80   // itself must stay guard-free for local()). Runs unconditionally at
81   // process exit, mirroring the original global pool destructor, and is 81   // process exit, mirroring the original global pool destructor, and is
82   // locked because a worker thread may still be in a slow path. This path 82   // locked because a worker thread may still be in a slow path. This path
83   // is cold (slow paths only), so the holder's guard costs nothing hot. 83   // is cold (slow paths only), so the holder's guard costs nothing hot.
84   struct holder 84   struct holder
85   { 85   {
86   pool p; 86   pool p;
87   87  
HITCBC 88   33 ~holder() 88   33 ~holder()
89   { 89   {
HITCBC 90   33 std::lock_guard<std::mutex> lock(global_mutex()); 90   33 std::lock_guard<std::mutex> lock(global_mutex());
HITCBC 91   231 for(auto& b : p.buckets) 91   231 for(auto& b : p.buckets)
HITCBC 92   5320 while(b.count > 0) 92   5320 while(b.count > 0)
HITCBC 93   5122 ::operator delete(b.pop()); 93   5122 ::operator delete(b.pop());
HITCBC 94   33 } 94   33 }
95   }; 95   };
HITCBC 96   569107 static holder h; 96   569107 static holder h;
HITCBC 97   569107 return h.p; 97   569107 return h.p;
98   } 98   }
99   99  
100   std::mutex& 100   std::mutex&
HITCBC 101   569140 recycling_memory_resource::global_mutex() noexcept 101   569140 recycling_memory_resource::global_mutex() noexcept
102   { 102   {
103   // Never destroyed: it is locked during process-exit teardown (in 103   // Never destroyed: it is locked during process-exit teardown (in
104   // global()'s holder destructor), after a function-local `static 104   // global()'s holder destructor), after a function-local `static
105   // std::mutex` could itself have been destroyed. Placement-new into 105   // std::mutex` could itself have been destroyed. Placement-new into
106   // static storage owns no heap allocation, so there is nothing to leak. 106   // static storage owns no heap allocation, so there is nothing to leak.
107   alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)]; 107   alignas(std::mutex) static unsigned char storage[sizeof(std::mutex)];
108   static std::mutex* const mtx = 108   static std::mutex* const mtx =
HITCBC 109   569140 ::new(static_cast<void*>(storage)) std::mutex(); 109   569140 ::new(static_cast<void*>(storage)) std::mutex();
HITCBC 110   569140 return *mtx; 110   569140 return *mtx;
111   } 111   }
112   112  
113   void* 113   void*
HITCBC 114   302444 recycling_memory_resource::allocate_slow( 114   302444 recycling_memory_resource::allocate_slow(
115   std::size_t rounded, std::size_t idx) 115   std::size_t rounded, std::size_t idx)
116   { 116   {
117   // Once this thread's pool was drained at exit, take one block and 117   // Once this thread's pool was drained at exit, take one block and
118   // leave the pool empty: nothing would release a refill. 118   // leave the pool empty: nothing would release a refill.
HITCBC 119   302444 bool const cache = arm_thread_cleanup(); 119   302444 bool const cache = arm_thread_cleanup();
HITCBC 120   302444 void* p = nullptr; 120   302444 void* p = nullptr;
HITCBC 121   302444 std::size_t refilled = 0; 121   302444 std::size_t refilled = 0;
122   { 122   {
HITCBC 123   302444 std::lock_guard<std::mutex> lock(global_mutex()); 123   302444 std::lock_guard<std::mutex> lock(global_mutex());
HITCBC 124   302444 auto& b = global().buckets[idx]; 124   302444 auto& b = global().buckets[idx];
HITCBC 125   302444 if(cache) 125   302444 if(cache)
126   { 126   {
HITCBC 127   302443 auto& lb = local().buckets[idx]; 127   302443 auto& lb = local().buckets[idx];
HITCBC 128   302443 p = b.pop(lb); 128   302443 p = b.pop(lb);
HITCBC 129   302443 refilled = lb.count; 129   302443 refilled = lb.count;
130   } 130   }
131   else 131   else
132   { 132   {
HITCBC 133   1 p = b.pop(); 133   1 p = b.pop();
134   } 134   }
HITCBC 135   302444 } 135   302444 }
HITCBC 136   302444 if(p == nullptr) 136   302444 if(p == nullptr)
HITCBC 137   302415 return ::operator new(rounded); 137   302415 return ::operator new(rounded);
138   138  
139   // A thread that only allocates never frees into its pool, so its 139   // A thread that only allocates never frees into its pool, so its
140   // refills are what drive its decay passes. 140   // refills are what drive its decay passes.
HITCBC 141   29 auto& lp = local(); 141   29 auto& lp = local();
HITCBC 142   29 lp.arrivals += refilled; 142   29 lp.arrivals += refilled;
HITCBC 143   29 if(lp.arrivals >= decay_period) 143   29 if(lp.arrivals >= decay_period)
HITCBC 144   7 decay(lp); 144   7 decay(lp);
HITCBC 145   29 return p; 145   29 return p;
146   } 146   }
147   147  
148   void 148   void
HITCBC 149   266384 recycling_memory_resource::deallocate_slow( 149   266384 recycling_memory_resource::deallocate_slow(
150   void* p, std::size_t idx) 150   void* p, std::size_t idx)
151   { 151   {
152   { 152   {
HITCBC 153   266384 std::lock_guard<std::mutex> lock(global_mutex()); 153   266384 std::lock_guard<std::mutex> lock(global_mutex());
HITCBC 154   266384 if(global().buckets[idx].push(p, bucket_capacity(idx))) 154   266384 if(global().buckets[idx].push(p, bucket_capacity(idx)))
HITCBC 155   59059 return; 155   59059 return;
HITCBC 156   266384 } 156   266384 }
HITCBC 157   207325 ::operator delete(p); 157   207325 ::operator delete(p);
158   } 158   }
159   159  
160   void 160   void
HITCBC 161   279 recycling_memory_resource::decay(pool& lp) noexcept 161   279 recycling_memory_resource::decay(pool& lp) noexcept
162   { 162   {
HITCBC 163   279 lp.arrivals = 0; 163   279 lp.arrivals = 0;
164   164  
165   // Unlink each bucket's share before taking the lock, so the walk 165   // Unlink each bucket's share before taking the lock, so the walk
166   // through the blocks happens outside it. 166   // through the blocks happens outside it.
HITCBC 167   279 node* heads[num_classes] = {}; 167   279 node* heads[num_classes] = {};
HITCBC 168   279 node* tails[num_classes] = {}; 168   279 node* tails[num_classes] = {};
HITCBC 169   279 std::size_t counts[num_classes] = {}; 169   279 std::size_t counts[num_classes] = {};
HITCBC 170   1953 for(std::size_t i = 0; i < num_classes; ++i) 170   1953 for(std::size_t i = 0; i < num_classes; ++i)
171   { 171   {
HITCBC 172   1674 auto& b = lp.buckets[i]; 172   1674 auto& b = lp.buckets[i];
HITCBC 173   1674 std::size_t const share = (b.low + 1) / 2; 173   1674 std::size_t const share = (b.low + 1) / 2;
HITCBC 174   1674 if(share > 0) 174   1674 if(share > 0)
175   { 175   {
HITCBC 176   122 heads[i] = b.head; 176   122 heads[i] = b.head;
HITCBC 177   122 node* t = b.head; 177   122 node* t = b.head;
HITCBC 178   48938 for(std::size_t k = 1; k < share; ++k) 178   48938 for(std::size_t k = 1; k < share; ++k)
HITCBC 179   48816 t = t->next; 179   48816 t = t->next;
HITCBC 180   122 b.head = t->next; 180   122 b.head = t->next;
HITCBC 181   122 b.count -= share; 181   122 b.count -= share;
HITCBC 182   122 tails[i] = t; 182   122 tails[i] = t;
HITCBC 183   122 counts[i] = share; 183   122 counts[i] = share;
184   } 184   }
HITCBC 185   1674 b.low = b.count; 185   1674 b.low = b.count;
186   } 186   }
187   187  
HITCBC 188   279 node* spill = nullptr; 188   279 node* spill = nullptr;
189   { 189   {
HITCBC 190   279 std::lock_guard<std::mutex> lock(global_mutex()); 190   279 std::lock_guard<std::mutex> lock(global_mutex());
HITCBC 191   279 auto& gp = global(); 191   279 auto& gp = global();
HITCBC 192   1953 for(std::size_t i = 0; i < num_classes; ++i) 192   1953 for(std::size_t i = 0; i < num_classes; ++i)
193   { 193   {
HITCBC 194   1674 if(counts[i] == 0) 194   1674 if(counts[i] == 0)
HITCBC 195   1552 continue; 195   1552 continue;
HITCBC 196   122 auto& g = gp.buckets[i]; 196   122 auto& g = gp.buckets[i];
197   // A share that doesn't fit whole is released whole: topping 197   // A share that doesn't fit whole is released whole: topping
198   // off a bucket already at least half full would mean walking 198   // off a bucket already at least half full would mean walking
199   // the chain under the lock. 199   // the chain under the lock.
HITCBC 200   122 if(g.count + counts[i] <= bucket_capacity(i)) 200   122 if(g.count + counts[i] <= bucket_capacity(i))
201   { 201   {
HITCBC 202   43 tails[i]->next = g.head; 202   43 tails[i]->next = g.head;
HITCBC 203   43 g.head = heads[i]; 203   43 g.head = heads[i];
HITCBC 204   43 g.count += counts[i]; 204   43 g.count += counts[i];
205   } 205   }
206   else 206   else
207   { 207   {
HITCBC 208   79 tails[i]->next = spill; 208   79 tails[i]->next = spill;
HITCBC 209   79 spill = heads[i]; 209   79 spill = heads[i];
210   } 210   }
211   } 211   }
HITCBC 212   279 } 212   279 }
HITCBC 213   39940 while(spill != nullptr) 213   39940 while(spill != nullptr)
214   { 214   {
HITCBC 215   39661 node* n = spill; 215   39661 node* n = spill;
HITCBC 216   39661 spill = n->next; 216   39661 spill = n->next;
HITCBC 217   39661 ::operator delete(n); 217   39661 ::operator delete(n);
218   } 218   }
HITCBC 219   279 } 219   279 }
220   220  
221   void* 221   void*
HITCBC 222   102458 recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment) 222   102458 recycling_memory_resource::do_allocate(std::size_t bytes, std::size_t alignment)
223   { 223   {
HITCBC 224   102458 return allocate_fast(bytes, alignment); 224   102458 return allocate_fast(bytes, alignment);
225   } 225   }
226   226  
227   void 227   void
HITCBC 228   102458 recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment) 228   102458 recycling_memory_resource::do_deallocate(void* p, std::size_t bytes, std::size_t alignment)
229   { 229   {
HITCBC 230   102458 deallocate_fast(p, bytes, alignment); 230   102458 deallocate_fast(p, bytes, alignment);
HITCBC 231   102458 } 231   102458 }
232   232  
233   std::pmr::memory_resource* 233   std::pmr::memory_resource*
HITCBC 234   4535 get_recycling_memory_resource() noexcept 234   4549 get_recycling_memory_resource() noexcept
235   { 235   {
HITCBC 236   4535 static recycling_memory_resource instance; 236   4549 static recycling_memory_resource instance;
HITCBC 237   4535 return &instance; 237   4549 return &instance;
238   } 238   }
239   239  
240   } // namespace capy 240   } // namespace capy
241   } // namespace boost 241   } // namespace boost