Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * dsa.c
4 : : * Dynamic shared memory areas.
5 : : *
6 : : * This module provides dynamic shared memory areas which are built on top of
7 : : * DSM segments. While dsm.c allows segments of memory of shared memory to be
8 : : * created and shared between backends, it isn't designed to deal with small
9 : : * objects. A DSA area is a shared memory heap usually backed by one or more
10 : : * DSM segments which can allocate memory using dsa_allocate() and dsa_free().
11 : : * Alternatively, it can be created in pre-existing shared memory, including a
12 : : * DSM segment, and then create extra DSM segments as required. Unlike the
13 : : * regular system heap, it deals in pseudo-pointers which must be converted to
14 : : * backend-local pointers before they are dereferenced. These pseudo-pointers
15 : : * can however be shared with other backends, and can be used to construct
16 : : * shared data structures.
17 : : *
18 : : * Each DSA area manages a set of DSM segments, adding new segments as
19 : : * required and detaching them when they are no longer needed. Each segment
20 : : * contains a number of 4KB pages, a free page manager for tracking
21 : : * consecutive runs of free pages, and a page map for tracking the source of
22 : : * objects allocated on each page. Allocation requests above 8KB are handled
23 : : * by choosing a segment and finding consecutive free pages in its free page
24 : : * manager. Allocation requests for smaller sizes are handled using pools of
25 : : * objects of a selection of sizes. Each pool consists of a number of 16 page
26 : : * (64KB) superblocks allocated in the same way as large objects. Allocation
27 : : * of large objects and new superblocks is serialized by a single LWLock, but
28 : : * allocation of small objects from pre-existing superblocks uses one LWLock
29 : : * per pool. Currently there is one pool, and therefore one lock, per size
30 : : * class. Per-core pools to increase concurrency and strategies for reducing
31 : : * the resulting fragmentation are areas for future research. Each superblock
32 : : * is managed with a 'span', which tracks the superblock's freelist. Free
33 : : * requests are handled by looking in the page map to find which span an
34 : : * address was allocated from, so that small objects can be returned to the
35 : : * appropriate free list, and large object pages can be returned directly to
36 : : * the free page map. When allocating, simple heuristics for selecting
37 : : * segments and superblocks try to encourage occupied memory to be
38 : : * concentrated, increasing the likelihood that whole superblocks can become
39 : : * empty and be returned to the free page manager, and whole segments can
40 : : * become empty and be returned to the operating system.
41 : : *
42 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
43 : : * Portions Copyright (c) 1994, Regents of the University of California
44 : : *
45 : : * IDENTIFICATION
46 : : * src/backend/utils/mmgr/dsa.c
47 : : *
48 : : *-------------------------------------------------------------------------
49 : : */
50 : :
51 : : #include "postgres.h"
52 : :
53 : : #include "port/atomics.h"
54 : : #include "port/pg_bitutils.h"
55 : : #include "storage/dsm.h"
56 : : #include "storage/lwlock.h"
57 : : #include "utils/dsa.h"
58 : : #include "utils/freepage.h"
59 : : #include "utils/memutils.h"
60 : : #include "utils/resowner.h"
61 : :
62 : : /*
63 : : * How many segments to create before we double the segment size. If this is
64 : : * low, then there is likely to be a lot of wasted space in the largest
65 : : * segment. If it is high, then we risk running out of segment slots (see
66 : : * dsm.c's limits on total number of segments), or limiting the total size
67 : : * an area can manage when using small pointers.
68 : : */
69 : : #define DSA_NUM_SEGMENTS_AT_EACH_SIZE 2
70 : :
71 : : /*
72 : : * The maximum number of DSM segments that an area can own, determined by
73 : : * the number of bits remaining (but capped at 1024).
74 : : */
75 : : #define DSA_MAX_SEGMENTS \
76 : : Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
77 : :
78 : : /* The bitmask for extracting the offset from a dsa_pointer. */
79 : : #define DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
80 : :
81 : : /* Number of pages (see FPM_PAGE_SIZE) per regular superblock. */
82 : : #define DSA_PAGES_PER_SUPERBLOCK 16
83 : :
84 : : /*
85 : : * A magic number used as a sanity check for following DSM segments belonging
86 : : * to a DSA area (this number will be XORed with the area handle and
87 : : * the segment index).
88 : : */
89 : : #define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608
90 : :
91 : : /* Build a dsa_pointer given a segment number and offset. */
92 : : #define DSA_MAKE_POINTER(segment_number, offset) \
93 : : (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
94 : :
95 : : /* Extract the segment number from a dsa_pointer. */
96 : : #define DSA_EXTRACT_SEGMENT_NUMBER(dp) ((dp) >> DSA_OFFSET_WIDTH)
97 : :
98 : : /* Extract the offset from a dsa_pointer. */
99 : : #define DSA_EXTRACT_OFFSET(dp) ((dp) & DSA_OFFSET_BITMASK)
100 : :
101 : : /* The type used for index segment indexes (zero based). */
102 : : typedef size_t dsa_segment_index;
103 : :
104 : : /* Sentinel value for dsa_segment_index indicating 'none' or 'end'. */
105 : : #define DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)
106 : :
107 : : /*
108 : : * How many bins of segments do we have? The bins are used to categorize
109 : : * segments by their largest contiguous run of free pages.
110 : : */
111 : : #define DSA_NUM_SEGMENT_BINS 16
112 : :
113 : : /*
114 : : * What is the lowest bin that holds segments that *might* have n contiguous
115 : : * free pages? There is no point in looking in segments in lower bins; they
116 : : * definitely can't service a request for n free pages.
117 : : */
118 : : static inline size_t
119 : 27974 : contiguous_pages_to_segment_bin(size_t n)
120 : : {
121 : : size_t bin;
122 : :
123 [ + + ]: 27974 : if (n == 0)
124 : 1046 : bin = 0;
125 : : else
126 : 26928 : bin = pg_leftmost_one_pos_size_t(n) + 1;
127 : :
128 : 27974 : return Min(bin, DSA_NUM_SEGMENT_BINS - 1);
129 : : }
130 : :
131 : : /* Macros for access to locks. */
132 : : #define DSA_AREA_LOCK(area) (&area->control->lock)
133 : : #define DSA_SCLASS_LOCK(area, sclass) (&area->control->pools[sclass].lock)
134 : :
135 : : /*
136 : : * The header for an individual segment. This lives at the start of each DSM
137 : : * segment owned by a DSA area including the first segment (where it appears
138 : : * as part of the dsa_area_control struct).
139 : : */
140 : : typedef struct
141 : : {
142 : : /* Sanity check magic value. */
143 : : uint32 magic;
144 : : /* Total number of pages in this segment (excluding metadata area). */
145 : : size_t usable_pages;
146 : : /* Total size of this segment in bytes. */
147 : : size_t size;
148 : :
149 : : /*
150 : : * Index of the segment that precedes this one in the same segment bin, or
151 : : * DSA_SEGMENT_INDEX_NONE if this is the first one.
152 : : */
153 : : dsa_segment_index prev;
154 : :
155 : : /*
156 : : * Index of the segment that follows this one in the same segment bin, or
157 : : * DSA_SEGMENT_INDEX_NONE if this is the last one.
158 : : */
159 : : dsa_segment_index next;
160 : : /* The index of the bin that contains this segment. */
161 : : size_t bin;
162 : :
163 : : /*
164 : : * A flag raised to indicate that this segment is being returned to the
165 : : * operating system and has been unpinned.
166 : : */
167 : : bool freed;
168 : : } dsa_segment_header;
169 : :
170 : : /*
171 : : * Metadata for one superblock.
172 : : *
173 : : * For most blocks, span objects are stored out-of-line; that is, the span
174 : : * object is not stored within the block itself. But, as an exception, for a
175 : : * "span of spans", the span object is stored "inline". The allocation is
176 : : * always exactly one page, and the dsa_area_span object is located at
177 : : * the beginning of that page. The size class is DSA_SCLASS_BLOCK_OF_SPANS,
178 : : * and the remaining fields are used just as they would be in an ordinary
179 : : * block. We can't allocate spans out of ordinary superblocks because
180 : : * creating an ordinary superblock requires us to be able to allocate a span
181 : : * *first*. Doing it this way avoids that circularity.
182 : : */
183 : : typedef struct
184 : : {
185 : : dsa_pointer pool; /* Containing pool. */
186 : : dsa_pointer prevspan; /* Previous span. */
187 : : dsa_pointer nextspan; /* Next span. */
188 : : dsa_pointer start; /* Starting address. */
189 : : size_t npages; /* Length of span in pages. */
190 : : uint16 size_class; /* Size class. */
191 : : uint16 ninitialized; /* Maximum number of objects ever allocated. */
192 : : uint16 nallocatable; /* Number of objects currently allocatable. */
193 : : uint16 firstfree; /* First object on free list. */
194 : : uint16 nmax; /* Maximum number of objects ever possible. */
195 : : uint16 fclass; /* Current fullness class. */
196 : : } dsa_area_span;
197 : :
198 : : /*
199 : : * Given a pointer to an object in a span, access the index of the next free
200 : : * object in the same span (ie in the span's freelist) as an L-value.
201 : : */
202 : : #define NextFreeObjectIndex(object) (* (uint16 *) (object))
203 : :
204 : : /*
205 : : * Small allocations are handled by dividing a single block of memory into
206 : : * many small objects of equal size. The possible allocation sizes are
207 : : * defined by the following array. Larger size classes are spaced more widely
208 : : * than smaller size classes. We fudge the spacing for size classes >1kB to
209 : : * avoid space wastage: based on the knowledge that we plan to allocate 64kB
210 : : * blocks, we bump the maximum object size up to the largest multiple of
211 : : * 8 bytes that still lets us fit the same number of objects into one block.
212 : : *
213 : : * NB: Because of this fudging, if we were ever to use differently-sized blocks
214 : : * for small allocations, these size classes would need to be reworked to be
215 : : * optimal for the new size.
216 : : *
217 : : * NB: The optimal spacing for size classes, as well as the size of the blocks
218 : : * out of which small objects are allocated, is not a question that has one
219 : : * right answer. Some allocators (such as tcmalloc) use more closely-spaced
220 : : * size classes than we do here, while others (like aset.c) use more
221 : : * widely-spaced classes. Spacing the classes more closely avoids wasting
222 : : * memory within individual chunks, but also means a larger number of
223 : : * potentially-unfilled blocks.
224 : : */
225 : : static const uint16 dsa_size_classes[] = {
226 : : sizeof(dsa_area_span), 0, /* special size classes */
227 : : 8, 16, 24, 32, 40, 48, 56, 64, /* 8 classes separated by 8 bytes */
228 : : 80, 96, 112, 128, /* 4 classes separated by 16 bytes */
229 : : 160, 192, 224, 256, /* 4 classes separated by 32 bytes */
230 : : 320, 384, 448, 512, /* 4 classes separated by 64 bytes */
231 : : 640, 768, 896, 1024, /* 4 classes separated by 128 bytes */
232 : : 1280, 1560, 1816, 2048, /* 4 classes separated by ~256 bytes */
233 : : 2616, 3120, 3640, 4096, /* 4 classes separated by ~512 bytes */
234 : : 5456, 6552, 7280, 8192 /* 4 classes separated by ~1024 bytes */
235 : : };
236 : : #define DSA_NUM_SIZE_CLASSES lengthof(dsa_size_classes)
237 : :
238 : : /* Special size classes. */
239 : : #define DSA_SCLASS_BLOCK_OF_SPANS 0
240 : : #define DSA_SCLASS_SPAN_LARGE 1
241 : :
242 : : /*
243 : : * The following lookup table is used to map the size of small objects
244 : : * (less than 1kB) onto the corresponding size class. To use this table,
245 : : * round the size of the object up to the next multiple of 8 bytes, and then
246 : : * index into this array.
247 : : */
248 : : static const uint8 dsa_size_class_map[] = {
249 : : 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
250 : : 14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
251 : : 18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
252 : : 20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
253 : : 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
254 : : 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
255 : : 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
256 : : 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
257 : : };
258 : : #define DSA_SIZE_CLASS_MAP_QUANTUM 8
259 : :
260 : : /*
261 : : * Superblocks are binned by how full they are. Generally, each fullness
262 : : * class corresponds to one quartile, but the block being used for
263 : : * allocations is always at the head of the list for fullness class 1,
264 : : * regardless of how full it really is.
265 : : */
266 : : #define DSA_FULLNESS_CLASSES 4
267 : :
268 : : /*
269 : : * A dsa_area_pool represents a set of objects of a given size class.
270 : : *
271 : : * Perhaps there should be multiple pools for the same size class for
272 : : * contention avoidance, but for now there is just one!
273 : : */
274 : : typedef struct
275 : : {
276 : : /* A lock protecting access to this pool. */
277 : : LWLock lock;
278 : : /* A set of linked lists of spans, arranged by fullness. */
279 : : dsa_pointer spans[DSA_FULLNESS_CLASSES];
280 : : /* Should we pad this out to a cacheline boundary? */
281 : : } dsa_area_pool;
282 : :
283 : : /*
284 : : * The control block for an area. This lives in shared memory, at the start of
285 : : * the first DSM segment controlled by this area.
286 : : */
287 : : typedef struct
288 : : {
289 : : /* The segment header for the first segment. */
290 : : dsa_segment_header segment_header;
291 : : /* The handle for this area. */
292 : : dsa_handle handle;
293 : : /* The handles of the segments owned by this area. */
294 : : dsm_handle segment_handles[DSA_MAX_SEGMENTS];
295 : : /* Lists of segments, binned by maximum contiguous run of free pages. */
296 : : dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS];
297 : : /* The object pools for each size class. */
298 : : dsa_area_pool pools[DSA_NUM_SIZE_CLASSES];
299 : : /* initial allocation segment size */
300 : : size_t init_segment_size;
301 : : /* maximum allocation segment size */
302 : : size_t max_segment_size;
303 : : /* The total size of all active segments. */
304 : : size_t total_segment_size;
305 : : /* The maximum total size of backing storage we are allowed. */
306 : : size_t max_total_segment_size;
307 : : /* Highest used segment index in the history of this area. */
308 : : dsa_segment_index high_segment_index;
309 : : /* The reference count for this area. */
310 : : int refcnt;
311 : : /* A flag indicating that this area has been pinned. */
312 : : bool pinned;
313 : : /* The number of times that segments have been freed. */
314 : : size_t freed_segment_counter;
315 : : /* The LWLock tranche ID. */
316 : : int lwlock_tranche_id;
317 : : /* The general lock (protects everything except object pools). */
318 : : LWLock lock;
319 : : } dsa_area_control;
320 : :
321 : : /* Given a pointer to a pool, find a dsa_pointer. */
322 : : #define DsaAreaPoolToDsaPointer(area, p) \
323 : : DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
324 : :
325 : : /*
326 : : * A dsa_segment_map is stored within the backend-private memory of each
327 : : * individual backend. It holds the base address of the segment within that
328 : : * backend, plus the addresses of key objects within the segment. Those
329 : : * could instead be derived from the base address but it's handy to have them
330 : : * around.
331 : : */
332 : : typedef struct
333 : : {
334 : : dsm_segment *segment; /* DSM segment */
335 : : char *mapped_address; /* Address at which segment is mapped */
336 : : dsa_segment_header *header; /* Header (same as mapped_address) */
337 : : FreePageManager *fpm; /* Free page manager within segment. */
338 : : dsa_pointer *pagemap; /* Page map within segment. */
339 : : } dsa_segment_map;
340 : :
341 : : /*
342 : : * Per-backend state for a storage area. Backends obtain one of these by
343 : : * creating an area or attaching to an existing one using a handle. Each
344 : : * process that needs to use an area uses its own object to track where the
345 : : * segments are mapped.
346 : : */
347 : : struct dsa_area
348 : : {
349 : : /* Pointer to the control object in shared memory. */
350 : : dsa_area_control *control;
351 : :
352 : : /*
353 : : * All the mappings are owned by this. The dsa_area itself is not
354 : : * directly tracked by the ResourceOwner, but the effect is the same. NULL
355 : : * if the attachment has session lifespan, i.e if dsa_pin_mapping() has
356 : : * been called.
357 : : */
358 : : ResourceOwner resowner;
359 : :
360 : : /*
361 : : * This backend's array of segment maps, ordered by segment index
362 : : * corresponding to control->segment_handles. Some of the area's segments
363 : : * may not be mapped in this backend yet, and some slots may have been
364 : : * freed and need to be detached; these operations happen on demand.
365 : : */
366 : : dsa_segment_map segment_maps[DSA_MAX_SEGMENTS];
367 : :
368 : : /* The highest segment index this backend has ever mapped. */
369 : : dsa_segment_index high_segment_index;
370 : :
371 : : /* The last observed freed_segment_counter. */
372 : : size_t freed_segment_counter;
373 : : };
374 : :
375 : : #define DSA_SPAN_NOTHING_FREE ((uint16) -1)
376 : : #define DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
377 : :
378 : : /* Given a pointer to a segment_map, obtain a segment index number. */
379 : : #define get_segment_index(area, segment_map_ptr) \
380 : : (segment_map_ptr - &area->segment_maps[0])
381 : :
382 : : static void init_span(dsa_area *area, dsa_pointer span_pointer,
383 : : dsa_area_pool *pool, dsa_pointer start, size_t npages,
384 : : uint16 size_class);
385 : : static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool,
386 : : int fromclass, int toclass);
387 : : static inline dsa_pointer alloc_object(dsa_area *area, int size_class);
388 : : static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool,
389 : : int size_class);
390 : : static dsa_segment_map *get_segment_by_index(dsa_area *area,
391 : : dsa_segment_index index);
392 : : static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer);
393 : : static void unlink_span(dsa_area *area, dsa_area_span *span);
394 : : static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span,
395 : : dsa_pointer span_pointer, int fclass);
396 : : static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map);
397 : : static dsa_segment_map *get_best_segment(dsa_area *area, size_t npages);
398 : : static dsa_segment_map *make_new_segment(dsa_area *area, size_t requested_pages);
399 : : static dsa_area *create_internal(void *place, size_t size,
400 : : int tranche_id,
401 : : dsm_handle control_handle,
402 : : dsm_segment *control_segment,
403 : : size_t init_segment_size,
404 : : size_t max_segment_size);
405 : : static dsa_area *attach_internal(void *place, dsm_segment *segment,
406 : : dsa_handle handle);
407 : : static void check_for_freed_segments(dsa_area *area);
408 : : static void check_for_freed_segments_locked(dsa_area *area);
409 : : static void rebin_segment(dsa_area *area, dsa_segment_map *segment_map);
410 : :
411 : : /*
412 : : * Create a new shared area in a new DSM segment. Further DSM segments will
413 : : * be allocated as required to extend the available space.
414 : : *
415 : : * We can't allocate a LWLock tranche_id within this function, because tranche
416 : : * IDs are a scarce resource; there are only 64k available, using low numbers
417 : : * when possible matters, and we have no provision for recycling them. So,
418 : : * we require the caller to provide one.
419 : : */
420 : : dsa_area *
421 : 163 : dsa_create_ext(int tranche_id, size_t init_segment_size, size_t max_segment_size)
422 : : {
423 : : dsm_segment *segment;
424 : : dsa_area *area;
425 : :
426 : : /*
427 : : * Create the DSM segment that will hold the shared control object and the
428 : : * first segment of usable space.
429 : : */
430 : 163 : segment = dsm_create(init_segment_size, 0);
431 : :
432 : : /*
433 : : * All segments backing this area are pinned, so that DSA can explicitly
434 : : * control their lifetime (otherwise a newly created segment belonging to
435 : : * this area might be freed when the only backend that happens to have it
436 : : * mapped in ends, corrupting the area).
437 : : */
438 : 163 : dsm_pin_segment(segment);
439 : :
440 : : /* Create a new DSA area with the control object in this segment. */
441 : 163 : area = create_internal(dsm_segment_address(segment),
442 : : init_segment_size,
443 : : tranche_id,
444 : : dsm_segment_handle(segment), segment,
445 : : init_segment_size, max_segment_size);
446 : :
447 : : /* Clean up when the control segment detaches. */
448 : 163 : on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
449 : 163 : PointerGetDatum(dsm_segment_address(segment)));
450 : :
451 : 163 : return area;
452 : : }
453 : :
454 : : /*
455 : : * Create a new shared area in an existing shared memory space, which may be
456 : : * either DSM or Postmaster-initialized memory. DSM segments will be
457 : : * allocated as required to extend the available space, though that can be
458 : : * prevented with dsa_set_size_limit(area, size) using the same size provided
459 : : * to dsa_create_in_place.
460 : : *
461 : : * Areas created in-place must eventually be released by the backend that
462 : : * created them and all backends that attach to them. This can be done
463 : : * explicitly with dsa_release_in_place, or, in the special case that 'place'
464 : : * happens to be in a pre-existing DSM segment, by passing in a pointer to the
465 : : * segment so that a detach hook can be registered with the containing DSM
466 : : * segment.
467 : : *
468 : : * See dsa_create() for a note about the tranche arguments.
469 : : */
470 : : dsa_area *
471 : 1891 : dsa_create_in_place_ext(void *place, size_t size,
472 : : int tranche_id, dsm_segment *segment,
473 : : size_t init_segment_size, size_t max_segment_size)
474 : : {
475 : : dsa_area *area;
476 : :
477 : 1891 : area = create_internal(place, size, tranche_id,
478 : : DSM_HANDLE_INVALID, NULL,
479 : : init_segment_size, max_segment_size);
480 : :
481 : : /*
482 : : * Clean up when the control segment detaches, if a containing DSM segment
483 : : * was provided.
484 : : */
485 [ + + ]: 1891 : if (segment != NULL)
486 : 641 : on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
487 : : PointerGetDatum(place));
488 : :
489 : 1891 : return area;
490 : : }
491 : :
492 : : /*
493 : : * Obtain a handle that can be passed to other processes so that they can
494 : : * attach to the given area. Cannot be called for areas created with
495 : : * dsa_create_in_place.
496 : : */
497 : : dsa_handle
498 : 149 : dsa_get_handle(dsa_area *area)
499 : : {
500 : : Assert(area->control->handle != DSA_HANDLE_INVALID);
501 : 149 : return area->control->handle;
502 : : }
503 : :
504 : : /*
505 : : * Attach to an area given a handle generated (possibly in another process) by
506 : : * dsa_get_handle. The area must have been created with dsa_create (not
507 : : * dsa_create_in_place).
508 : : */
509 : : dsa_area *
510 : 443 : dsa_attach(dsa_handle handle)
511 : : {
512 : : dsm_segment *segment;
513 : : dsa_area *area;
514 : :
515 : : /*
516 : : * An area handle is really a DSM segment handle for the first segment, so
517 : : * we go ahead and attach to that.
518 : : */
519 : 443 : segment = dsm_attach(handle);
520 [ - + ]: 443 : if (segment == NULL)
521 [ # # ]: 0 : ereport(ERROR,
522 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
523 : : errmsg("could not attach to dynamic shared area")));
524 : :
525 : 443 : area = attach_internal(dsm_segment_address(segment), segment, handle);
526 : :
527 : : /* Clean up when the control segment detaches. */
528 : 443 : on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
529 : 443 : PointerGetDatum(dsm_segment_address(segment)));
530 : :
531 : 443 : return area;
532 : : }
533 : :
534 : : /*
535 : : * Returns whether the area with the given handle was already attached by the
536 : : * current process. The area must have been created with dsa_create (not
537 : : * dsa_create_in_place).
538 : : */
539 : : bool
540 : 14 : dsa_is_attached(dsa_handle handle)
541 : : {
542 : : /*
543 : : * An area handle is really a DSM segment handle for the first segment, so
544 : : * we can just search for that.
545 : : */
546 : 14 : return dsm_find_mapping(handle) != NULL;
547 : : }
548 : :
549 : : /*
550 : : * Attach to an area that was created with dsa_create_in_place. The caller
551 : : * must somehow know the location in memory that was used when the area was
552 : : * created, though it may be mapped at a different virtual address in this
553 : : * process.
554 : : *
555 : : * See dsa_create_in_place for note about releasing in-place areas, and the
556 : : * optional 'segment' argument which can be provided to allow automatic
557 : : * release if the containing memory happens to be a DSM segment.
558 : : */
559 : : dsa_area *
560 : 28734 : dsa_attach_in_place(void *place, dsm_segment *segment)
561 : : {
562 : : dsa_area *area;
563 : :
564 : 28734 : area = attach_internal(place, NULL, DSA_HANDLE_INVALID);
565 : :
566 : : /*
567 : : * Clean up when the control segment detaches, if a containing DSM segment
568 : : * was provided.
569 : : */
570 [ + + ]: 28734 : if (segment != NULL)
571 : 3826 : on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
572 : : PointerGetDatum(place));
573 : :
574 : 28734 : return area;
575 : : }
576 : :
577 : : /*
578 : : * Release a DSA area that was produced by dsa_create_in_place or
579 : : * dsa_attach_in_place. The 'segment' argument is ignored but provides an
580 : : * interface suitable for on_dsm_detach, for the convenience of users who want
581 : : * to create a DSA segment inside an existing DSM segment and have it
582 : : * automatically released when the containing DSM segment is detached.
583 : : * 'place' should be the address of the place where the area was created.
584 : : *
585 : : * This callback is automatically registered for the DSM segment containing
586 : : * the control object of in-place areas when a segment is provided to
587 : : * dsa_create_in_place or dsa_attach_in_place, and also for all areas created
588 : : * with dsa_create.
589 : : */
590 : : void
591 : 5073 : dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
592 : : {
593 : 5073 : dsa_release_in_place(DatumGetPointer(place));
594 : 5073 : }
595 : :
596 : : /*
597 : : * Release a DSA area that was produced by dsa_create_in_place or
598 : : * dsa_attach_in_place. The 'code' argument is ignored but provides an
599 : : * interface suitable for on_shmem_exit or before_shmem_exit, for the
600 : : * convenience of users who want to create a DSA segment inside shared memory
601 : : * other than a DSM segment and have it automatically release at backend exit.
602 : : * 'place' should be the address of the place where the area was created.
603 : : */
604 : : void
605 : 0 : dsa_on_shmem_exit_release_in_place(int code, Datum place)
606 : : {
607 : 0 : dsa_release_in_place(DatumGetPointer(place));
608 : 0 : }
609 : :
610 : : /*
611 : : * Release a DSA area that was produced by dsa_create_in_place or
612 : : * dsa_attach_in_place. It is preferable to use one of the 'dsa_on_XXX'
613 : : * callbacks so that this is managed automatically, because failure to release
614 : : * an area created in-place leaks its segments permanently.
615 : : *
616 : : * This is also called automatically for areas produced by dsa_create or
617 : : * dsa_attach as an implementation detail.
618 : : */
619 : : void
620 : 29981 : dsa_release_in_place(void *place)
621 : : {
622 : 29981 : dsa_area_control *control = (dsa_area_control *) place;
623 : :
624 : 29981 : LWLockAcquire(&control->lock, LW_EXCLUSIVE);
625 : : Assert(control->segment_header.magic ==
626 : : (DSA_SEGMENT_HEADER_MAGIC ^ control->handle ^ 0));
627 : : Assert(control->refcnt > 0);
628 [ + + ]: 29981 : if (--control->refcnt == 0)
629 : : {
630 [ + + ]: 1605 : for (dsa_segment_index i = 0; i <= control->high_segment_index; ++i)
631 : : {
632 : : dsm_handle handle;
633 : :
634 : 903 : handle = control->segment_handles[i];
635 [ + + ]: 903 : if (handle != DSM_HANDLE_INVALID)
636 : 262 : dsm_unpin_segment(handle);
637 : : }
638 : : }
639 : 29981 : LWLockRelease(&control->lock);
640 : 29981 : }
641 : :
642 : : /*
643 : : * Keep a DSA area attached until end of session or explicit detach.
644 : : *
645 : : * By default, areas are owned by the current resource owner, which means they
646 : : * are detached automatically when that scope ends.
647 : : */
648 : : void
649 : 27538 : dsa_pin_mapping(dsa_area *area)
650 : : {
651 [ + + ]: 27538 : if (area->resowner != NULL)
652 : : {
653 : 2552 : area->resowner = NULL;
654 : :
655 [ + + ]: 5134 : for (dsa_segment_index i = 0; i <= area->high_segment_index; ++i)
656 [ + + ]: 2582 : if (area->segment_maps[i].segment != NULL)
657 : 451 : dsm_pin_mapping(area->segment_maps[i].segment);
658 : : }
659 : 27538 : }
660 : :
661 : : /*
662 : : * Allocate memory in this storage area. The return value is a dsa_pointer
663 : : * that can be passed to other processes, and converted to a local pointer
664 : : * with dsa_get_address. 'flags' is a bitmap which should be constructed
665 : : * from the following values:
666 : : *
667 : : * DSA_ALLOC_HUGE allows allocations >= 1GB. Otherwise, such allocations
668 : : * will result in an ERROR.
669 : : *
670 : : * DSA_ALLOC_NO_OOM causes this function to return InvalidDsaPointer when
671 : : * no memory is available or a size limit established by dsa_set_size_limit
672 : : * would be exceeded. Otherwise, such allocations will result in an ERROR.
673 : : *
674 : : * DSA_ALLOC_ZERO causes the allocated memory to be zeroed. Otherwise, the
675 : : * contents of newly-allocated memory are indeterminate.
676 : : *
677 : : * These flags correspond to similarly named flags used by
678 : : * MemoryContextAllocExtended(). See also the macros dsa_allocate and
679 : : * dsa_allocate0 which expand to a call to this function with commonly used
680 : : * flags.
681 : : */
682 : : dsa_pointer
683 : 818647 : dsa_allocate_extended(dsa_area *area, size_t size, int flags)
684 : : {
685 : : uint16 size_class;
686 : : dsa_pointer start_pointer;
687 : : dsa_segment_map *segment_map;
688 : : dsa_pointer result;
689 : :
690 : : Assert(size > 0);
691 : :
692 : : /* Sanity check on huge individual allocation size. */
693 [ + + + - ]: 818647 : if (((flags & DSA_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
694 [ + + - + ]: 818647 : ((flags & DSA_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
695 [ # # ]: 0 : elog(ERROR, "invalid DSA memory alloc request size %zu", size);
696 : :
697 : : /*
698 : : * If bigger than the largest size class, just grab a run of pages from
699 : : * the free page manager, instead of allocating an object from a pool.
700 : : * There will still be a span, but it's a special class of span that
701 : : * manages this whole allocation and simply gives all pages back to the
702 : : * free page manager when dsa_free is called.
703 : : */
704 [ + + ]: 818647 : if (size > dsa_size_classes[lengthof(dsa_size_classes) - 1])
705 : : {
706 : 3692 : size_t npages = fpm_size_to_pages(size);
707 : : size_t first_page;
708 : : dsa_pointer span_pointer;
709 : 3692 : dsa_area_pool *pool = &area->control->pools[DSA_SCLASS_SPAN_LARGE];
710 : :
711 : : /* Obtain a span object. */
712 : 3692 : span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
713 [ - + ]: 3692 : if (!DsaPointerIsValid(span_pointer))
714 : : {
715 : : /* Raise error unless asked not to. */
716 [ # # ]: 0 : if ((flags & DSA_ALLOC_NO_OOM) == 0)
717 [ # # ]: 0 : ereport(ERROR,
718 : : (errcode(ERRCODE_OUT_OF_MEMORY),
719 : : errmsg("out of memory"),
720 : : errdetail("Failed on DSA request of size %zu.",
721 : : size)));
722 : 0 : return InvalidDsaPointer;
723 : : }
724 : :
725 : 3692 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
726 : :
727 : : /* Find a segment from which to allocate. */
728 : 3692 : segment_map = get_best_segment(area, npages);
729 [ + + ]: 3692 : if (segment_map == NULL)
730 : 40 : segment_map = make_new_segment(area, npages);
731 [ - + ]: 3692 : if (segment_map == NULL)
732 : : {
733 : : /* Can't make any more segments: game over. */
734 : 0 : LWLockRelease(DSA_AREA_LOCK(area));
735 : 0 : dsa_free(area, span_pointer);
736 : :
737 : : /* Raise error unless asked not to. */
738 [ # # ]: 0 : if ((flags & DSA_ALLOC_NO_OOM) == 0)
739 [ # # ]: 0 : ereport(ERROR,
740 : : (errcode(ERRCODE_OUT_OF_MEMORY),
741 : : errmsg("out of memory"),
742 : : errdetail("Failed on DSA request of size %zu.",
743 : : size)));
744 : 0 : return InvalidDsaPointer;
745 : : }
746 : :
747 : : /*
748 : : * Ask the free page manager for a run of pages. This should always
749 : : * succeed, since both get_best_segment and make_new_segment should
750 : : * only return a non-NULL pointer if it actually contains enough
751 : : * contiguous freespace. If it does fail, something in our backend
752 : : * private state is out of whack, so use FATAL to kill the process.
753 : : */
754 [ - + ]: 3692 : if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
755 [ # # ]: 0 : elog(FATAL,
756 : : "dsa_allocate could not find %zu free pages", npages);
757 : 3692 : LWLockRelease(DSA_AREA_LOCK(area));
758 : :
759 : 3692 : start_pointer = DSA_MAKE_POINTER(get_segment_index(area, segment_map),
760 : : first_page * FPM_PAGE_SIZE);
761 : :
762 : : /* Initialize span and pagemap. */
763 : 3692 : LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
764 : : LW_EXCLUSIVE);
765 : 3692 : init_span(area, span_pointer, pool, start_pointer, npages,
766 : : DSA_SCLASS_SPAN_LARGE);
767 : 3692 : segment_map->pagemap[first_page] = span_pointer;
768 : 3692 : LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
769 : :
770 : : /* Zero-initialize the memory if requested. */
771 [ + + ]: 3692 : if ((flags & DSA_ALLOC_ZERO) != 0)
772 : 900 : memset(dsa_get_address(area, start_pointer), 0, size);
773 : :
774 : 3692 : return start_pointer;
775 : : }
776 : :
777 : : /* Map allocation to a size class. */
778 [ + + ]: 814955 : if (size < lengthof(dsa_size_class_map) * DSA_SIZE_CLASS_MAP_QUANTUM)
779 : : {
780 : : int mapidx;
781 : :
782 : : /* For smaller sizes we have a lookup table... */
783 : 787885 : mapidx = ((size + DSA_SIZE_CLASS_MAP_QUANTUM - 1) /
784 : 787885 : DSA_SIZE_CLASS_MAP_QUANTUM) - 1;
785 : 787885 : size_class = dsa_size_class_map[mapidx];
786 : : }
787 : : else
788 : : {
789 : : uint16 min;
790 : : uint16 max;
791 : :
792 : : /* ... and for the rest we search by binary chop. */
793 : 27070 : min = dsa_size_class_map[lengthof(dsa_size_class_map) - 1];
794 : 27070 : max = lengthof(dsa_size_classes) - 1;
795 : :
796 [ + + ]: 132859 : while (min < max)
797 : : {
798 : 105789 : uint16 mid = (min + max) / 2;
799 : 105789 : uint16 class_size = dsa_size_classes[mid];
800 : :
801 [ + + ]: 105789 : if (class_size < size)
802 : 29839 : min = mid + 1;
803 : : else
804 : 75950 : max = mid;
805 : : }
806 : :
807 : 27070 : size_class = min;
808 : : }
809 : : Assert(size <= dsa_size_classes[size_class]);
810 : : Assert(size_class == 0 || size > dsa_size_classes[size_class - 1]);
811 : :
812 : : /* Attempt to allocate an object from the appropriate pool. */
813 : 814955 : result = alloc_object(area, size_class);
814 : :
815 : : /* Check for failure to allocate. */
816 [ - + ]: 814955 : if (!DsaPointerIsValid(result))
817 : : {
818 : : /* Raise error unless asked not to. */
819 [ # # ]: 0 : if ((flags & DSA_ALLOC_NO_OOM) == 0)
820 [ # # ]: 0 : ereport(ERROR,
821 : : (errcode(ERRCODE_OUT_OF_MEMORY),
822 : : errmsg("out of memory"),
823 : : errdetail("Failed on DSA request of size %zu.", size)));
824 : 0 : return InvalidDsaPointer;
825 : : }
826 : :
827 : : /* Zero-initialize the memory if requested. */
828 [ + + ]: 814955 : if ((flags & DSA_ALLOC_ZERO) != 0)
829 : 402960 : memset(dsa_get_address(area, result), 0, size);
830 : :
831 : 814955 : return result;
832 : : }
833 : :
834 : : /*
835 : : * Free memory obtained with dsa_allocate.
836 : : */
837 : : void
838 : 158713 : dsa_free(dsa_area *area, dsa_pointer dp)
839 : : {
840 : : dsa_segment_map *segment_map;
841 : : int pageno;
842 : : dsa_pointer span_pointer;
843 : : dsa_area_span *span;
844 : : char *superblock;
845 : : char *object;
846 : : size_t size;
847 : : int size_class;
848 : :
849 : : /* Make sure we don't have a stale segment in the slot 'dp' refers to. */
850 : 158713 : check_for_freed_segments(area);
851 : :
852 : : /* Locate the object, span and pool. */
853 : 158713 : segment_map = get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(dp));
854 : 158713 : pageno = DSA_EXTRACT_OFFSET(dp) / FPM_PAGE_SIZE;
855 : 158713 : span_pointer = segment_map->pagemap[pageno];
856 : 158713 : span = dsa_get_address(area, span_pointer);
857 : 158713 : superblock = dsa_get_address(area, span->start);
858 : 158713 : object = dsa_get_address(area, dp);
859 : 158713 : size_class = span->size_class;
860 : 158713 : size = dsa_size_classes[size_class];
861 : :
862 : : /*
863 : : * Special case for large objects that live in a special span: we return
864 : : * those pages directly to the free page manager and free the span.
865 : : */
866 [ + + ]: 158713 : if (span->size_class == DSA_SCLASS_SPAN_LARGE)
867 : : {
868 : :
869 : : #ifdef CLOBBER_FREED_MEMORY
870 : : memset(object, 0x7f, span->npages * FPM_PAGE_SIZE);
871 : : #endif
872 : :
873 : : /* Give pages back to free page manager. */
874 : 2914 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
875 : 2914 : FreePageManagerPut(segment_map->fpm,
876 : 2914 : DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
877 : : span->npages);
878 : :
879 : : /* Move segment to appropriate bin if necessary. */
880 : 2914 : rebin_segment(area, segment_map);
881 : 2914 : LWLockRelease(DSA_AREA_LOCK(area));
882 : :
883 : : /* Unlink span. */
884 : 2914 : LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
885 : : LW_EXCLUSIVE);
886 : 2914 : unlink_span(area, span);
887 : 2914 : LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
888 : : /* Free the span object so it can be reused. */
889 : 2914 : dsa_free(area, span_pointer);
890 : 2914 : return;
891 : : }
892 : :
893 : : #ifdef CLOBBER_FREED_MEMORY
894 : : memset(object, 0x7f, size);
895 : : #endif
896 : :
897 : 155799 : LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
898 : :
899 : : /* Put the object on the span's freelist. */
900 : : Assert(object >= superblock);
901 : : Assert(object < superblock + DSA_SUPERBLOCK_SIZE);
902 : : Assert((object - superblock) % size == 0);
903 : 155799 : NextFreeObjectIndex(object) = span->firstfree;
904 : 155799 : span->firstfree = (object - superblock) / size;
905 : 155799 : ++span->nallocatable;
906 : :
907 : : /*
908 : : * See if the span needs to moved to a different fullness class, or be
909 : : * freed so its pages can be given back to the segment.
910 : : */
911 [ + + + - ]: 155799 : if (span->nallocatable == 1 && span->fclass == DSA_FULLNESS_CLASSES - 1)
912 : : {
913 : : /*
914 : : * The block was completely full and is located in the
915 : : * highest-numbered fullness class, which is never scanned for free
916 : : * chunks. We must move it to the next-lower fullness class.
917 : : */
918 : 258 : unlink_span(area, span);
919 : 258 : add_span_to_fullness_class(area, span, span_pointer,
920 : : DSA_FULLNESS_CLASSES - 2);
921 : :
922 : : /*
923 : : * If this is the only span, and there is no active span, then we
924 : : * should probably move this span to fullness class 1. (Otherwise if
925 : : * you allocate exactly all the objects in the only span, it moves to
926 : : * class 3, then you free them all, it moves to 2, and then is given
927 : : * back, leaving no active span).
928 : : */
929 : : }
930 [ + + ]: 155541 : else if (span->nallocatable == span->nmax &&
931 [ + + - + ]: 5894 : (span->fclass != 1 || span->prevspan != InvalidDsaPointer))
932 : : {
933 : : /*
934 : : * This entire block is free, and it's not the active block for this
935 : : * size class. Return the memory to the free page manager. We don't
936 : : * do this for the active block to prevent hysteresis: if we
937 : : * repeatedly allocate and free the only chunk in the active block, it
938 : : * will be very inefficient if we deallocate and reallocate the block
939 : : * every time.
940 : : */
941 : 16 : destroy_superblock(area, span_pointer);
942 : : }
943 : :
944 : 155799 : LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
945 : : }
946 : :
947 : : /*
948 : : * Obtain a backend-local address for a dsa_pointer. 'dp' must point to
949 : : * memory allocated by the given area (possibly in another process) that
950 : : * hasn't yet been freed. This may cause a segment to be mapped into the
951 : : * current process if required, and may cause freed segments to be unmapped.
952 : : */
953 : : void *
954 : 11645813 : dsa_get_address(dsa_area *area, dsa_pointer dp)
955 : : {
956 : : dsa_segment_index index;
957 : : size_t offset;
958 : :
959 : : /* Convert InvalidDsaPointer to NULL. */
960 [ + + ]: 11645813 : if (!DsaPointerIsValid(dp))
961 : 1853672 : return NULL;
962 : :
963 : : /* Process any requests to detach from freed segments. */
964 : 9792141 : check_for_freed_segments(area);
965 : :
966 : : /* Break the dsa_pointer into its components. */
967 : 9792141 : index = DSA_EXTRACT_SEGMENT_NUMBER(dp);
968 : 9792141 : offset = DSA_EXTRACT_OFFSET(dp);
969 : : Assert(index < DSA_MAX_SEGMENTS);
970 : :
971 : : /* Check if we need to cause this segment to be mapped in. */
972 [ + + ]: 9792141 : if (unlikely(area->segment_maps[index].mapped_address == NULL))
973 : : {
974 : : /* Call for effect (we don't need the result). */
975 : 22387 : get_segment_by_index(area, index);
976 : : }
977 : :
978 : 9792141 : return area->segment_maps[index].mapped_address + offset;
979 : : }
980 : :
981 : : /*
982 : : * Pin this area, so that it will continue to exist even if all backends
983 : : * detach from it. In that case, the area can still be reattached to if a
984 : : * handle has been recorded somewhere.
985 : : */
986 : : void
987 : 1352 : dsa_pin(dsa_area *area)
988 : : {
989 : 1352 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
990 [ - + ]: 1352 : if (area->control->pinned)
991 : : {
992 : 0 : LWLockRelease(DSA_AREA_LOCK(area));
993 [ # # ]: 0 : elog(ERROR, "dsa_area already pinned");
994 : : }
995 : 1352 : area->control->pinned = true;
996 : 1352 : ++area->control->refcnt;
997 : 1352 : LWLockRelease(DSA_AREA_LOCK(area));
998 : 1352 : }
999 : :
1000 : : /*
1001 : : * Undo the effects of dsa_pin, so that the given area can be freed when no
1002 : : * backends are attached to it. May be called only if dsa_pin has been
1003 : : * called.
1004 : : */
1005 : : void
1006 : 0 : dsa_unpin(dsa_area *area)
1007 : : {
1008 : 0 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
1009 : : Assert(area->control->refcnt > 1);
1010 [ # # ]: 0 : if (!area->control->pinned)
1011 : : {
1012 : 0 : LWLockRelease(DSA_AREA_LOCK(area));
1013 [ # # ]: 0 : elog(ERROR, "dsa_area not pinned");
1014 : : }
1015 : 0 : area->control->pinned = false;
1016 : 0 : --area->control->refcnt;
1017 : 0 : LWLockRelease(DSA_AREA_LOCK(area));
1018 : 0 : }
1019 : :
1020 : : /*
1021 : : * Set the total size limit for this area. This limit is checked whenever new
1022 : : * segments need to be allocated from the operating system. If the new size
1023 : : * limit is already exceeded, this has no immediate effect.
1024 : : *
1025 : : * Note that the total virtual memory usage may be temporarily larger than
1026 : : * this limit when segments have been freed, but not yet detached by all
1027 : : * backends that have attached to them.
1028 : : */
1029 : : void
1030 : 2500 : dsa_set_size_limit(dsa_area *area, size_t limit)
1031 : : {
1032 : 2500 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
1033 : 2500 : area->control->max_total_segment_size = limit;
1034 : 2500 : LWLockRelease(DSA_AREA_LOCK(area));
1035 : 2500 : }
1036 : :
1037 : : /* Return the total size of all active segments */
1038 : : size_t
1039 : 1570 : dsa_get_total_size(dsa_area *area)
1040 : : {
1041 : : size_t size;
1042 : :
1043 : 1570 : LWLockAcquire(DSA_AREA_LOCK(area), LW_SHARED);
1044 : 1570 : size = area->control->total_segment_size;
1045 : 1570 : LWLockRelease(DSA_AREA_LOCK(area));
1046 : :
1047 : 1570 : return size;
1048 : : }
1049 : :
1050 : : /*
1051 : : * Same as dsa_get_total_size(), but accepts a DSA handle. The area must have
1052 : : * been created with dsa_create (not dsa_create_in_place).
1053 : : */
1054 : : size_t
1055 : 2 : dsa_get_total_size_from_handle(dsa_handle handle)
1056 : : {
1057 : : size_t size;
1058 : : bool already_attached;
1059 : : dsm_segment *segment;
1060 : : dsa_area_control *control;
1061 : :
1062 : 2 : already_attached = dsa_is_attached(handle);
1063 [ - + ]: 2 : if (already_attached)
1064 : 0 : segment = dsm_find_mapping(handle);
1065 : : else
1066 : 2 : segment = dsm_attach(handle);
1067 : :
1068 [ - + ]: 2 : if (segment == NULL)
1069 [ # # ]: 0 : ereport(ERROR,
1070 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1071 : : errmsg("could not attach to dynamic shared area")));
1072 : :
1073 : 2 : control = (dsa_area_control *) dsm_segment_address(segment);
1074 : :
1075 : 2 : LWLockAcquire(&control->lock, LW_SHARED);
1076 : 2 : size = control->total_segment_size;
1077 : 2 : LWLockRelease(&control->lock);
1078 : :
1079 [ + - ]: 2 : if (!already_attached)
1080 : 2 : dsm_detach(segment);
1081 : :
1082 : 2 : return size;
1083 : : }
1084 : :
1085 : : /*
1086 : : * Aggressively free all spare memory in the hope of returning DSM segments to
1087 : : * the operating system.
1088 : : */
1089 : : void
1090 : 0 : dsa_trim(dsa_area *area)
1091 : : {
1092 : : int size_class;
1093 : :
1094 : : /*
1095 : : * Trim in reverse pool order so we get to the spans-of-spans last, just
1096 : : * in case any become entirely free while processing all the other pools.
1097 : : */
1098 [ # # ]: 0 : for (size_class = DSA_NUM_SIZE_CLASSES - 1; size_class >= 0; --size_class)
1099 : : {
1100 : 0 : dsa_area_pool *pool = &area->control->pools[size_class];
1101 : : dsa_pointer span_pointer;
1102 : :
1103 [ # # ]: 0 : if (size_class == DSA_SCLASS_SPAN_LARGE)
1104 : : {
1105 : : /* Large object frees give back segments aggressively already. */
1106 : 0 : continue;
1107 : : }
1108 : :
1109 : : /*
1110 : : * Search fullness class 1 only. That is where we expect to find an
1111 : : * entirely empty superblock (entirely empty superblocks in other
1112 : : * fullness classes are returned to the free page map by dsa_free).
1113 : : */
1114 : 0 : LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
1115 : 0 : span_pointer = pool->spans[1];
1116 [ # # ]: 0 : while (DsaPointerIsValid(span_pointer))
1117 : : {
1118 : 0 : dsa_area_span *span = dsa_get_address(area, span_pointer);
1119 : 0 : dsa_pointer next = span->nextspan;
1120 : :
1121 [ # # ]: 0 : if (span->nallocatable == span->nmax)
1122 : 0 : destroy_superblock(area, span_pointer);
1123 : :
1124 : 0 : span_pointer = next;
1125 : : }
1126 : 0 : LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
1127 : : }
1128 : 0 : }
1129 : :
1130 : : /*
1131 : : * Print out debugging information about the internal state of the shared
1132 : : * memory area.
1133 : : */
1134 : : void
1135 : 0 : dsa_dump(dsa_area *area)
1136 : : {
1137 : : size_t i,
1138 : : j;
1139 : :
1140 : : /*
1141 : : * Note: This gives an inconsistent snapshot as it acquires and releases
1142 : : * individual locks as it goes...
1143 : : */
1144 : :
1145 : 0 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
1146 : 0 : check_for_freed_segments_locked(area);
1147 : 0 : fprintf(stderr, "dsa_area handle %x:\n", area->control->handle);
1148 : 0 : fprintf(stderr, " max_total_segment_size: %zu\n",
1149 : 0 : area->control->max_total_segment_size);
1150 : 0 : fprintf(stderr, " total_segment_size: %zu\n",
1151 : 0 : area->control->total_segment_size);
1152 : 0 : fprintf(stderr, " refcnt: %d\n", area->control->refcnt);
1153 [ # # ]: 0 : fprintf(stderr, " pinned: %c\n", area->control->pinned ? 't' : 'f');
1154 : 0 : fprintf(stderr, " segment bins:\n");
1155 [ # # ]: 0 : for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
1156 : : {
1157 [ # # ]: 0 : if (area->control->segment_bins[i] != DSA_SEGMENT_INDEX_NONE)
1158 : : {
1159 : : dsa_segment_index segment_index;
1160 : :
1161 [ # # ]: 0 : if (i == 0)
1162 : 0 : fprintf(stderr,
1163 : : " segment bin %zu (no contiguous free pages):\n", i);
1164 : : else
1165 : 0 : fprintf(stderr,
1166 : : " segment bin %zu (at least %d contiguous pages free):\n",
1167 : 0 : i, 1 << (i - 1));
1168 : 0 : segment_index = area->control->segment_bins[i];
1169 [ # # ]: 0 : while (segment_index != DSA_SEGMENT_INDEX_NONE)
1170 : : {
1171 : : dsa_segment_map *segment_map;
1172 : :
1173 : : segment_map =
1174 : 0 : get_segment_by_index(area, segment_index);
1175 : :
1176 : 0 : fprintf(stderr,
1177 : : " segment index %zu, usable_pages = %zu, "
1178 : : "contiguous_pages = %zu, mapped at %p\n",
1179 : : segment_index,
1180 : 0 : segment_map->header->usable_pages,
1181 : 0 : fpm_largest(segment_map->fpm),
1182 : : segment_map->mapped_address);
1183 : 0 : segment_index = segment_map->header->next;
1184 : : }
1185 : : }
1186 : : }
1187 : 0 : LWLockRelease(DSA_AREA_LOCK(area));
1188 : :
1189 : 0 : fprintf(stderr, " pools:\n");
1190 [ # # ]: 0 : for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
1191 : : {
1192 : 0 : bool found = false;
1193 : :
1194 : 0 : LWLockAcquire(DSA_SCLASS_LOCK(area, i), LW_EXCLUSIVE);
1195 [ # # ]: 0 : for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
1196 [ # # ]: 0 : if (DsaPointerIsValid(area->control->pools[i].spans[j]))
1197 : 0 : found = true;
1198 [ # # ]: 0 : if (found)
1199 : : {
1200 [ # # ]: 0 : if (i == DSA_SCLASS_BLOCK_OF_SPANS)
1201 : 0 : fprintf(stderr, " pool for blocks of span objects:\n");
1202 [ # # ]: 0 : else if (i == DSA_SCLASS_SPAN_LARGE)
1203 : 0 : fprintf(stderr, " pool for large object spans:\n");
1204 : : else
1205 : 0 : fprintf(stderr,
1206 : : " pool for size class %zu (object size %hu bytes):\n",
1207 : 0 : i, dsa_size_classes[i]);
1208 [ # # ]: 0 : for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
1209 : : {
1210 [ # # ]: 0 : if (!DsaPointerIsValid(area->control->pools[i].spans[j]))
1211 : 0 : fprintf(stderr, " fullness class %zu is empty\n", j);
1212 : : else
1213 : : {
1214 : 0 : dsa_pointer span_pointer = area->control->pools[i].spans[j];
1215 : :
1216 : 0 : fprintf(stderr, " fullness class %zu:\n", j);
1217 [ # # ]: 0 : while (DsaPointerIsValid(span_pointer))
1218 : : {
1219 : : dsa_area_span *span;
1220 : :
1221 : 0 : span = dsa_get_address(area, span_pointer);
1222 : 0 : fprintf(stderr,
1223 : : " span descriptor at "
1224 : : DSA_POINTER_FORMAT ", superblock at "
1225 : : DSA_POINTER_FORMAT
1226 : : ", pages = %zu, objects free = %hu/%hu\n",
1227 : : span_pointer, span->start, span->npages,
1228 : 0 : span->nallocatable, span->nmax);
1229 : 0 : span_pointer = span->nextspan;
1230 : : }
1231 : : }
1232 : : }
1233 : : }
1234 : 0 : LWLockRelease(DSA_SCLASS_LOCK(area, i));
1235 : : }
1236 : 0 : }
1237 : :
1238 : : /*
1239 : : * Return the smallest size that you can successfully provide to
1240 : : * dsa_create_in_place.
1241 : : */
1242 : : size_t
1243 : 2577 : dsa_minimum_size(void)
1244 : : {
1245 : : size_t size;
1246 : 2577 : size_t pages = 0;
1247 : :
1248 : 2577 : size = MAXALIGN(sizeof(dsa_area_control)) +
1249 : : MAXALIGN(sizeof(FreePageManager));
1250 : :
1251 : : /* Figure out how many pages we need, including the page map... */
1252 [ + + ]: 7731 : while (((size + FPM_PAGE_SIZE - 1) / FPM_PAGE_SIZE) > pages)
1253 : : {
1254 : 5154 : ++pages;
1255 : 5154 : size += sizeof(dsa_pointer);
1256 : : }
1257 : :
1258 : 2577 : return pages * FPM_PAGE_SIZE;
1259 : : }
1260 : :
1261 : : /*
1262 : : * Workhorse function for dsa_create and dsa_create_in_place.
1263 : : */
1264 : : static dsa_area *
1265 : 2054 : create_internal(void *place, size_t size,
1266 : : int tranche_id,
1267 : : dsm_handle control_handle,
1268 : : dsm_segment *control_segment,
1269 : : size_t init_segment_size, size_t max_segment_size)
1270 : : {
1271 : : dsa_area_control *control;
1272 : : dsa_area *area;
1273 : : dsa_segment_map *segment_map;
1274 : : size_t usable_pages;
1275 : : size_t total_pages;
1276 : : size_t metadata_bytes;
1277 : :
1278 : : /* Check the initial and maximum block sizes */
1279 : : Assert(init_segment_size >= DSA_MIN_SEGMENT_SIZE);
1280 : : Assert(max_segment_size >= init_segment_size);
1281 : : Assert(max_segment_size <= DSA_MAX_SEGMENT_SIZE);
1282 : :
1283 : : /* Sanity check on the space we have to work in. */
1284 [ - + ]: 2054 : if (size < dsa_minimum_size())
1285 [ # # ]: 0 : elog(ERROR, "dsa_area space must be at least %zu, but %zu provided",
1286 : : dsa_minimum_size(), size);
1287 : :
1288 : : /* Now figure out how much space is usable */
1289 : 2054 : total_pages = size / FPM_PAGE_SIZE;
1290 : 2054 : metadata_bytes =
1291 : : MAXALIGN(sizeof(dsa_area_control)) +
1292 : 2054 : MAXALIGN(sizeof(FreePageManager)) +
1293 : : total_pages * sizeof(dsa_pointer);
1294 : : /* Add padding up to next page boundary. */
1295 [ + - ]: 2054 : if (metadata_bytes % FPM_PAGE_SIZE != 0)
1296 : 2054 : metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
1297 : : Assert(metadata_bytes <= size);
1298 : 2054 : usable_pages = (size - metadata_bytes) / FPM_PAGE_SIZE;
1299 : :
1300 : : /*
1301 : : * Initialize the dsa_area_control object located at the start of the
1302 : : * space.
1303 : : */
1304 : 2054 : control = (dsa_area_control *) place;
1305 : 2054 : memset(place, 0, sizeof(*control));
1306 : 2054 : control->segment_header.magic =
1307 : 2054 : DSA_SEGMENT_HEADER_MAGIC ^ control_handle ^ 0;
1308 : 2054 : control->segment_header.next = DSA_SEGMENT_INDEX_NONE;
1309 : 2054 : control->segment_header.prev = DSA_SEGMENT_INDEX_NONE;
1310 : 2054 : control->segment_header.usable_pages = usable_pages;
1311 : 2054 : control->segment_header.freed = false;
1312 : 2054 : control->segment_header.size = size;
1313 : 2054 : control->handle = control_handle;
1314 : 2054 : control->init_segment_size = init_segment_size;
1315 : 2054 : control->max_segment_size = max_segment_size;
1316 : 2054 : control->max_total_segment_size = (size_t) -1;
1317 : 2054 : control->total_segment_size = size;
1318 : 2054 : control->segment_handles[0] = control_handle;
1319 [ + + ]: 34918 : for (int i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
1320 : 32864 : control->segment_bins[i] = DSA_SEGMENT_INDEX_NONE;
1321 : 2054 : control->refcnt = 1;
1322 : 2054 : control->lwlock_tranche_id = tranche_id;
1323 : :
1324 : : /*
1325 : : * Create the dsa_area object that this backend will use to access the
1326 : : * area. Other backends will need to obtain their own dsa_area object by
1327 : : * attaching.
1328 : : */
1329 : 2054 : area = palloc_object(dsa_area);
1330 : 2054 : area->control = control;
1331 : 2054 : area->resowner = CurrentResourceOwner;
1332 : 2054 : memset(area->segment_maps, 0, sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
1333 : 2054 : area->high_segment_index = 0;
1334 : 2054 : area->freed_segment_counter = 0;
1335 : 2054 : LWLockInitialize(&control->lock, control->lwlock_tranche_id);
1336 [ + + ]: 80106 : for (size_t i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
1337 : 78052 : LWLockInitialize(DSA_SCLASS_LOCK(area, i),
1338 : : control->lwlock_tranche_id);
1339 : :
1340 : : /* Set up the segment map for this process's mapping. */
1341 : 2054 : segment_map = &area->segment_maps[0];
1342 : 2054 : segment_map->segment = control_segment;
1343 : 2054 : segment_map->mapped_address = place;
1344 : 2054 : segment_map->header = (dsa_segment_header *) place;
1345 : 2054 : segment_map->fpm = (FreePageManager *)
1346 : 2054 : (segment_map->mapped_address +
1347 : : MAXALIGN(sizeof(dsa_area_control)));
1348 : 2054 : segment_map->pagemap = (dsa_pointer *)
1349 : 2054 : (segment_map->mapped_address +
1350 : 2054 : MAXALIGN(sizeof(dsa_area_control)) +
1351 : : MAXALIGN(sizeof(FreePageManager)));
1352 : :
1353 : : /* Set up the free page map. */
1354 : 2054 : FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
1355 : : /* There can be 0 usable pages if size is dsa_minimum_size(). */
1356 : :
1357 [ + + ]: 2054 : if (usable_pages > 0)
1358 : 1531 : FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
1359 : : usable_pages);
1360 : :
1361 : : /* Put this segment into the appropriate bin. */
1362 : 2054 : control->segment_bins[contiguous_pages_to_segment_bin(usable_pages)] = 0;
1363 : 2054 : segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
1364 : :
1365 : 2054 : return area;
1366 : : }
1367 : :
1368 : : /*
1369 : : * Workhorse function for dsa_attach and dsa_attach_in_place.
1370 : : */
1371 : : static dsa_area *
1372 : 29177 : attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
1373 : : {
1374 : : dsa_area_control *control;
1375 : : dsa_area *area;
1376 : : dsa_segment_map *segment_map;
1377 : :
1378 : 29177 : control = (dsa_area_control *) place;
1379 : : Assert(control->handle == handle);
1380 : : Assert(control->segment_handles[0] == handle);
1381 : : Assert(control->segment_header.magic ==
1382 : : (DSA_SEGMENT_HEADER_MAGIC ^ handle ^ 0));
1383 : :
1384 : : /* Build the backend-local area object. */
1385 : 29177 : area = palloc_object(dsa_area);
1386 : 29177 : area->control = control;
1387 : 29177 : area->resowner = CurrentResourceOwner;
1388 : 29177 : memset(&area->segment_maps[0], 0,
1389 : : sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
1390 : 29177 : area->high_segment_index = 0;
1391 : :
1392 : : /* Set up the segment map for this process's mapping. */
1393 : 29177 : segment_map = &area->segment_maps[0];
1394 : 29177 : segment_map->segment = segment; /* NULL for in-place */
1395 : 29177 : segment_map->mapped_address = place;
1396 : 29177 : segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
1397 : 29177 : segment_map->fpm = (FreePageManager *)
1398 : 29177 : (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)));
1399 : 29177 : segment_map->pagemap = (dsa_pointer *)
1400 : 29177 : (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)) +
1401 : : MAXALIGN(sizeof(FreePageManager)));
1402 : :
1403 : : /* Bump the reference count. */
1404 : 29177 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
1405 [ - + ]: 29177 : if (control->refcnt == 0)
1406 : : {
1407 : : /* We can't attach to a DSA area that has already been destroyed. */
1408 [ # # ]: 0 : ereport(ERROR,
1409 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1410 : : errmsg("could not attach to dynamic shared area")));
1411 : : }
1412 : 29177 : ++control->refcnt;
1413 : 29177 : area->freed_segment_counter = area->control->freed_segment_counter;
1414 : 29177 : LWLockRelease(DSA_AREA_LOCK(area));
1415 : :
1416 : 29177 : return area;
1417 : : }
1418 : :
1419 : : /*
1420 : : * Add a new span to fullness class 1 of the indicated pool.
1421 : : */
1422 : : static void
1423 : 17074 : init_span(dsa_area *area,
1424 : : dsa_pointer span_pointer,
1425 : : dsa_area_pool *pool, dsa_pointer start, size_t npages,
1426 : : uint16 size_class)
1427 : : {
1428 : 17074 : dsa_area_span *span = dsa_get_address(area, span_pointer);
1429 : 17074 : size_t obsize = dsa_size_classes[size_class];
1430 : :
1431 : : /*
1432 : : * The per-pool lock must be held because we manipulate the span list for
1433 : : * this pool.
1434 : : */
1435 : : Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1436 : :
1437 : : /* Push this span onto the front of the span list for fullness class 1. */
1438 [ + + ]: 17074 : if (DsaPointerIsValid(pool->spans[1]))
1439 : : {
1440 : : dsa_area_span *head = (dsa_area_span *)
1441 : 2717 : dsa_get_address(area, pool->spans[1]);
1442 : :
1443 : 2717 : head->prevspan = span_pointer;
1444 : : }
1445 : 17074 : span->pool = DsaAreaPoolToDsaPointer(area, pool);
1446 : 17074 : span->nextspan = pool->spans[1];
1447 : 17074 : span->prevspan = InvalidDsaPointer;
1448 : 17074 : pool->spans[1] = span_pointer;
1449 : :
1450 : 17074 : span->start = start;
1451 : 17074 : span->npages = npages;
1452 : 17074 : span->size_class = size_class;
1453 : 17074 : span->ninitialized = 0;
1454 [ + + ]: 17074 : if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1455 : : {
1456 : : /*
1457 : : * A block-of-spans contains its own descriptor, so mark one object as
1458 : : * initialized and reduce the count of allocatable objects by one.
1459 : : * Doing this here has the side effect of also reducing nmax by one,
1460 : : * which is important to make sure we free this object at the correct
1461 : : * time.
1462 : : */
1463 : 1645 : span->ninitialized = 1;
1464 : 1645 : span->nallocatable = FPM_PAGE_SIZE / obsize - 1;
1465 : : }
1466 [ + + ]: 15429 : else if (size_class != DSA_SCLASS_SPAN_LARGE)
1467 : 11737 : span->nallocatable = DSA_SUPERBLOCK_SIZE / obsize;
1468 : 17074 : span->firstfree = DSA_SPAN_NOTHING_FREE;
1469 : 17074 : span->nmax = span->nallocatable;
1470 : 17074 : span->fclass = 1;
1471 : 17074 : }
1472 : :
1473 : : /*
1474 : : * Transfer the first span in one fullness class to the head of another
1475 : : * fullness class.
1476 : : */
1477 : : static bool
1478 : 28252 : transfer_first_span(dsa_area *area,
1479 : : dsa_area_pool *pool, int fromclass, int toclass)
1480 : : {
1481 : : dsa_pointer span_pointer;
1482 : : dsa_area_span *span;
1483 : : dsa_area_span *nextspan;
1484 : :
1485 : : /* Can't do it if source list is empty. */
1486 : 28252 : span_pointer = pool->spans[fromclass];
1487 [ + + ]: 28252 : if (!DsaPointerIsValid(span_pointer))
1488 : 26771 : return false;
1489 : :
1490 : : /* Remove span from head of source list. */
1491 : 1481 : span = dsa_get_address(area, span_pointer);
1492 : 1481 : pool->spans[fromclass] = span->nextspan;
1493 [ + + ]: 1481 : if (DsaPointerIsValid(span->nextspan))
1494 : : {
1495 : : nextspan = (dsa_area_span *)
1496 : 91 : dsa_get_address(area, span->nextspan);
1497 : 91 : nextspan->prevspan = InvalidDsaPointer;
1498 : : }
1499 : :
1500 : : /* Add span to head of target list. */
1501 : 1481 : span->nextspan = pool->spans[toclass];
1502 : 1481 : pool->spans[toclass] = span_pointer;
1503 [ + + ]: 1481 : if (DsaPointerIsValid(span->nextspan))
1504 : : {
1505 : : nextspan = (dsa_area_span *)
1506 : 479 : dsa_get_address(area, span->nextspan);
1507 : 479 : nextspan->prevspan = span_pointer;
1508 : : }
1509 : 1481 : span->fclass = toclass;
1510 : :
1511 : 1481 : return true;
1512 : : }
1513 : :
1514 : : /*
1515 : : * Allocate one object of the requested size class from the given area.
1516 : : */
1517 : : static inline dsa_pointer
1518 : 830384 : alloc_object(dsa_area *area, int size_class)
1519 : : {
1520 : 830384 : dsa_area_pool *pool = &area->control->pools[size_class];
1521 : : dsa_area_span *span;
1522 : : dsa_pointer block;
1523 : : dsa_pointer result;
1524 : : char *object;
1525 : : size_t size;
1526 : :
1527 : : /*
1528 : : * Even though ensure_active_superblock can in turn call alloc_object if
1529 : : * it needs to allocate a new span, that's always from a different pool,
1530 : : * and the order of lock acquisition is always the same, so it's OK that
1531 : : * we hold this lock for the duration of this function.
1532 : : */
1533 : : Assert(!LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1534 : 830384 : LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
1535 : :
1536 : : /*
1537 : : * If there's no active superblock, we must successfully obtain one or
1538 : : * fail the request.
1539 : : */
1540 [ + + ]: 830384 : if (!DsaPointerIsValid(pool->spans[1]) &&
1541 [ - + ]: 13569 : !ensure_active_superblock(area, pool, size_class))
1542 : : {
1543 : 0 : result = InvalidDsaPointer;
1544 : : }
1545 : : else
1546 : : {
1547 : : /*
1548 : : * There should be a block in fullness class 1 at this point, and it
1549 : : * should never be completely full. Thus we can either pop an object
1550 : : * from the free list or, failing that, initialize a new object.
1551 : : */
1552 : : Assert(DsaPointerIsValid(pool->spans[1]));
1553 : : span = (dsa_area_span *)
1554 : 830384 : dsa_get_address(area, pool->spans[1]);
1555 : : Assert(span->nallocatable > 0);
1556 : 830384 : block = span->start;
1557 : : Assert(size_class < DSA_NUM_SIZE_CLASSES);
1558 : 830384 : size = dsa_size_classes[size_class];
1559 [ + + ]: 830384 : if (span->firstfree != DSA_SPAN_NOTHING_FREE)
1560 : : {
1561 : 132263 : result = block + span->firstfree * size;
1562 : 132263 : object = dsa_get_address(area, result);
1563 : 132263 : span->firstfree = NextFreeObjectIndex(object);
1564 : : }
1565 : : else
1566 : : {
1567 : 698121 : result = block + span->ninitialized * size;
1568 : 698121 : ++span->ninitialized;
1569 : : }
1570 : 830384 : --span->nallocatable;
1571 : :
1572 : : /* If it's now full, move it to the highest-numbered fullness class. */
1573 [ + + ]: 830384 : if (span->nallocatable == 0)
1574 : 1299 : transfer_first_span(area, pool, 1, DSA_FULLNESS_CLASSES - 1);
1575 : : }
1576 : :
1577 : : Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1578 : 830384 : LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
1579 : :
1580 : 830384 : return result;
1581 : : }
1582 : :
1583 : : /*
1584 : : * Ensure an active (i.e. fullness class 1) superblock, unless all existing
1585 : : * superblocks are completely full and no more can be allocated.
1586 : : *
1587 : : * Fullness classes K of 0..N are loosely intended to represent blocks whose
1588 : : * utilization percentage is at least K/N, but we only enforce this rigorously
1589 : : * for the highest-numbered fullness class, which always contains exactly
1590 : : * those blocks that are completely full. It's otherwise acceptable for a
1591 : : * block to be in a higher-numbered fullness class than the one to which it
1592 : : * logically belongs. In addition, the active block, which is always the
1593 : : * first block in fullness class 1, is permitted to have a higher allocation
1594 : : * percentage than would normally be allowable for that fullness class; we
1595 : : * don't move it until it's completely full, and then it goes to the
1596 : : * highest-numbered fullness class.
1597 : : *
1598 : : * It might seem odd that the active block is the head of fullness class 1
1599 : : * rather than fullness class 0, but experience with other allocators has
1600 : : * shown that it's usually better to allocate from a block that's moderately
1601 : : * full rather than one that's nearly empty. Insofar as is reasonably
1602 : : * possible, we want to avoid performing new allocations in a block that would
1603 : : * otherwise become empty soon.
1604 : : */
1605 : : static bool
1606 : 13569 : ensure_active_superblock(dsa_area *area, dsa_area_pool *pool,
1607 : : int size_class)
1608 : : {
1609 : : dsa_pointer span_pointer;
1610 : : dsa_pointer start_pointer;
1611 : 13569 : size_t obsize = dsa_size_classes[size_class];
1612 : : size_t nmax;
1613 : : int fclass;
1614 : 13569 : size_t npages = 1;
1615 : : size_t first_page;
1616 : : size_t i;
1617 : : dsa_segment_map *segment_map;
1618 : :
1619 : : Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
1620 : :
1621 : : /*
1622 : : * Compute the number of objects that will fit in a block of this size
1623 : : * class. Span-of-spans blocks are just a single page, and the first
1624 : : * object isn't available for use because it describes the block-of-spans
1625 : : * itself.
1626 : : */
1627 [ + + ]: 13569 : if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1628 : 1645 : nmax = FPM_PAGE_SIZE / obsize - 1;
1629 : : else
1630 : 11924 : nmax = DSA_SUPERBLOCK_SIZE / obsize;
1631 : :
1632 : : /*
1633 : : * If fullness class 1 is empty, try to find a span to put in it by
1634 : : * scanning higher-numbered fullness classes (excluding the last one,
1635 : : * whose blocks are certain to all be completely full).
1636 : : */
1637 [ + + ]: 27133 : for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
1638 : : {
1639 : 13569 : span_pointer = pool->spans[fclass];
1640 : :
1641 [ + + ]: 13893 : while (DsaPointerIsValid(span_pointer))
1642 : : {
1643 : : int tfclass;
1644 : : dsa_area_span *span;
1645 : : dsa_area_span *nextspan;
1646 : : dsa_area_span *prevspan;
1647 : : dsa_pointer next_span_pointer;
1648 : :
1649 : : span = (dsa_area_span *)
1650 : 324 : dsa_get_address(area, span_pointer);
1651 : 324 : next_span_pointer = span->nextspan;
1652 : :
1653 : : /* Figure out what fullness class should contain this span. */
1654 : 324 : tfclass = (nmax - span->nallocatable)
1655 : 324 : * (DSA_FULLNESS_CLASSES - 1) / nmax;
1656 : :
1657 : : /* Look up next span. */
1658 [ + + ]: 324 : if (DsaPointerIsValid(span->nextspan))
1659 : : nextspan = (dsa_area_span *)
1660 : 141 : dsa_get_address(area, span->nextspan);
1661 : : else
1662 : 183 : nextspan = NULL;
1663 : :
1664 : : /*
1665 : : * If utilization has dropped enough that this now belongs in some
1666 : : * other fullness class, move it there.
1667 : : */
1668 [ + + ]: 324 : if (tfclass < fclass)
1669 : : {
1670 : : /* Remove from the current fullness class list. */
1671 [ + + ]: 14 : if (pool->spans[fclass] == span_pointer)
1672 : : {
1673 : : /* It was the head; remove it. */
1674 : : Assert(!DsaPointerIsValid(span->prevspan));
1675 : 7 : pool->spans[fclass] = span->nextspan;
1676 [ + + ]: 7 : if (nextspan != NULL)
1677 : 2 : nextspan->prevspan = InvalidDsaPointer;
1678 : : }
1679 : : else
1680 : : {
1681 : : /* It was not the head. */
1682 : : Assert(DsaPointerIsValid(span->prevspan));
1683 : : prevspan = (dsa_area_span *)
1684 : 7 : dsa_get_address(area, span->prevspan);
1685 : 7 : prevspan->nextspan = span->nextspan;
1686 : : }
1687 [ + + ]: 14 : if (nextspan != NULL)
1688 : 2 : nextspan->prevspan = span->prevspan;
1689 : :
1690 : : /* Push onto the head of the new fullness class list. */
1691 : 14 : span->nextspan = pool->spans[tfclass];
1692 : 14 : pool->spans[tfclass] = span_pointer;
1693 : 14 : span->prevspan = InvalidDsaPointer;
1694 [ + + ]: 14 : if (DsaPointerIsValid(span->nextspan))
1695 : : {
1696 : : nextspan = (dsa_area_span *)
1697 : 1 : dsa_get_address(area, span->nextspan);
1698 : 1 : nextspan->prevspan = span_pointer;
1699 : : }
1700 : 14 : span->fclass = tfclass;
1701 : : }
1702 : :
1703 : : /* Advance to next span on list. */
1704 : 324 : span_pointer = next_span_pointer;
1705 : : }
1706 : :
1707 : : /* Stop now if we found a suitable block. */
1708 [ + + ]: 13569 : if (DsaPointerIsValid(pool->spans[1]))
1709 : 5 : return true;
1710 : : }
1711 : :
1712 : : /*
1713 : : * If there are no blocks that properly belong in fullness class 1, pick
1714 : : * one from some other fullness class and move it there anyway, so that we
1715 : : * have an allocation target. Our last choice is to transfer a block
1716 : : * that's almost empty (and might become completely empty soon if left
1717 : : * alone), but even that is better than failing, which is what we must do
1718 : : * if there are no blocks at all with freespace.
1719 : : */
1720 : : Assert(!DsaPointerIsValid(pool->spans[1]));
1721 [ + + ]: 26953 : for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
1722 [ + + ]: 13564 : if (transfer_first_span(area, pool, fclass, 1))
1723 : 175 : return true;
1724 [ + - + + ]: 26778 : if (!DsaPointerIsValid(pool->spans[1]) &&
1725 : 13389 : transfer_first_span(area, pool, 0, 1))
1726 : 7 : return true;
1727 : :
1728 : : /*
1729 : : * We failed to find an existing span with free objects, so we need to
1730 : : * allocate a new superblock and construct a new span to manage it.
1731 : : *
1732 : : * First, get a dsa_area_span object to describe the new superblock block
1733 : : * ... unless this allocation is for a dsa_area_span object, in which case
1734 : : * that's surely not going to work. We handle that case by storing the
1735 : : * span describing a block-of-spans inline.
1736 : : */
1737 [ + + ]: 13382 : if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
1738 : : {
1739 : 11737 : span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
1740 [ - + ]: 11737 : if (!DsaPointerIsValid(span_pointer))
1741 : 0 : return false;
1742 : 11737 : npages = DSA_PAGES_PER_SUPERBLOCK;
1743 : : }
1744 : :
1745 : : /* Find or create a segment and allocate the superblock. */
1746 : 13382 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
1747 : 13382 : segment_map = get_best_segment(area, npages);
1748 [ + + ]: 13382 : if (segment_map == NULL)
1749 : : {
1750 : 1244 : segment_map = make_new_segment(area, npages);
1751 [ - + ]: 1244 : if (segment_map == NULL)
1752 : : {
1753 : 0 : LWLockRelease(DSA_AREA_LOCK(area));
1754 : 0 : return false;
1755 : : }
1756 : : }
1757 : :
1758 : : /*
1759 : : * This shouldn't happen: get_best_segment() or make_new_segment()
1760 : : * promised that we can successfully allocate npages.
1761 : : */
1762 [ - + ]: 13382 : if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
1763 [ # # ]: 0 : elog(FATAL,
1764 : : "dsa_allocate could not find %zu free pages for superblock",
1765 : : npages);
1766 : 13382 : LWLockRelease(DSA_AREA_LOCK(area));
1767 : :
1768 : : /* Compute the start of the superblock. */
1769 : 13382 : start_pointer =
1770 : 13382 : DSA_MAKE_POINTER(get_segment_index(area, segment_map),
1771 : : first_page * FPM_PAGE_SIZE);
1772 : :
1773 : : /*
1774 : : * If this is a block-of-spans, carve the descriptor right out of the
1775 : : * allocated space.
1776 : : */
1777 [ + + ]: 13382 : if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
1778 : : {
1779 : : /*
1780 : : * We have a pointer into the segment. We need to build a dsa_pointer
1781 : : * from the segment index and offset into the segment.
1782 : : */
1783 : 1645 : span_pointer = start_pointer;
1784 : : }
1785 : :
1786 : : /* Initialize span and pagemap. */
1787 : 13382 : init_span(area, span_pointer, pool, start_pointer, npages, size_class);
1788 [ + + ]: 202819 : for (i = 0; i < npages; ++i)
1789 : 189437 : segment_map->pagemap[first_page + i] = span_pointer;
1790 : :
1791 : 13382 : return true;
1792 : : }
1793 : :
1794 : : /*
1795 : : * Return the segment map corresponding to a given segment index, mapping the
1796 : : * segment in if necessary. For internal segment book-keeping, this is called
1797 : : * with the area lock held. It is also called by dsa_free and dsa_get_address
1798 : : * without any locking, relying on the fact they have a known live segment
1799 : : * index and they always call check_for_freed_segments to ensures that any
1800 : : * freed segment occupying the same slot is detached first.
1801 : : */
1802 : : static dsa_segment_map *
1803 : 198088 : get_segment_by_index(dsa_area *area, dsa_segment_index index)
1804 : : {
1805 [ + + ]: 198088 : if (unlikely(area->segment_maps[index].mapped_address == NULL))
1806 : : {
1807 : : dsm_handle handle;
1808 : : dsm_segment *segment;
1809 : : dsa_segment_map *segment_map;
1810 : : ResourceOwner oldowner;
1811 : :
1812 : : /*
1813 : : * If we are reached by dsa_free or dsa_get_address, there must be at
1814 : : * least one object allocated in the referenced segment. Otherwise,
1815 : : * their caller has a double-free or access-after-free bug, which we
1816 : : * have no hope of detecting. So we know it's safe to access this
1817 : : * array slot without holding a lock; it won't change underneath us.
1818 : : * Furthermore, we know that we can see the latest contents of the
1819 : : * slot, as explained in check_for_freed_segments, which those
1820 : : * functions call before arriving here.
1821 : : */
1822 : 23053 : handle = area->control->segment_handles[index];
1823 : :
1824 : : /* It's an error to try to access an unused slot. */
1825 [ - + ]: 23053 : if (handle == DSM_HANDLE_INVALID)
1826 [ # # ]: 0 : elog(ERROR,
1827 : : "dsa_area could not attach to a segment that has been freed");
1828 : :
1829 : 23053 : oldowner = CurrentResourceOwner;
1830 : 23053 : CurrentResourceOwner = area->resowner;
1831 : 23053 : segment = dsm_attach(handle);
1832 : 23053 : CurrentResourceOwner = oldowner;
1833 [ - + ]: 23053 : if (segment == NULL)
1834 [ # # ]: 0 : elog(ERROR, "dsa_area could not attach to segment");
1835 : 23053 : segment_map = &area->segment_maps[index];
1836 : 23053 : segment_map->segment = segment;
1837 : 23053 : segment_map->mapped_address = dsm_segment_address(segment);
1838 : 23053 : segment_map->header =
1839 : 23053 : (dsa_segment_header *) segment_map->mapped_address;
1840 : 23053 : segment_map->fpm = (FreePageManager *)
1841 : 23053 : (segment_map->mapped_address +
1842 : : MAXALIGN(sizeof(dsa_segment_header)));
1843 : 23053 : segment_map->pagemap = (dsa_pointer *)
1844 : 23053 : (segment_map->mapped_address +
1845 : 23053 : MAXALIGN(sizeof(dsa_segment_header)) +
1846 : : MAXALIGN(sizeof(FreePageManager)));
1847 : :
1848 : : /* Remember the highest index this backend has ever mapped. */
1849 [ + + ]: 23053 : if (area->high_segment_index < index)
1850 : 22890 : area->high_segment_index = index;
1851 : :
1852 : : Assert(segment_map->header->magic ==
1853 : : (DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ index));
1854 : : }
1855 : :
1856 : : /*
1857 : : * Callers of dsa_get_address() and dsa_free() don't hold the area lock,
1858 : : * but it's a bug in the calling code and undefined behavior if the
1859 : : * address is not live (ie if the segment might possibly have been freed,
1860 : : * they're trying to use a dangling pointer).
1861 : : *
1862 : : * For dsa.c code that holds the area lock to manipulate segment_bins
1863 : : * lists, it would be a bug if we ever reach a freed segment here. After
1864 : : * it's marked as freed, the only thing any backend should do with it is
1865 : : * unmap it, and it should always have done that in
1866 : : * check_for_freed_segments_locked() before arriving here to resolve an
1867 : : * index to a segment_map.
1868 : : *
1869 : : * Either way we can assert that we aren't returning a freed segment.
1870 : : */
1871 : : Assert(!area->segment_maps[index].header->freed);
1872 : :
1873 : 198088 : return &area->segment_maps[index];
1874 : : }
1875 : :
1876 : : /*
1877 : : * Return a superblock to the free page manager. If the underlying segment
1878 : : * has become entirely free, then return it to the operating system.
1879 : : *
1880 : : * The appropriate pool lock must be held.
1881 : : */
1882 : : static void
1883 : 16 : destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
1884 : : {
1885 : 16 : dsa_area_span *span = dsa_get_address(area, span_pointer);
1886 : 16 : int size_class = span->size_class;
1887 : : dsa_segment_map *segment_map;
1888 : :
1889 : :
1890 : : /* Remove it from its fullness class list. */
1891 : 16 : unlink_span(area, span);
1892 : :
1893 : : /*
1894 : : * Note: Here we acquire the area lock while we already hold a per-pool
1895 : : * lock. We never hold the area lock and then take a pool lock, or we
1896 : : * could deadlock.
1897 : : */
1898 : 16 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
1899 : 16 : check_for_freed_segments_locked(area);
1900 : : segment_map =
1901 : 16 : get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(span->start));
1902 : 16 : FreePageManagerPut(segment_map->fpm,
1903 : 16 : DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
1904 : : span->npages);
1905 : : /* Check if the segment is now entirely free. */
1906 [ - + ]: 16 : if (fpm_largest(segment_map->fpm) == segment_map->header->usable_pages)
1907 : : {
1908 : 0 : dsa_segment_index index = get_segment_index(area, segment_map);
1909 : :
1910 : : /* If it's not the segment with extra control data, free it. */
1911 [ # # ]: 0 : if (index != 0)
1912 : : {
1913 : : /*
1914 : : * Give it back to the OS, and allow other backends to detect that
1915 : : * they need to detach.
1916 : : */
1917 : 0 : unlink_segment(area, segment_map);
1918 : 0 : segment_map->header->freed = true;
1919 : : Assert(area->control->total_segment_size >=
1920 : : segment_map->header->size);
1921 : 0 : area->control->total_segment_size -=
1922 : 0 : segment_map->header->size;
1923 : 0 : dsm_unpin_segment(dsm_segment_handle(segment_map->segment));
1924 : 0 : dsm_detach(segment_map->segment);
1925 : 0 : area->control->segment_handles[index] = DSM_HANDLE_INVALID;
1926 : 0 : ++area->control->freed_segment_counter;
1927 : 0 : segment_map->segment = NULL;
1928 : 0 : segment_map->header = NULL;
1929 : 0 : segment_map->mapped_address = NULL;
1930 : : }
1931 : : }
1932 : :
1933 : : /* Move segment to appropriate bin if necessary. */
1934 [ + - ]: 16 : if (segment_map->header != NULL)
1935 : 16 : rebin_segment(area, segment_map);
1936 : :
1937 : 16 : LWLockRelease(DSA_AREA_LOCK(area));
1938 : :
1939 : : /*
1940 : : * Span-of-spans blocks store the span which describes them within the
1941 : : * block itself, so freeing the storage implicitly frees the descriptor
1942 : : * also. If this is a block of any other type, we need to separately free
1943 : : * the span object also. This recursive call to dsa_free will acquire the
1944 : : * span pool's lock. We can't deadlock because the acquisition order is
1945 : : * always some other pool and then the span pool.
1946 : : */
1947 [ + - ]: 16 : if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
1948 : 16 : dsa_free(area, span_pointer);
1949 : 16 : }
1950 : :
1951 : : static void
1952 : 3188 : unlink_span(dsa_area *area, dsa_area_span *span)
1953 : : {
1954 [ + + ]: 3188 : if (DsaPointerIsValid(span->nextspan))
1955 : : {
1956 : 2612 : dsa_area_span *next = dsa_get_address(area, span->nextspan);
1957 : :
1958 : 2612 : next->prevspan = span->prevspan;
1959 : : }
1960 [ + + ]: 3188 : if (DsaPointerIsValid(span->prevspan))
1961 : : {
1962 : 1250 : dsa_area_span *prev = dsa_get_address(area, span->prevspan);
1963 : :
1964 : 1250 : prev->nextspan = span->nextspan;
1965 : : }
1966 : : else
1967 : : {
1968 : 1938 : dsa_area_pool *pool = dsa_get_address(area, span->pool);
1969 : :
1970 : 1938 : pool->spans[span->fclass] = span->nextspan;
1971 : : }
1972 : 3188 : }
1973 : :
1974 : : static void
1975 : 258 : add_span_to_fullness_class(dsa_area *area, dsa_area_span *span,
1976 : : dsa_pointer span_pointer,
1977 : : int fclass)
1978 : : {
1979 : 258 : dsa_area_pool *pool = dsa_get_address(area, span->pool);
1980 : :
1981 [ + + ]: 258 : if (DsaPointerIsValid(pool->spans[fclass]))
1982 : : {
1983 : 113 : dsa_area_span *head = dsa_get_address(area,
1984 : : pool->spans[fclass]);
1985 : :
1986 : 113 : head->prevspan = span_pointer;
1987 : : }
1988 : 258 : span->prevspan = InvalidDsaPointer;
1989 : 258 : span->nextspan = pool->spans[fclass];
1990 : 258 : pool->spans[fclass] = span_pointer;
1991 : 258 : span->fclass = fclass;
1992 : 258 : }
1993 : :
1994 : : /*
1995 : : * Detach from an area that was either created or attached to by this process.
1996 : : */
1997 : : void
1998 : 30591 : dsa_detach(dsa_area *area)
1999 : : {
2000 : : /* Detach from all segments. */
2001 [ + + ]: 85481 : for (dsa_segment_index i = 0; i <= area->high_segment_index; ++i)
2002 [ + + ]: 54890 : if (area->segment_maps[i].segment != NULL)
2003 : 24402 : dsm_detach(area->segment_maps[i].segment);
2004 : :
2005 : : /*
2006 : : * Note that 'detaching' (= detaching from DSM segments) doesn't include
2007 : : * 'releasing' (= adjusting the reference count). It would be nice to
2008 : : * combine these operations, but client code might never get around to
2009 : : * calling dsa_detach because of an error path, and a detach hook on any
2010 : : * particular segment is too late to detach other segments in the area
2011 : : * without risking a 'leak' warning in the non-error path.
2012 : : */
2013 : :
2014 : : /* Free the backend-local area object. */
2015 : 30591 : pfree(area);
2016 : 30591 : }
2017 : :
2018 : : /*
2019 : : * Unlink a segment from the bin that contains it.
2020 : : */
2021 : : static void
2022 : 2802 : unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
2023 : : {
2024 [ + + ]: 2802 : if (segment_map->header->prev != DSA_SEGMENT_INDEX_NONE)
2025 : : {
2026 : : dsa_segment_map *prev;
2027 : :
2028 : 4 : prev = get_segment_by_index(area, segment_map->header->prev);
2029 : 4 : prev->header->next = segment_map->header->next;
2030 : : }
2031 : : else
2032 : : {
2033 : : Assert(area->control->segment_bins[segment_map->header->bin] ==
2034 : : get_segment_index(area, segment_map));
2035 : 2798 : area->control->segment_bins[segment_map->header->bin] =
2036 : 2798 : segment_map->header->next;
2037 : : }
2038 [ + + ]: 2802 : if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
2039 : : {
2040 : : dsa_segment_map *next;
2041 : :
2042 : 1 : next = get_segment_by_index(area, segment_map->header->next);
2043 : 1 : next->header->prev = segment_map->header->prev;
2044 : : }
2045 : 2802 : }
2046 : :
2047 : : /*
2048 : : * Find a segment that could satisfy a request for 'npages' of contiguous
2049 : : * memory, or return NULL if none can be found. This may involve attaching to
2050 : : * segments that weren't previously attached so that we can query their free
2051 : : * pages map.
2052 : : */
2053 : : static dsa_segment_map *
2054 : 17074 : get_best_segment(dsa_area *area, size_t npages)
2055 : : {
2056 : : size_t bin;
2057 : :
2058 : : Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
2059 : 17074 : check_for_freed_segments_locked(area);
2060 : :
2061 : : /*
2062 : : * Start searching from the first bin that *might* have enough contiguous
2063 : : * pages.
2064 : : */
2065 : 17074 : for (bin = contiguous_pages_to_segment_bin(npages);
2066 [ + + ]: 74650 : bin < DSA_NUM_SEGMENT_BINS;
2067 : 57576 : ++bin)
2068 : : {
2069 : : /*
2070 : : * The minimum contiguous size that any segment in this bin should
2071 : : * have. We'll re-bin if we see segments with fewer.
2072 : : */
2073 : 73366 : size_t threshold = (size_t) 1 << (bin - 1);
2074 : : dsa_segment_index segment_index;
2075 : :
2076 : : /* Search this bin for a segment with enough contiguous space. */
2077 : 73366 : segment_index = area->control->segment_bins[bin];
2078 [ + + ]: 74527 : while (segment_index != DSA_SEGMENT_INDEX_NONE)
2079 : : {
2080 : : dsa_segment_map *segment_map;
2081 : : dsa_segment_index next_segment_index;
2082 : : size_t contiguous_pages;
2083 : :
2084 : 16951 : segment_map = get_segment_by_index(area, segment_index);
2085 : 16951 : next_segment_index = segment_map->header->next;
2086 : 16951 : contiguous_pages = fpm_largest(segment_map->fpm);
2087 : :
2088 : : /* Not enough for the request, still enough for this bin. */
2089 [ + + - + ]: 16951 : if (contiguous_pages >= threshold && contiguous_pages < npages)
2090 : : {
2091 : 0 : segment_index = next_segment_index;
2092 : 0 : continue;
2093 : : }
2094 : :
2095 : : /* Re-bin it if it's no longer in the appropriate bin. */
2096 [ + + ]: 16951 : if (contiguous_pages < threshold)
2097 : : {
2098 : 2578 : rebin_segment(area, segment_map);
2099 : :
2100 : : /*
2101 : : * But fall through to see if it's enough to satisfy this
2102 : : * request anyway....
2103 : : */
2104 : : }
2105 : :
2106 : : /* Check if we are done. */
2107 [ + + ]: 16951 : if (contiguous_pages >= npages)
2108 : 15790 : return segment_map;
2109 : :
2110 : : /* Continue searching the same bin. */
2111 : 1161 : segment_index = next_segment_index;
2112 : : }
2113 : : }
2114 : :
2115 : : /* Not found. */
2116 : 1284 : return NULL;
2117 : : }
2118 : :
2119 : : /*
2120 : : * Create a new segment that can handle at least requested_pages. Returns
2121 : : * NULL if the requested total size limit or maximum allowed number of
2122 : : * segments would be exceeded.
2123 : : */
2124 : : static dsa_segment_map *
2125 : 1284 : make_new_segment(dsa_area *area, size_t requested_pages)
2126 : : {
2127 : : dsa_segment_index new_index;
2128 : : size_t metadata_bytes;
2129 : : size_t total_size;
2130 : : size_t total_pages;
2131 : : size_t usable_pages;
2132 : : dsa_segment_map *segment_map;
2133 : : dsm_segment *segment;
2134 : : ResourceOwner oldowner;
2135 : :
2136 : : Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
2137 : :
2138 : : /* Find a segment slot that is not in use (linearly for now). */
2139 [ + - ]: 1334 : for (new_index = 1; new_index < DSA_MAX_SEGMENTS; ++new_index)
2140 : : {
2141 [ + + ]: 1334 : if (area->control->segment_handles[new_index] == DSM_HANDLE_INVALID)
2142 : 1284 : break;
2143 : : }
2144 [ - + ]: 1284 : if (new_index == DSA_MAX_SEGMENTS)
2145 : 0 : return NULL;
2146 : :
2147 : : /*
2148 : : * If the total size limit is already exceeded, then we exit early and
2149 : : * avoid arithmetic wraparound in the unsigned expressions below.
2150 : : */
2151 : 1284 : if (area->control->total_segment_size >=
2152 [ - + ]: 1284 : area->control->max_total_segment_size)
2153 : 0 : return NULL;
2154 : :
2155 : : /*
2156 : : * The size should be at least as big as requested, and at least big
2157 : : * enough to follow a geometric series that approximately doubles the
2158 : : * total storage each time we create a new segment. We use geometric
2159 : : * growth because the underlying DSM system isn't designed for large
2160 : : * numbers of segments (otherwise we might even consider just using one
2161 : : * DSM segment for each large allocation and for each superblock, and then
2162 : : * we wouldn't need to use FreePageManager).
2163 : : *
2164 : : * We decide on a total segment size first, so that we produce tidy
2165 : : * power-of-two sized segments. This is a good property to have if we
2166 : : * move to huge pages in the future. Then we work back to the number of
2167 : : * pages we can fit.
2168 : : */
2169 : 1284 : total_size = area->control->init_segment_size *
2170 : 1284 : ((size_t) 1 << (new_index / DSA_NUM_SEGMENTS_AT_EACH_SIZE));
2171 : 1284 : total_size = Min(total_size, area->control->max_segment_size);
2172 : 1284 : total_size = Min(total_size,
2173 : : area->control->max_total_segment_size -
2174 : : area->control->total_segment_size);
2175 : :
2176 : 1284 : total_pages = total_size / FPM_PAGE_SIZE;
2177 : 1284 : metadata_bytes =
2178 : : MAXALIGN(sizeof(dsa_segment_header)) +
2179 : 1284 : MAXALIGN(sizeof(FreePageManager)) +
2180 : : sizeof(dsa_pointer) * total_pages;
2181 : :
2182 : : /* Add padding up to next page boundary. */
2183 [ + - ]: 1284 : if (metadata_bytes % FPM_PAGE_SIZE != 0)
2184 : 1284 : metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
2185 [ - + ]: 1284 : if (total_size <= metadata_bytes)
2186 : 0 : return NULL;
2187 : 1284 : usable_pages = (total_size - metadata_bytes) / FPM_PAGE_SIZE;
2188 : : Assert(metadata_bytes + usable_pages * FPM_PAGE_SIZE <= total_size);
2189 : :
2190 : : /* See if that is enough... */
2191 [ + + ]: 1284 : if (requested_pages > usable_pages)
2192 : : {
2193 : : size_t total_requested_pages PG_USED_FOR_ASSERTS_ONLY;
2194 : :
2195 : : /*
2196 : : * We'll make an odd-sized segment, working forward from the requested
2197 : : * number of pages.
2198 : : */
2199 : 11 : usable_pages = requested_pages;
2200 : 11 : metadata_bytes =
2201 : : MAXALIGN(sizeof(dsa_segment_header)) +
2202 : 11 : MAXALIGN(sizeof(FreePageManager)) +
2203 : : usable_pages * sizeof(dsa_pointer);
2204 : :
2205 : : /*
2206 : : * We must also account for pagemap entries needed to cover the
2207 : : * metadata pages themselves. The pagemap must track all pages in the
2208 : : * segment, including the pages occupied by metadata.
2209 : : *
2210 : : * This formula uses integer ceiling division to compute the exact
2211 : : * number of additional entries needed. The divisor (FPM_PAGE_SIZE -
2212 : : * sizeof(dsa_pointer)) accounts for the fact that each metadata page
2213 : : * consumes one pagemap entry of sizeof(dsa_pointer) bytes, leaving
2214 : : * only (FPM_PAGE_SIZE - sizeof(dsa_pointer)) net bytes per metadata
2215 : : * page.
2216 : : */
2217 : 11 : metadata_bytes +=
2218 : 11 : ((metadata_bytes + (FPM_PAGE_SIZE - sizeof(dsa_pointer)) - 1) /
2219 : 11 : (FPM_PAGE_SIZE - sizeof(dsa_pointer))) *
2220 : : sizeof(dsa_pointer);
2221 : :
2222 : : /* Add padding up to next page boundary. */
2223 [ + - ]: 11 : if (metadata_bytes % FPM_PAGE_SIZE != 0)
2224 : 11 : metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
2225 : 11 : total_size = metadata_bytes + usable_pages * FPM_PAGE_SIZE;
2226 : 11 : total_requested_pages = total_size / FPM_PAGE_SIZE;
2227 : :
2228 : : /*
2229 : : * Verify that we allocated enough pagemap entries for metadata and
2230 : : * usable pages. This reverse-engineers the new calculation of
2231 : : * "metadata_bytes" done based on the new "requested_pages" for an
2232 : : * odd-sized segment.
2233 : : */
2234 : : Assert((metadata_bytes - MAXALIGN(sizeof(dsa_segment_header)) -
2235 : : MAXALIGN(sizeof(FreePageManager))) / sizeof(dsa_pointer) >= total_requested_pages);
2236 : :
2237 : : /* Is that too large for dsa_pointer's addressing scheme? */
2238 [ - + ]: 11 : if (total_size > DSA_MAX_SEGMENT_SIZE)
2239 : 0 : return NULL;
2240 : :
2241 : : /* Would that exceed the limit? */
2242 : 11 : if (total_size > area->control->max_total_segment_size -
2243 [ - + ]: 11 : area->control->total_segment_size)
2244 : 0 : return NULL;
2245 : : }
2246 : :
2247 : : /* Create the segment. */
2248 : 1284 : oldowner = CurrentResourceOwner;
2249 : 1284 : CurrentResourceOwner = area->resowner;
2250 : 1284 : segment = dsm_create(total_size, 0);
2251 : 1284 : CurrentResourceOwner = oldowner;
2252 [ - + ]: 1284 : if (segment == NULL)
2253 : 0 : return NULL;
2254 : 1284 : dsm_pin_segment(segment);
2255 : :
2256 : : /* Store the handle in shared memory to be found by index. */
2257 : 2568 : area->control->segment_handles[new_index] =
2258 : 1284 : dsm_segment_handle(segment);
2259 : : /* Track the highest segment index in the history of the area. */
2260 [ + - ]: 1284 : if (area->control->high_segment_index < new_index)
2261 : 1284 : area->control->high_segment_index = new_index;
2262 : : /* Track the highest segment index this backend has ever mapped. */
2263 [ + - ]: 1284 : if (area->high_segment_index < new_index)
2264 : 1284 : area->high_segment_index = new_index;
2265 : : /* Track total size of all segments. */
2266 : 1284 : area->control->total_segment_size += total_size;
2267 : : Assert(area->control->total_segment_size <=
2268 : : area->control->max_total_segment_size);
2269 : :
2270 : : /* Build a segment map for this segment in this backend. */
2271 : 1284 : segment_map = &area->segment_maps[new_index];
2272 : 1284 : segment_map->segment = segment;
2273 : 1284 : segment_map->mapped_address = dsm_segment_address(segment);
2274 : 1284 : segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
2275 : 1284 : segment_map->fpm = (FreePageManager *)
2276 : 1284 : (segment_map->mapped_address +
2277 : : MAXALIGN(sizeof(dsa_segment_header)));
2278 : 1284 : segment_map->pagemap = (dsa_pointer *)
2279 : 1284 : (segment_map->mapped_address +
2280 : 1284 : MAXALIGN(sizeof(dsa_segment_header)) +
2281 : : MAXALIGN(sizeof(FreePageManager)));
2282 : :
2283 : : /* Set up the free page map. */
2284 : 1284 : FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
2285 : 1284 : FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
2286 : : usable_pages);
2287 : :
2288 : : /* Set up the segment header and put it in the appropriate bin. */
2289 : 1284 : segment_map->header->magic =
2290 : 1284 : DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ new_index;
2291 : 1284 : segment_map->header->usable_pages = usable_pages;
2292 : 1284 : segment_map->header->size = total_size;
2293 : 1284 : segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
2294 : 1284 : segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
2295 : 1284 : segment_map->header->next =
2296 : 1284 : area->control->segment_bins[segment_map->header->bin];
2297 : 1284 : segment_map->header->freed = false;
2298 : 1284 : area->control->segment_bins[segment_map->header->bin] = new_index;
2299 [ - + ]: 1284 : if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
2300 : : {
2301 : : dsa_segment_map *next =
2302 : 0 : get_segment_by_index(area, segment_map->header->next);
2303 : :
2304 : : Assert(next->header->bin == segment_map->header->bin);
2305 : 0 : next->header->prev = new_index;
2306 : : }
2307 : :
2308 : 1284 : return segment_map;
2309 : : }
2310 : :
2311 : : /*
2312 : : * Check if any segments have been freed by destroy_superblock, so we can
2313 : : * detach from them in this backend. This function is called by
2314 : : * dsa_get_address and dsa_free to make sure that a dsa_pointer they have
2315 : : * received can be resolved to the correct segment.
2316 : : *
2317 : : * The danger we want to defend against is that there could be an old segment
2318 : : * mapped into a given slot in this backend, and the dsa_pointer they have
2319 : : * might refer to some new segment in the same slot. So those functions must
2320 : : * be sure to process all instructions to detach from a freed segment that had
2321 : : * been generated by the time this process received the dsa_pointer, before
2322 : : * they call get_segment_by_index.
2323 : : */
2324 : : static void
2325 : 9950854 : check_for_freed_segments(dsa_area *area)
2326 : : {
2327 : : size_t freed_segment_counter;
2328 : :
2329 : : /*
2330 : : * Any other process that has freed a segment has incremented
2331 : : * freed_segment_counter while holding an LWLock, and that must precede
2332 : : * any backend creating a new segment in the same slot while holding an
2333 : : * LWLock, and that must precede the creation of any dsa_pointer pointing
2334 : : * into the new segment which might reach us here, and the caller must
2335 : : * have sent the dsa_pointer to this process using appropriate memory
2336 : : * synchronization (some kind of locking or atomic primitive or system
2337 : : * call). So all we need to do on the reading side is ask for the load of
2338 : : * freed_segment_counter to follow the caller's load of the dsa_pointer it
2339 : : * has, and we can be sure to detect any segments that had been freed as
2340 : : * of the time that the dsa_pointer reached this process.
2341 : : */
2342 : 9950854 : pg_read_barrier();
2343 : 9950854 : freed_segment_counter = area->control->freed_segment_counter;
2344 [ - + ]: 9950854 : if (unlikely(area->freed_segment_counter != freed_segment_counter))
2345 : : {
2346 : : /* Check all currently mapped segments to find what's been freed. */
2347 : 0 : LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
2348 : 0 : check_for_freed_segments_locked(area);
2349 : 0 : LWLockRelease(DSA_AREA_LOCK(area));
2350 : : }
2351 : 9950854 : }
2352 : :
2353 : : /*
2354 : : * Workhorse for check_for_freed_segments(), and also used directly in path
2355 : : * where the area lock is already held. This should be called after acquiring
2356 : : * the lock but before looking up any segment by index number, to make sure we
2357 : : * unmap any stale segments that might have previously had the same index as a
2358 : : * current segment.
2359 : : */
2360 : : static void
2361 : 17090 : check_for_freed_segments_locked(dsa_area *area)
2362 : : {
2363 : : size_t freed_segment_counter;
2364 : :
2365 : : Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
2366 : 17090 : freed_segment_counter = area->control->freed_segment_counter;
2367 [ - + ]: 17090 : if (unlikely(area->freed_segment_counter != freed_segment_counter))
2368 : : {
2369 [ # # ]: 0 : for (dsa_segment_index i = 0; i <= area->high_segment_index; ++i)
2370 : : {
2371 [ # # ]: 0 : if (area->segment_maps[i].header != NULL &&
2372 [ # # ]: 0 : area->segment_maps[i].header->freed)
2373 : : {
2374 : 0 : dsm_detach(area->segment_maps[i].segment);
2375 : 0 : area->segment_maps[i].segment = NULL;
2376 : 0 : area->segment_maps[i].header = NULL;
2377 : 0 : area->segment_maps[i].mapped_address = NULL;
2378 : : }
2379 : : }
2380 : 0 : area->freed_segment_counter = freed_segment_counter;
2381 : : }
2382 : 17090 : }
2383 : :
2384 : : /*
2385 : : * Re-bin segment if it's no longer in the appropriate bin.
2386 : : */
2387 : : static void
2388 : 5508 : rebin_segment(dsa_area *area, dsa_segment_map *segment_map)
2389 : : {
2390 : : size_t new_bin;
2391 : : dsa_segment_index segment_index;
2392 : :
2393 : 5508 : new_bin = contiguous_pages_to_segment_bin(fpm_largest(segment_map->fpm));
2394 [ + + ]: 5508 : if (segment_map->header->bin == new_bin)
2395 : 2706 : return;
2396 : :
2397 : : /* Remove it from its current bin. */
2398 : 2802 : unlink_segment(area, segment_map);
2399 : :
2400 : : /* Push it onto the front of its new bin. */
2401 : 2802 : segment_index = get_segment_index(area, segment_map);
2402 : 2802 : segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
2403 : 2802 : segment_map->header->next = area->control->segment_bins[new_bin];
2404 : 2802 : segment_map->header->bin = new_bin;
2405 : 2802 : area->control->segment_bins[new_bin] = segment_index;
2406 [ + + ]: 2802 : if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
2407 : : {
2408 : : dsa_segment_map *next;
2409 : :
2410 : 16 : next = get_segment_by_index(area, segment_map->header->next);
2411 : : Assert(next->header->bin == new_bin);
2412 : 16 : next->header->prev = segment_index;
2413 : : }
2414 : : }
|
