Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * slab.c
4 : : * SLAB allocator definitions.
5 : : *
6 : : * SLAB is a MemoryContext implementation designed for cases where large
7 : : * numbers of equally-sized objects can be allocated and freed efficiently
8 : : * with minimal memory wastage and fragmentation.
9 : : *
10 : : *
11 : : * Portions Copyright (c) 2017-2026, PostgreSQL Global Development Group
12 : : *
13 : : * IDENTIFICATION
14 : : * src/backend/utils/mmgr/slab.c
15 : : *
16 : : *
17 : : * NOTE:
18 : : * The constant allocation size allows significant simplification and various
19 : : * optimizations over more general purpose allocators. The blocks are carved
20 : : * into chunks of exactly the right size, wasting only the space required to
21 : : * MAXALIGN the allocated chunks.
22 : : *
23 : : * Slab can also help reduce memory fragmentation in cases where longer-lived
24 : : * chunks remain stored on blocks while most of the other chunks have already
25 : : * been pfree'd. We give priority to putting new allocations into the
26 : : * "fullest" block. This help avoid having too many sparsely used blocks
27 : : * around and allows blocks to more easily become completely unused which
28 : : * allows them to be eventually free'd.
29 : : *
30 : : * We identify the "fullest" block to put new allocations on by using a block
31 : : * from the lowest populated element of the context's "blocklist" array.
32 : : * This is an array of dlists containing blocks which we partition by the
33 : : * number of free chunks which block has. Blocks with fewer free chunks are
34 : : * stored in a lower indexed dlist array slot. Full blocks go on the 0th
35 : : * element of the blocklist array. So that we don't have to have too many
36 : : * elements in the array, each dlist in the array is responsible for a range
37 : : * of free chunks. When a chunk is palloc'd or pfree'd we may need to move
38 : : * the block onto another dlist if the number of free chunks crosses the
39 : : * range boundary that the current list is responsible for. Having just a
40 : : * few blocklist elements reduces the number of times we must move the block
41 : : * onto another dlist element.
42 : : *
43 : : * We keep track of free chunks within each block by using a block-level free
44 : : * list. We consult this list when we allocate a new chunk in the block.
45 : : * The free list is a linked list, the head of which is pointed to with
46 : : * SlabBlock's freehead field. Each subsequent list item is stored in the
47 : : * free chunk's memory. We ensure chunks are large enough to store this
48 : : * address.
49 : : *
50 : : * When we allocate a new block, technically all chunks are free, however, to
51 : : * avoid having to write out the entire block to set the linked list for the
52 : : * free chunks for every chunk in the block, we instead store a pointer to
53 : : * the next "unused" chunk on the block and keep track of how many of these
54 : : * unused chunks there are. When a new block is malloc'd, all chunks are
55 : : * unused. The unused pointer starts with the first chunk on the block and
56 : : * as chunks are allocated, the unused pointer is incremented. As chunks are
57 : : * pfree'd, the unused pointer never goes backwards. The unused pointer can
58 : : * be thought of as a high watermark for the maximum number of chunks in the
59 : : * block which have been in use concurrently. When a chunk is pfree'd the
60 : : * chunk is put onto the head of the free list and the unused pointer is not
61 : : * changed. We only consume more unused chunks if we run out of free chunks
62 : : * on the free list. This method effectively gives priority to using
63 : : * previously used chunks over previously unused chunks, which should perform
64 : : * better due to CPU caching effects.
65 : : *
66 : : *-------------------------------------------------------------------------
67 : : */
68 : :
69 : : #include "postgres.h"
70 : :
71 : : #include "lib/ilist.h"
72 : : #include "utils/memdebug.h"
73 : : #include "utils/memutils.h"
74 : : #include "utils/memutils_internal.h"
75 : : #include "utils/memutils_memorychunk.h"
76 : :
77 : : #define Slab_BLOCKHDRSZ MAXALIGN(sizeof(SlabBlock))
78 : :
79 : : #ifdef MEMORY_CONTEXT_CHECKING
80 : : /*
81 : : * Size of the memory required to store the SlabContext.
82 : : * MEMORY_CONTEXT_CHECKING builds need some extra memory for the isChunkFree
83 : : * array.
84 : : */
85 : : #define Slab_CONTEXT_HDRSZ(chunksPerBlock) \
86 : : (sizeof(SlabContext) + ((chunksPerBlock) * sizeof(bool)))
87 : : #else
88 : : #define Slab_CONTEXT_HDRSZ(chunksPerBlock) sizeof(SlabContext)
89 : : #endif
90 : :
91 : : /*
92 : : * The number of partitions to divide the blocklist into based their number of
93 : : * free chunks. There must be at least 2.
94 : : */
95 : : #define SLAB_BLOCKLIST_COUNT 3
96 : :
97 : : /* The maximum number of completely empty blocks to keep around for reuse. */
98 : : #define SLAB_MAXIMUM_EMPTY_BLOCKS 10
99 : :
100 : : /*
101 : : * SlabContext is a specialized implementation of MemoryContext.
102 : : */
103 : : typedef struct SlabContext
104 : : {
105 : : MemoryContextData header; /* Standard memory-context fields */
106 : : /* Allocation parameters for this context: */
107 : : uint32 chunkSize; /* the requested (non-aligned) chunk size */
108 : : uint32 fullChunkSize; /* chunk size with chunk header and alignment */
109 : : uint32 blockSize; /* the size to make each block of chunks */
110 : : int32 chunksPerBlock; /* number of chunks that fit in 1 block */
111 : : int32 curBlocklistIndex; /* index into the blocklist[] element
112 : : * containing the fullest, blocks */
113 : : #ifdef MEMORY_CONTEXT_CHECKING
114 : : bool *isChunkFree; /* array to mark free chunks in a block during
115 : : * SlabCheck */
116 : : #endif
117 : :
118 : : int32 blocklist_shift; /* number of bits to shift the nfree count
119 : : * by to get the index into blocklist[] */
120 : : dclist_head emptyblocks; /* empty blocks to use up first instead of
121 : : * mallocing new blocks */
122 : :
123 : : /*
124 : : * Blocks with free space, grouped by the number of free chunks they
125 : : * contain. Completely full blocks are stored in the 0th element.
126 : : * Completely empty blocks are stored in emptyblocks or free'd if we have
127 : : * enough empty blocks already.
128 : : */
129 : : dlist_head blocklist[SLAB_BLOCKLIST_COUNT];
130 : : } SlabContext;
131 : :
132 : : /*
133 : : * SlabBlock
134 : : * Structure of a single slab block.
135 : : *
136 : : * slab: pointer back to the owning MemoryContext
137 : : * nfree: number of chunks on the block which are unallocated
138 : : * nunused: number of chunks on the block unallocated and not on the block's
139 : : * freelist.
140 : : * freehead: linked-list header storing a pointer to the first free chunk on
141 : : * the block. Subsequent pointers are stored in the chunk's memory. NULL
142 : : * indicates the end of the list.
143 : : * unused: pointer to the next chunk which has yet to be used.
144 : : * node: doubly-linked list node for the context's blocklist
145 : : */
146 : : typedef struct SlabBlock
147 : : {
148 : : SlabContext *slab; /* owning context */
149 : : int32 nfree; /* number of chunks on free + unused chunks */
150 : : int32 nunused; /* number of unused chunks */
151 : : MemoryChunk *freehead; /* pointer to the first free chunk */
152 : : MemoryChunk *unused; /* pointer to the next unused chunk */
153 : : dlist_node node; /* doubly-linked list for blocklist[] */
154 : : } SlabBlock;
155 : :
156 : :
157 : : #define Slab_CHUNKHDRSZ sizeof(MemoryChunk)
158 : : #define SlabChunkGetPointer(chk) \
159 : : ((void *) (((char *) (chk)) + sizeof(MemoryChunk)))
160 : :
161 : : /*
162 : : * SlabBlockGetChunk
163 : : * Obtain a pointer to the nth (0-based) chunk in the block
164 : : */
165 : : #define SlabBlockGetChunk(slab, block, n) \
166 : : ((MemoryChunk *) ((char *) (block) + Slab_BLOCKHDRSZ \
167 : : + ((n) * (slab)->fullChunkSize)))
168 : :
169 : : #if defined(MEMORY_CONTEXT_CHECKING) || defined(USE_ASSERT_CHECKING)
170 : :
171 : : /*
172 : : * SlabChunkIndex
173 : : * Get the 0-based index of how many chunks into the block the given
174 : : * chunk is.
175 : : */
176 : : #define SlabChunkIndex(slab, block, chunk) \
177 : : (((char *) (chunk) - (char *) SlabBlockGetChunk(slab, block, 0)) / \
178 : : (slab)->fullChunkSize)
179 : :
180 : : /*
181 : : * SlabChunkMod
182 : : * A MemoryChunk should always be at an address which is a multiple of
183 : : * fullChunkSize starting from the 0th chunk position. This will return
184 : : * non-zero if it's not.
185 : : */
186 : : #define SlabChunkMod(slab, block, chunk) \
187 : : (((char *) (chunk) - (char *) SlabBlockGetChunk(slab, block, 0)) % \
188 : : (slab)->fullChunkSize)
189 : :
190 : : #endif
191 : :
192 : : /*
193 : : * SlabIsValid
194 : : * True iff set is a valid slab allocation set.
195 : : */
196 : : #define SlabIsValid(set) ((set) && IsA(set, SlabContext))
197 : :
198 : : /*
199 : : * SlabBlockIsValid
200 : : * True iff block is a valid block of slab allocation set.
201 : : */
202 : : #define SlabBlockIsValid(block) \
203 : : ((block) && SlabIsValid((block)->slab))
204 : :
205 : : /*
206 : : * SlabBlocklistIndex
207 : : * Determine the blocklist index that a block should be in for the given
208 : : * number of free chunks.
209 : : */
210 : : static inline int32
211 : 5856878 : SlabBlocklistIndex(SlabContext *slab, int nfree)
212 : : {
213 : : int32 index;
214 : 5856878 : int32 blocklist_shift = slab->blocklist_shift;
215 : :
216 : : Assert(nfree >= 0 && nfree <= slab->chunksPerBlock);
217 : :
218 : : /*
219 : : * Determine the blocklist index based on the number of free chunks. We
220 : : * must ensure that 0 free chunks is dedicated to index 0. Everything
221 : : * else must be >= 1 and < SLAB_BLOCKLIST_COUNT.
222 : : *
223 : : * To make this as efficient as possible, we exploit some two's complement
224 : : * arithmetic where we reverse the sign before bit shifting. This results
225 : : * in an nfree of 0 using index 0 and anything non-zero staying non-zero.
226 : : * This is exploiting 0 and -0 being the same in two's complement. When
227 : : * we're done, we just need to flip the sign back over again for a
228 : : * positive index.
229 : : */
230 : 5856878 : index = -((-nfree) >> blocklist_shift);
231 : :
232 : : if (nfree == 0)
233 : : Assert(index == 0);
234 : : else
235 : : Assert(index >= 1 && index < SLAB_BLOCKLIST_COUNT);
236 : :
237 : 5856878 : return index;
238 : : }
239 : :
240 : : /*
241 : : * SlabFindNextBlockListIndex
242 : : * Search blocklist for blocks which have free chunks and return the
243 : : * index of the blocklist found containing at least 1 block with free
244 : : * chunks. If no block can be found we return 0.
245 : : *
246 : : * Note: We give priority to fuller blocks so that these are filled before
247 : : * emptier blocks. This is done to increase the chances that mostly-empty
248 : : * blocks will eventually become completely empty so they can be free'd.
249 : : */
250 : : static int32
251 : 100178 : SlabFindNextBlockListIndex(SlabContext *slab)
252 : : {
253 : : /* start at 1 as blocklist[0] is for full blocks. */
254 [ + + ]: 176497 : for (int i = 1; i < SLAB_BLOCKLIST_COUNT; i++)
255 : : {
256 : : /* return the first found non-empty index */
257 [ + + ]: 141064 : if (!dlist_is_empty(&slab->blocklist[i]))
258 : 64745 : return i;
259 : : }
260 : :
261 : : /* no blocks with free space */
262 : 35433 : return 0;
263 : : }
264 : :
265 : : /*
266 : : * SlabGetNextFreeChunk
267 : : * Return the next free chunk in block and update the block to account
268 : : * for the returned chunk now being used.
269 : : */
270 : : static inline MemoryChunk *
271 : 1918267 : SlabGetNextFreeChunk(SlabContext *slab, SlabBlock *block)
272 : : {
273 : : MemoryChunk *chunk;
274 : :
275 : : Assert(block->nfree > 0);
276 : :
277 [ + + ]: 1918267 : if (block->freehead != NULL)
278 : : {
279 : 1673677 : chunk = block->freehead;
280 : :
281 : : /*
282 : : * Pop the chunk from the linked list of free chunks. The pointer to
283 : : * the next free chunk is stored in the chunk itself.
284 : : */
285 : : VALGRIND_MAKE_MEM_DEFINED(SlabChunkGetPointer(chunk), sizeof(MemoryChunk *));
286 : 1673677 : block->freehead = *(MemoryChunk **) SlabChunkGetPointer(chunk);
287 : :
288 : : /* check nothing stomped on the free chunk's memory */
289 : : Assert(block->freehead == NULL ||
290 : : (block->freehead >= SlabBlockGetChunk(slab, block, 0) &&
291 : : block->freehead <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1) &&
292 : : SlabChunkMod(slab, block, block->freehead) == 0));
293 : : }
294 : : else
295 : : {
296 : : Assert(block->nunused > 0);
297 : :
298 : 244590 : chunk = block->unused;
299 : 244590 : block->unused = (MemoryChunk *) (((char *) block->unused) + slab->fullChunkSize);
300 : 244590 : block->nunused--;
301 : : }
302 : :
303 : 1918267 : block->nfree--;
304 : :
305 : 1918267 : return chunk;
306 : : }
307 : :
308 : : /*
309 : : * SlabContextCreate
310 : : * Create a new Slab context.
311 : : *
312 : : * parent: parent context, or NULL if top-level context
313 : : * name: name of context (must be statically allocated)
314 : : * blockSize: allocation block size
315 : : * chunkSize: allocation chunk size
316 : : *
317 : : * The Slab_CHUNKHDRSZ + MAXALIGN(chunkSize + 1) may not exceed
318 : : * MEMORYCHUNK_MAX_VALUE.
319 : : * 'blockSize' may not exceed MEMORYCHUNK_MAX_BLOCKOFFSET.
320 : : */
321 : : MemoryContext
322 : 658790 : SlabContextCreate(MemoryContext parent,
323 : : const char *name,
324 : : Size blockSize,
325 : : Size chunkSize)
326 : : {
327 : : int chunksPerBlock;
328 : : Size fullChunkSize;
329 : : SlabContext *slab;
330 : : int i;
331 : :
332 : : /* ensure MemoryChunk's size is properly maxaligned */
333 : : StaticAssertDecl(Slab_CHUNKHDRSZ == MAXALIGN(Slab_CHUNKHDRSZ),
334 : : "sizeof(MemoryChunk) is not maxaligned");
335 : : Assert(blockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET);
336 : :
337 : : /*
338 : : * Ensure there's enough space to store the pointer to the next free chunk
339 : : * in the memory of the (otherwise) unused allocation.
340 : : */
341 [ - + ]: 658790 : if (chunkSize < sizeof(MemoryChunk *))
342 : 0 : chunkSize = sizeof(MemoryChunk *);
343 : :
344 : : /* length of the maxaligned chunk including the chunk header */
345 : : #ifdef MEMORY_CONTEXT_CHECKING
346 : : /* ensure there's always space for the sentinel byte */
347 : : fullChunkSize = Slab_CHUNKHDRSZ + MAXALIGN(chunkSize + 1);
348 : : #else
349 : 658790 : fullChunkSize = Slab_CHUNKHDRSZ + MAXALIGN(chunkSize);
350 : : #endif
351 : :
352 : : Assert(fullChunkSize <= MEMORYCHUNK_MAX_VALUE);
353 : :
354 : : /* compute the number of chunks that will fit on each block */
355 : 658790 : chunksPerBlock = (blockSize - Slab_BLOCKHDRSZ) / fullChunkSize;
356 : :
357 : : /* Make sure the block can store at least one chunk. */
358 [ - + ]: 658790 : if (chunksPerBlock == 0)
359 [ # # ]: 0 : elog(ERROR, "block size %zu for slab is too small for %zu-byte chunks",
360 : : blockSize, chunkSize);
361 : :
362 : :
363 : :
364 : 658790 : slab = (SlabContext *) malloc(Slab_CONTEXT_HDRSZ(chunksPerBlock));
365 [ - + ]: 658790 : if (slab == NULL)
366 : : {
367 : 0 : MemoryContextStats(TopMemoryContext);
368 [ # # ]: 0 : ereport(ERROR,
369 : : (errcode(ERRCODE_OUT_OF_MEMORY),
370 : : errmsg("out of memory"),
371 : : errdetail("Failed while creating memory context \"%s\".",
372 : : name)));
373 : : }
374 : :
375 : : /*
376 : : * Avoid writing code that can fail between here and MemoryContextCreate;
377 : : * we'd leak the header if we ereport in this stretch.
378 : : */
379 : :
380 : : /* See comments about Valgrind interactions in aset.c */
381 : : VALGRIND_CREATE_MEMPOOL(slab, 0, false);
382 : : /* This vchunk covers the SlabContext only */
383 : : VALGRIND_MEMPOOL_ALLOC(slab, slab, sizeof(SlabContext));
384 : :
385 : : /* Fill in SlabContext-specific header fields */
386 : 658790 : slab->chunkSize = (uint32) chunkSize;
387 : 658790 : slab->fullChunkSize = (uint32) fullChunkSize;
388 : 658790 : slab->blockSize = (uint32) blockSize;
389 : 658790 : slab->chunksPerBlock = chunksPerBlock;
390 : 658790 : slab->curBlocklistIndex = 0;
391 : :
392 : : /*
393 : : * Compute a shift that guarantees that shifting chunksPerBlock with it is
394 : : * < SLAB_BLOCKLIST_COUNT - 1. The reason that we subtract 1 from
395 : : * SLAB_BLOCKLIST_COUNT in this calculation is that we reserve the 0th
396 : : * blocklist element for blocks which have no free chunks.
397 : : *
398 : : * We calculate the number of bits to shift by rather than a divisor to
399 : : * divide by as performing division each time we need to find the
400 : : * blocklist index would be much slower.
401 : : */
402 : 658790 : slab->blocklist_shift = 0;
403 [ + + ]: 4215273 : while ((slab->chunksPerBlock >> slab->blocklist_shift) >= (SLAB_BLOCKLIST_COUNT - 1))
404 : 3556483 : slab->blocklist_shift++;
405 : :
406 : : /* initialize the list to store empty blocks to be reused */
407 : 658790 : dclist_init(&slab->emptyblocks);
408 : :
409 : : /* initialize each blocklist slot */
410 [ + + ]: 2635160 : for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
411 : 1976370 : dlist_init(&slab->blocklist[i]);
412 : :
413 : : #ifdef MEMORY_CONTEXT_CHECKING
414 : : /* set the isChunkFree pointer right after the end of the context */
415 : : slab->isChunkFree = (bool *) ((char *) slab + sizeof(SlabContext));
416 : : #endif
417 : :
418 : : /* Finally, do the type-independent part of context creation */
419 : 658790 : MemoryContextCreate((MemoryContext) slab,
420 : : T_SlabContext,
421 : : MCTX_SLAB_ID,
422 : : parent,
423 : : name);
424 : :
425 : 658790 : return (MemoryContext) slab;
426 : : }
427 : :
428 : : /*
429 : : * SlabReset
430 : : * Frees all memory which is allocated in the given set.
431 : : *
432 : : * The code simply frees all the blocks in the context - we don't keep any
433 : : * keeper blocks or anything like that.
434 : : */
435 : : void
436 : 658258 : SlabReset(MemoryContext context)
437 : : {
438 : 658258 : SlabContext *slab = (SlabContext *) context;
439 : : dlist_mutable_iter miter;
440 : : int i;
441 : :
442 : : Assert(SlabIsValid(slab));
443 : :
444 : : #ifdef MEMORY_CONTEXT_CHECKING
445 : : /* Check for corruption and leaks before freeing */
446 : : SlabCheck(context);
447 : : #endif
448 : :
449 : : /* release any retained empty blocks */
450 [ + - + + ]: 659795 : dclist_foreach_modify(miter, &slab->emptyblocks)
451 : : {
452 : 1537 : SlabBlock *block = dlist_container(SlabBlock, node, miter.cur);
453 : :
454 : 1537 : dclist_delete_from(&slab->emptyblocks, miter.cur);
455 : :
456 : : #ifdef CLOBBER_FREED_MEMORY
457 : : wipe_mem(block, slab->blockSize);
458 : : #endif
459 : :
460 : : /* As in aset.c, free block-header vchunks explicitly */
461 : : VALGRIND_MEMPOOL_FREE(slab, block);
462 : :
463 : 1537 : free(block);
464 : 1537 : context->mem_allocated -= slab->blockSize;
465 : : }
466 : :
467 : : /* walk over blocklist and free the blocks */
468 [ + + ]: 2633032 : for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
469 : : {
470 [ + - + + ]: 2106143 : dlist_foreach_modify(miter, &slab->blocklist[i])
471 : : {
472 : 131369 : SlabBlock *block = dlist_container(SlabBlock, node, miter.cur);
473 : :
474 : 131369 : dlist_delete(miter.cur);
475 : :
476 : : #ifdef CLOBBER_FREED_MEMORY
477 : : wipe_mem(block, slab->blockSize);
478 : : #endif
479 : :
480 : : /* As in aset.c, free block-header vchunks explicitly */
481 : : VALGRIND_MEMPOOL_FREE(slab, block);
482 : :
483 : 131369 : free(block);
484 : 131369 : context->mem_allocated -= slab->blockSize;
485 : : }
486 : : }
487 : :
488 : : /*
489 : : * Instruct Valgrind to throw away all the vchunks associated with this
490 : : * context, except for the one covering the SlabContext. This gets rid of
491 : : * the vchunks for whatever user data is getting discarded by the context
492 : : * reset.
493 : : */
494 : : VALGRIND_MEMPOOL_TRIM(slab, slab, sizeof(SlabContext));
495 : :
496 : 658258 : slab->curBlocklistIndex = 0;
497 : :
498 : : Assert(context->mem_allocated == 0);
499 : 658258 : }
500 : :
501 : : /*
502 : : * SlabDelete
503 : : * Free all memory which is allocated in the given context.
504 : : */
505 : : void
506 : 658258 : SlabDelete(MemoryContext context)
507 : : {
508 : : /* Reset to release all the SlabBlocks */
509 : 658258 : SlabReset(context);
510 : :
511 : : /* Destroy the vpool -- see notes in aset.c */
512 : : VALGRIND_DESTROY_MEMPOOL(context);
513 : :
514 : : /* And free the context header */
515 : 658258 : free(context);
516 : 658258 : }
517 : :
518 : : /*
519 : : * Small helper for allocating a new chunk from a chunk, to avoid duplicating
520 : : * the code between SlabAlloc() and SlabAllocFromNewBlock().
521 : : */
522 : : static inline void *
523 : 2053744 : SlabAllocSetupNewChunk(MemoryContext context, SlabBlock *block,
524 : : MemoryChunk *chunk, Size size)
525 : : {
526 : 2053744 : SlabContext *slab = (SlabContext *) context;
527 : :
528 : : /*
529 : : * Check that the chunk pointer is actually somewhere on the block and is
530 : : * aligned as expected.
531 : : */
532 : : Assert(chunk >= SlabBlockGetChunk(slab, block, 0));
533 : : Assert(chunk <= SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1));
534 : : Assert(SlabChunkMod(slab, block, chunk) == 0);
535 : :
536 : : /* Prepare to initialize the chunk header. */
537 : : VALGRIND_MAKE_MEM_UNDEFINED(chunk, Slab_CHUNKHDRSZ);
538 : :
539 : 2053744 : MemoryChunkSetHdrMask(chunk, block, MAXALIGN(slab->chunkSize), MCTX_SLAB_ID);
540 : :
541 : : #ifdef MEMORY_CONTEXT_CHECKING
542 : : chunk->requested_size = size;
543 : : /* slab mark to catch clobber of "unused" space */
544 : : Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
545 : : set_sentinel(MemoryChunkGetPointer(chunk), size);
546 : : VALGRIND_MAKE_MEM_NOACCESS(((char *) chunk) + Slab_CHUNKHDRSZ +
547 : : slab->chunkSize,
548 : : slab->fullChunkSize -
549 : : (slab->chunkSize + Slab_CHUNKHDRSZ));
550 : : #endif
551 : :
552 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
553 : : /* fill the allocated space with junk */
554 : : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
555 : : #endif
556 : :
557 : : /* Disallow access to the chunk header. */
558 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
559 : :
560 : 2053744 : return MemoryChunkGetPointer(chunk);
561 : : }
562 : :
563 : : pg_noinline
564 : : static void *
565 : 166815 : SlabAllocFromNewBlock(MemoryContext context, Size size, int flags)
566 : : {
567 : 166815 : SlabContext *slab = (SlabContext *) context;
568 : : SlabBlock *block;
569 : : MemoryChunk *chunk;
570 : : dlist_head *blocklist;
571 : : int blocklist_idx;
572 : :
573 : : /* to save allocating a new one, first check the empty blocks list */
574 [ + + ]: 166815 : if (dclist_count(&slab->emptyblocks) > 0)
575 : : {
576 : 31338 : dlist_node *node = dclist_pop_head_node(&slab->emptyblocks);
577 : :
578 : 31338 : block = dlist_container(SlabBlock, node, node);
579 : :
580 : : /*
581 : : * SlabFree() should have left this block in a valid state with all
582 : : * chunks free. Ensure that's the case.
583 : : */
584 : : Assert(block->nfree == slab->chunksPerBlock);
585 : :
586 : : /* fetch the next chunk from this block */
587 : 31338 : chunk = SlabGetNextFreeChunk(slab, block);
588 : : }
589 : : else
590 : : {
591 : 135477 : block = (SlabBlock *) malloc(slab->blockSize);
592 : :
593 [ - + ]: 135477 : if (unlikely(block == NULL))
594 : 0 : return MemoryContextAllocationFailure(context, size, flags);
595 : :
596 : : /* Make a vchunk covering the new block's header */
597 : : VALGRIND_MEMPOOL_ALLOC(slab, block, Slab_BLOCKHDRSZ);
598 : :
599 : 135477 : block->slab = slab;
600 : 135477 : context->mem_allocated += slab->blockSize;
601 : :
602 : : /* use the first chunk in the new block */
603 : 135477 : chunk = SlabBlockGetChunk(slab, block, 0);
604 : :
605 : 135477 : block->nfree = slab->chunksPerBlock - 1;
606 : 135477 : block->unused = SlabBlockGetChunk(slab, block, 1);
607 : 135477 : block->freehead = NULL;
608 : 135477 : block->nunused = slab->chunksPerBlock - 1;
609 : : }
610 : :
611 : : /* find the blocklist element for storing blocks with 1 used chunk */
612 : 166815 : blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
613 : 166815 : blocklist = &slab->blocklist[blocklist_idx];
614 : :
615 : : /* this better be empty. We just added a block thinking it was */
616 : : Assert(dlist_is_empty(blocklist));
617 : :
618 : 166815 : dlist_push_head(blocklist, &block->node);
619 : :
620 : 166815 : slab->curBlocklistIndex = blocklist_idx;
621 : :
622 : 166815 : return SlabAllocSetupNewChunk(context, block, chunk, size);
623 : : }
624 : :
625 : : /*
626 : : * SlabAllocInvalidSize
627 : : * Handle raising an ERROR for an invalid size request. We don't do this
628 : : * in slab alloc as calling the elog functions would force the compiler
629 : : * to setup the stack frame in SlabAlloc. For performance reasons, we
630 : : * want to avoid that.
631 : : */
632 : : pg_noreturn
633 : : pg_noinline
634 : : static void
635 : 0 : SlabAllocInvalidSize(MemoryContext context, Size size)
636 : : {
637 : 0 : SlabContext *slab = (SlabContext *) context;
638 : :
639 [ # # ]: 0 : elog(ERROR, "unexpected alloc chunk size %zu (expected %u)", size,
640 : : slab->chunkSize);
641 : : }
642 : :
643 : : /*
644 : : * SlabAlloc
645 : : * Returns a pointer to a newly allocated memory chunk or raises an ERROR
646 : : * on allocation failure, or returns NULL when flags contains
647 : : * MCXT_ALLOC_NO_OOM. 'size' must be the same size as was specified
648 : : * during SlabContextCreate().
649 : : *
650 : : * This function should only contain the most common code paths. Everything
651 : : * else should be in pg_noinline helper functions, thus avoiding the overhead
652 : : * of creating a stack frame for the common cases. Allocating memory is often
653 : : * a bottleneck in many workloads, so avoiding stack frame setup is
654 : : * worthwhile. Helper functions should always directly return the newly
655 : : * allocated memory so that we can just return that address directly as a tail
656 : : * call.
657 : : */
658 : : void *
659 : 2053744 : SlabAlloc(MemoryContext context, Size size, int flags)
660 : : {
661 : 2053744 : SlabContext *slab = (SlabContext *) context;
662 : : SlabBlock *block;
663 : : MemoryChunk *chunk;
664 : :
665 : : Assert(SlabIsValid(slab));
666 : :
667 : : /* sanity check that this is pointing to a valid blocklist */
668 : : Assert(slab->curBlocklistIndex >= 0);
669 : : Assert(slab->curBlocklistIndex <= SlabBlocklistIndex(slab, slab->chunksPerBlock));
670 : :
671 : : /*
672 : : * Make sure we only allow correct request size. This doubles as the
673 : : * MemoryContextCheckSize check.
674 : : */
675 [ - + ]: 2053744 : if (unlikely(size != slab->chunkSize))
676 : 0 : SlabAllocInvalidSize(context, size);
677 : :
678 [ + + ]: 2053744 : if (unlikely(slab->curBlocklistIndex == 0))
679 : : {
680 : : /*
681 : : * Handle the case when there are no partially filled blocks
682 : : * available. This happens either when the last allocation took the
683 : : * last chunk in the block, or when SlabFree() free'd the final block.
684 : : */
685 : 166815 : return SlabAllocFromNewBlock(context, size, flags);
686 : : }
687 : : else
688 : : {
689 : 1886929 : dlist_head *blocklist = &slab->blocklist[slab->curBlocklistIndex];
690 : : int new_blocklist_idx;
691 : :
692 : : Assert(!dlist_is_empty(blocklist));
693 : :
694 : : /* grab the block from the blocklist */
695 : 1886929 : block = dlist_head_element(SlabBlock, node, blocklist);
696 : :
697 : : /* make sure we actually got a valid block, with matching nfree */
698 : : Assert(block != NULL);
699 : : Assert(slab->curBlocklistIndex == SlabBlocklistIndex(slab, block->nfree));
700 : : Assert(block->nfree > 0);
701 : :
702 : : /* fetch the next chunk from this block */
703 : 1886929 : chunk = SlabGetNextFreeChunk(slab, block);
704 : :
705 : : /* get the new blocklist index based on the new free chunk count */
706 : 1886929 : new_blocklist_idx = SlabBlocklistIndex(slab, block->nfree);
707 : :
708 : : /*
709 : : * Handle the case where the blocklist index changes. This also deals
710 : : * with blocks becoming full as only full blocks go at index 0.
711 : : */
712 [ + + ]: 1886929 : if (unlikely(slab->curBlocklistIndex != new_blocklist_idx))
713 : : {
714 : 43103 : dlist_delete_from(blocklist, &block->node);
715 : 43103 : dlist_push_head(&slab->blocklist[new_blocklist_idx], &block->node);
716 : :
717 [ + + ]: 43103 : if (dlist_is_empty(blocklist))
718 : 40836 : slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
719 : : }
720 : : }
721 : :
722 : 1886929 : return SlabAllocSetupNewChunk(context, block, chunk, size);
723 : : }
724 : :
725 : : /*
726 : : * SlabFree
727 : : * Frees allocated memory; memory is removed from the slab.
728 : : */
729 : : void
730 : 1901567 : SlabFree(void *pointer)
731 : : {
732 : 1901567 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
733 : : SlabBlock *block;
734 : : SlabContext *slab;
735 : : int curBlocklistIdx;
736 : : int newBlocklistIdx;
737 : :
738 : : /* Allow access to the chunk header. */
739 : : VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
740 : :
741 : 1901567 : block = MemoryChunkGetBlock(chunk);
742 : :
743 : : /*
744 : : * For speed reasons we just Assert that the referenced block is good.
745 : : * Future field experience may show that this Assert had better become a
746 : : * regular runtime test-and-elog check.
747 : : */
748 : : Assert(SlabBlockIsValid(block));
749 : 1901567 : slab = block->slab;
750 : :
751 : : #ifdef MEMORY_CONTEXT_CHECKING
752 : : /* See comments in AllocSetFree about uses of ERROR and WARNING here */
753 : : /* Test for previously-freed chunk */
754 : : if (unlikely(chunk->requested_size == InvalidAllocSize))
755 : : elog(ERROR, "detected double pfree in %s %p",
756 : : slab->header.name, chunk);
757 : : /* Test for someone scribbling on unused space in chunk */
758 : : Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
759 : : if (!sentinel_ok(pointer, slab->chunkSize))
760 : : elog(WARNING, "detected write past chunk end in %s %p",
761 : : slab->header.name, chunk);
762 : : /* Reset requested_size to InvalidAllocSize in free chunks */
763 : : chunk->requested_size = InvalidAllocSize;
764 : : #endif
765 : :
766 : : /* push this chunk onto the head of the block's free list */
767 : 1901567 : *(MemoryChunk **) pointer = block->freehead;
768 : 1901567 : block->freehead = chunk;
769 : :
770 : 1901567 : block->nfree++;
771 : :
772 : : Assert(block->nfree > 0);
773 : : Assert(block->nfree <= slab->chunksPerBlock);
774 : :
775 : : #ifdef CLOBBER_FREED_MEMORY
776 : : /* don't wipe the free list MemoryChunk pointer stored in the chunk */
777 : : wipe_mem((char *) pointer + sizeof(MemoryChunk *),
778 : : slab->chunkSize - sizeof(MemoryChunk *));
779 : : #endif
780 : :
781 : 1901567 : curBlocklistIdx = SlabBlocklistIndex(slab, block->nfree - 1);
782 : 1901567 : newBlocklistIdx = SlabBlocklistIndex(slab, block->nfree);
783 : :
784 : : /*
785 : : * Check if the block needs to be moved to another element on the
786 : : * blocklist based on it now having 1 more free chunk.
787 : : */
788 [ + + ]: 1901567 : if (unlikely(curBlocklistIdx != newBlocklistIdx))
789 : : {
790 : : /* do the move */
791 : 42655 : dlist_delete_from(&slab->blocklist[curBlocklistIdx], &block->node);
792 : 42655 : dlist_push_head(&slab->blocklist[newBlocklistIdx], &block->node);
793 : :
794 : : /*
795 : : * The blocklist[curBlocklistIdx] may now be empty or we may now be
796 : : * able to use a lower-element blocklist. We'll need to redetermine
797 : : * what the slab->curBlocklistIndex is if the current blocklist was
798 : : * changed or if a lower element one was changed. We must ensure we
799 : : * use the list with the fullest block(s).
800 : : */
801 [ + - ]: 42655 : if (slab->curBlocklistIndex >= curBlocklistIdx)
802 : : {
803 : 42655 : slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
804 : :
805 : : /*
806 : : * We know there must be a block with at least 1 unused chunk as
807 : : * we just pfree'd one. Ensure curBlocklistIndex reflects this.
808 : : */
809 : : Assert(slab->curBlocklistIndex > 0);
810 : : }
811 : : }
812 : :
813 : : /* Handle when a block becomes completely empty */
814 [ + + ]: 1901567 : if (unlikely(block->nfree == slab->chunksPerBlock))
815 : : {
816 : : /* remove the block */
817 : 35186 : dlist_delete_from(&slab->blocklist[newBlocklistIdx], &block->node);
818 : :
819 : : /*
820 : : * To avoid thrashing malloc/free, we keep a list of empty blocks that
821 : : * we can reuse again instead of having to malloc a new one.
822 : : */
823 [ + + ]: 35186 : if (dclist_count(&slab->emptyblocks) < SLAB_MAXIMUM_EMPTY_BLOCKS)
824 : 33434 : dclist_push_head(&slab->emptyblocks, &block->node);
825 : : else
826 : : {
827 : : /*
828 : : * When we have enough empty blocks stored already, we actually
829 : : * free the block.
830 : : */
831 : : #ifdef CLOBBER_FREED_MEMORY
832 : : wipe_mem(block, slab->blockSize);
833 : : #endif
834 : :
835 : : /* As in aset.c, free block-header vchunks explicitly */
836 : : VALGRIND_MEMPOOL_FREE(slab, block);
837 : :
838 : 1752 : free(block);
839 : 1752 : slab->header.mem_allocated -= slab->blockSize;
840 : : }
841 : :
842 : : /*
843 : : * Check if we need to reset the blocklist index. This is required
844 : : * when the blocklist this block is on has become completely empty.
845 : : */
846 [ + + + + ]: 54652 : if (slab->curBlocklistIndex == newBlocklistIdx &&
847 : 19466 : dlist_is_empty(&slab->blocklist[newBlocklistIdx]))
848 : 16687 : slab->curBlocklistIndex = SlabFindNextBlockListIndex(slab);
849 : : }
850 : 1901567 : }
851 : :
852 : : /*
853 : : * SlabRealloc
854 : : * Change the allocated size of a chunk.
855 : : *
856 : : * As Slab is designed for allocating equally-sized chunks of memory, it can't
857 : : * do an actual chunk size change. We try to be gentle and allow calls with
858 : : * exactly the same size, as in that case we can simply return the same
859 : : * chunk. When the size differs, we throw an error.
860 : : *
861 : : * We could also allow requests with size < chunkSize. That however seems
862 : : * rather pointless - Slab is meant for chunks of constant size, and moreover
863 : : * realloc is usually used to enlarge the chunk.
864 : : */
865 : : void *
866 : 0 : SlabRealloc(void *pointer, Size size, int flags)
867 : : {
868 : 0 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
869 : : SlabBlock *block;
870 : : SlabContext *slab;
871 : :
872 : : /* Allow access to the chunk header. */
873 : : VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
874 : :
875 : 0 : block = MemoryChunkGetBlock(chunk);
876 : :
877 : : /* Disallow access to the chunk header. */
878 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
879 : :
880 : : /*
881 : : * Try to verify that we have a sane block pointer: the block header
882 : : * should reference a slab context. (We use a test-and-elog, not just
883 : : * Assert, because it seems highly likely that we're here in error in the
884 : : * first place.)
885 : : */
886 [ # # # # : 0 : if (!SlabBlockIsValid(block))
# # ]
887 [ # # ]: 0 : elog(ERROR, "could not find block containing chunk %p", chunk);
888 : 0 : slab = block->slab;
889 : :
890 : : /* can't do actual realloc with slab, but let's try to be gentle */
891 [ # # ]: 0 : if (size == slab->chunkSize)
892 : 0 : return pointer;
893 : :
894 [ # # ]: 0 : elog(ERROR, "slab allocator does not support realloc()");
895 : : return NULL; /* keep compiler quiet */
896 : : }
897 : :
898 : : /*
899 : : * SlabGetChunkContext
900 : : * Return the MemoryContext that 'pointer' belongs to.
901 : : */
902 : : MemoryContext
903 : 0 : SlabGetChunkContext(void *pointer)
904 : : {
905 : 0 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
906 : : SlabBlock *block;
907 : :
908 : : /* Allow access to the chunk header. */
909 : : VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
910 : :
911 : 0 : block = MemoryChunkGetBlock(chunk);
912 : :
913 : : /* Disallow access to the chunk header. */
914 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
915 : :
916 : : Assert(SlabBlockIsValid(block));
917 : :
918 : 0 : return &block->slab->header;
919 : : }
920 : :
921 : : /*
922 : : * SlabGetChunkSpace
923 : : * Given a currently-allocated chunk, determine the total space
924 : : * it occupies (including all memory-allocation overhead).
925 : : */
926 : : Size
927 : 0 : SlabGetChunkSpace(void *pointer)
928 : : {
929 : 0 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
930 : : SlabBlock *block;
931 : : SlabContext *slab;
932 : :
933 : : /* Allow access to the chunk header. */
934 : : VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
935 : :
936 : 0 : block = MemoryChunkGetBlock(chunk);
937 : :
938 : : /* Disallow access to the chunk header. */
939 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
940 : :
941 : : Assert(SlabBlockIsValid(block));
942 : 0 : slab = block->slab;
943 : :
944 : 0 : return slab->fullChunkSize;
945 : : }
946 : :
947 : : /*
948 : : * SlabIsEmpty
949 : : * Is the slab empty of any allocated space?
950 : : */
951 : : bool
952 : 0 : SlabIsEmpty(MemoryContext context)
953 : : {
954 : : Assert(SlabIsValid((SlabContext *) context));
955 : :
956 : 0 : return (context->mem_allocated == 0);
957 : : }
958 : :
959 : : /*
960 : : * SlabStats
961 : : * Compute stats about memory consumption of a Slab context.
962 : : *
963 : : * printfunc: if not NULL, pass a human-readable stats string to this.
964 : : * passthru: pass this pointer through to printfunc.
965 : : * totals: if not NULL, add stats about this context into *totals.
966 : : * print_to_stderr: print stats to stderr if true, elog otherwise.
967 : : */
968 : : void
969 : 0 : SlabStats(MemoryContext context,
970 : : MemoryStatsPrintFunc printfunc, void *passthru,
971 : : MemoryContextCounters *totals,
972 : : bool print_to_stderr)
973 : : {
974 : 0 : SlabContext *slab = (SlabContext *) context;
975 : 0 : Size nblocks = 0;
976 : 0 : Size freechunks = 0;
977 : : Size totalspace;
978 : 0 : Size freespace = 0;
979 : : int i;
980 : :
981 : : Assert(SlabIsValid(slab));
982 : :
983 : : /* Include context header in totalspace */
984 : 0 : totalspace = Slab_CONTEXT_HDRSZ(slab->chunksPerBlock);
985 : :
986 : : /* Add the space consumed by blocks in the emptyblocks list */
987 : 0 : totalspace += dclist_count(&slab->emptyblocks) * slab->blockSize;
988 : :
989 [ # # ]: 0 : for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
990 : : {
991 : : dlist_iter iter;
992 : :
993 [ # # # # ]: 0 : dlist_foreach(iter, &slab->blocklist[i])
994 : : {
995 : 0 : SlabBlock *block = dlist_container(SlabBlock, node, iter.cur);
996 : :
997 : 0 : nblocks++;
998 : 0 : totalspace += slab->blockSize;
999 : 0 : freespace += slab->fullChunkSize * block->nfree;
1000 : 0 : freechunks += block->nfree;
1001 : : }
1002 : : }
1003 : :
1004 [ # # ]: 0 : if (printfunc)
1005 : : {
1006 : : char stats_string[200];
1007 : :
1008 : : /* XXX should we include free chunks on empty blocks? */
1009 : 0 : snprintf(stats_string, sizeof(stats_string),
1010 : : "%zu total in %zu blocks; %u empty blocks; %zu free (%zu chunks); %zu used",
1011 : 0 : totalspace, nblocks, dclist_count(&slab->emptyblocks),
1012 : : freespace, freechunks, totalspace - freespace);
1013 : 0 : printfunc(context, passthru, stats_string, print_to_stderr);
1014 : : }
1015 : :
1016 [ # # ]: 0 : if (totals)
1017 : : {
1018 : 0 : totals->nblocks += nblocks;
1019 : 0 : totals->freechunks += freechunks;
1020 : 0 : totals->totalspace += totalspace;
1021 : 0 : totals->freespace += freespace;
1022 : : }
1023 : 0 : }
1024 : :
1025 : :
1026 : : #ifdef MEMORY_CONTEXT_CHECKING
1027 : :
1028 : : /*
1029 : : * SlabCheck
1030 : : * Walk through all blocks looking for inconsistencies.
1031 : : *
1032 : : * NOTE: report errors as WARNING, *not* ERROR or FATAL. Otherwise you'll
1033 : : * find yourself in an infinite loop when trouble occurs, because this
1034 : : * routine will be entered again when elog cleanup tries to release memory!
1035 : : */
1036 : : void
1037 : : SlabCheck(MemoryContext context)
1038 : : {
1039 : : SlabContext *slab = (SlabContext *) context;
1040 : : int i;
1041 : : int nblocks = 0;
1042 : : const char *name = slab->header.name;
1043 : : dlist_iter iter;
1044 : :
1045 : : Assert(SlabIsValid(slab));
1046 : : Assert(slab->chunksPerBlock > 0);
1047 : :
1048 : : /*
1049 : : * Have a look at the empty blocks. These should have all their chunks
1050 : : * marked as free. Ensure that's the case.
1051 : : */
1052 : : dclist_foreach(iter, &slab->emptyblocks)
1053 : : {
1054 : : SlabBlock *block = dlist_container(SlabBlock, node, iter.cur);
1055 : :
1056 : : if (block->nfree != slab->chunksPerBlock)
1057 : : elog(WARNING, "problem in slab %s: empty block %p should have %d free chunks but has %d chunks free",
1058 : : name, block, slab->chunksPerBlock, block->nfree);
1059 : : }
1060 : :
1061 : : /* walk the non-empty block lists */
1062 : : for (i = 0; i < SLAB_BLOCKLIST_COUNT; i++)
1063 : : {
1064 : : int j,
1065 : : nfree;
1066 : :
1067 : : /* walk all blocks on this blocklist */
1068 : : dlist_foreach(iter, &slab->blocklist[i])
1069 : : {
1070 : : SlabBlock *block = dlist_container(SlabBlock, node, iter.cur);
1071 : : MemoryChunk *cur_chunk;
1072 : :
1073 : : /*
1074 : : * Make sure the number of free chunks (in the block header)
1075 : : * matches the position in the blocklist.
1076 : : */
1077 : : if (SlabBlocklistIndex(slab, block->nfree) != i)
1078 : : elog(WARNING, "problem in slab %s: block %p is on blocklist %d but should be on blocklist %d",
1079 : : name, block, i, SlabBlocklistIndex(slab, block->nfree));
1080 : :
1081 : : /* make sure the block is not empty */
1082 : : if (block->nfree >= slab->chunksPerBlock)
1083 : : elog(WARNING, "problem in slab %s: empty block %p incorrectly stored on blocklist element %d",
1084 : : name, block, i);
1085 : :
1086 : : /* make sure the slab pointer correctly points to this context */
1087 : : if (block->slab != slab)
1088 : : elog(WARNING, "problem in slab %s: bogus slab link in block %p",
1089 : : name, block);
1090 : :
1091 : : /* reset the array of free chunks for this block */
1092 : : memset(slab->isChunkFree, 0, (slab->chunksPerBlock * sizeof(bool)));
1093 : : nfree = 0;
1094 : :
1095 : : /* walk through the block's free list chunks */
1096 : : cur_chunk = block->freehead;
1097 : : while (cur_chunk != NULL)
1098 : : {
1099 : : int chunkidx = SlabChunkIndex(slab, block, cur_chunk);
1100 : :
1101 : : /*
1102 : : * Ensure the free list link points to something on the block
1103 : : * at an address aligned according to the full chunk size.
1104 : : */
1105 : : if (cur_chunk < SlabBlockGetChunk(slab, block, 0) ||
1106 : : cur_chunk > SlabBlockGetChunk(slab, block, slab->chunksPerBlock - 1) ||
1107 : : SlabChunkMod(slab, block, cur_chunk) != 0)
1108 : : elog(WARNING, "problem in slab %s: bogus free list link %p in block %p",
1109 : : name, cur_chunk, block);
1110 : :
1111 : : /* count the chunk and mark it free on the free chunk array */
1112 : : nfree++;
1113 : : slab->isChunkFree[chunkidx] = true;
1114 : :
1115 : : /* read pointer of the next free chunk */
1116 : : VALGRIND_MAKE_MEM_DEFINED(MemoryChunkGetPointer(cur_chunk), sizeof(MemoryChunk *));
1117 : : cur_chunk = *(MemoryChunk **) SlabChunkGetPointer(cur_chunk);
1118 : : }
1119 : :
1120 : : /* check that the unused pointer matches what nunused claims */
1121 : : if (SlabBlockGetChunk(slab, block, slab->chunksPerBlock - block->nunused) !=
1122 : : block->unused)
1123 : : elog(WARNING, "problem in slab %s: mismatch detected between nunused chunks and unused pointer in block %p",
1124 : : name, block);
1125 : :
1126 : : /*
1127 : : * count the remaining free chunks that have yet to make it onto
1128 : : * the block's free list.
1129 : : */
1130 : : cur_chunk = block->unused;
1131 : : for (j = 0; j < block->nunused; j++)
1132 : : {
1133 : : int chunkidx = SlabChunkIndex(slab, block, cur_chunk);
1134 : :
1135 : :
1136 : : /* count the chunk as free and mark it as so in the array */
1137 : : nfree++;
1138 : : if (chunkidx < slab->chunksPerBlock)
1139 : : slab->isChunkFree[chunkidx] = true;
1140 : :
1141 : : /* move forward 1 chunk */
1142 : : cur_chunk = (MemoryChunk *) (((char *) cur_chunk) + slab->fullChunkSize);
1143 : : }
1144 : :
1145 : : for (j = 0; j < slab->chunksPerBlock; j++)
1146 : : {
1147 : : if (!slab->isChunkFree[j])
1148 : : {
1149 : : MemoryChunk *chunk = SlabBlockGetChunk(slab, block, j);
1150 : : SlabBlock *chunkblock;
1151 : :
1152 : : /* Allow access to the chunk header. */
1153 : : VALGRIND_MAKE_MEM_DEFINED(chunk, Slab_CHUNKHDRSZ);
1154 : :
1155 : : chunkblock = (SlabBlock *) MemoryChunkGetBlock(chunk);
1156 : :
1157 : : /* Disallow access to the chunk header. */
1158 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, Slab_CHUNKHDRSZ);
1159 : :
1160 : : /*
1161 : : * check the chunk's blockoffset correctly points back to
1162 : : * the block
1163 : : */
1164 : : if (chunkblock != block)
1165 : : elog(WARNING, "problem in slab %s: bogus block link in block %p, chunk %p",
1166 : : name, block, chunk);
1167 : :
1168 : : /* check the sentinel byte is intact */
1169 : : Assert(slab->chunkSize < (slab->fullChunkSize - Slab_CHUNKHDRSZ));
1170 : : if (!sentinel_ok(chunk, Slab_CHUNKHDRSZ + slab->chunkSize))
1171 : : elog(WARNING, "problem in slab %s: detected write past chunk end in block %p, chunk %p",
1172 : : name, block, chunk);
1173 : : }
1174 : : }
1175 : :
1176 : : /*
1177 : : * Make sure we got the expected number of free chunks (as tracked
1178 : : * in the block header).
1179 : : */
1180 : : if (nfree != block->nfree)
1181 : : elog(WARNING, "problem in slab %s: nfree in block %p is %d but %d chunk were found as free",
1182 : : name, block, block->nfree, nfree);
1183 : :
1184 : : nblocks++;
1185 : : }
1186 : : }
1187 : :
1188 : : /* the stored empty blocks are tracked in mem_allocated too */
1189 : : nblocks += dclist_count(&slab->emptyblocks);
1190 : :
1191 : : Assert(nblocks * slab->blockSize == context->mem_allocated);
1192 : : }
1193 : :
1194 : : #endif /* MEMORY_CONTEXT_CHECKING */
|
