Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * aset.c
4 : : * Allocation set definitions.
5 : : *
6 : : * AllocSet is our standard implementation of the abstract MemoryContext
7 : : * type.
8 : : *
9 : : *
10 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
11 : : * Portions Copyright (c) 1994, Regents of the University of California
12 : : *
13 : : * IDENTIFICATION
14 : : * src/backend/utils/mmgr/aset.c
15 : : *
16 : : * NOTE:
17 : : * This is a new (Feb. 05, 1999) implementation of the allocation set
18 : : * routines. AllocSet...() does not use OrderedSet...() any more.
19 : : * Instead it manages allocations in a block pool by itself, combining
20 : : * many small allocations in a few bigger blocks. AllocSetFree() normally
21 : : * doesn't free() memory really. It just add's the free'd area to some
22 : : * list for later reuse by AllocSetAlloc(). All memory blocks are free()'d
23 : : * at once on AllocSetReset(), which happens when the memory context gets
24 : : * destroyed.
25 : : * Jan Wieck
26 : : *
27 : : * Performance improvement from Tom Lane, 8/99: for extremely large request
28 : : * sizes, we do want to be able to give the memory back to free() as soon
29 : : * as it is pfree()'d. Otherwise we risk tying up a lot of memory in
30 : : * freelist entries that might never be usable. This is specially needed
31 : : * when the caller is repeatedly repalloc()'ing a block bigger and bigger;
32 : : * the previous instances of the block were guaranteed to be wasted until
33 : : * AllocSetReset() under the old way.
34 : : *
35 : : * Further improvement 12/00: as the code stood, request sizes in the
36 : : * midrange between "small" and "large" were handled very inefficiently,
37 : : * because any sufficiently large free chunk would be used to satisfy a
38 : : * request, even if it was much larger than necessary. This led to more
39 : : * and more wasted space in allocated chunks over time. To fix, get rid
40 : : * of the midrange behavior: we now handle only "small" power-of-2-size
41 : : * chunks as chunks. Anything "large" is passed off to malloc(). Change
42 : : * the number of freelists to change the small/large boundary.
43 : : *
44 : : *-------------------------------------------------------------------------
45 : : */
46 : :
47 : : #include "postgres.h"
48 : :
49 : : #include "port/pg_bitutils.h"
50 : : #include "utils/memdebug.h"
51 : : #include "utils/memutils.h"
52 : : #include "utils/memutils_internal.h"
53 : : #include "utils/memutils_memorychunk.h"
54 : :
55 : : /*--------------------
56 : : * Chunk freelist k holds chunks of size 1 << (k + ALLOC_MINBITS),
57 : : * for k = 0 .. ALLOCSET_NUM_FREELISTS-1.
58 : : *
59 : : * Note that all chunks in the freelists have power-of-2 sizes. This
60 : : * improves recyclability: we may waste some space, but the wasted space
61 : : * should stay pretty constant as requests are made and released.
62 : : *
63 : : * A request too large for the last freelist is handled by allocating a
64 : : * dedicated block from malloc(). The block still has a block header and
65 : : * chunk header, but when the chunk is freed we'll return the whole block
66 : : * to malloc(), not put it on our freelists.
67 : : *
68 : : * CAUTION: ALLOC_MINBITS must be large enough so that
69 : : * 1<<ALLOC_MINBITS is at least MAXALIGN,
70 : : * or we may fail to align the smallest chunks adequately.
71 : : * 8-byte alignment is enough on all currently known machines. This 8-byte
72 : : * minimum also allows us to store a pointer to the next freelist item within
73 : : * the chunk of memory itself.
74 : : *
75 : : * With the current parameters, request sizes up to 8K are treated as chunks,
76 : : * larger requests go into dedicated blocks. Change ALLOCSET_NUM_FREELISTS
77 : : * to adjust the boundary point; and adjust ALLOCSET_SEPARATE_THRESHOLD in
78 : : * memutils.h to agree. (Note: in contexts with small maxBlockSize, we may
79 : : * set the allocChunkLimit to less than 8K, so as to avoid space wastage.)
80 : : *--------------------
81 : : */
82 : :
83 : : #define ALLOC_MINBITS 3 /* smallest chunk size is 8 bytes */
84 : : #define ALLOCSET_NUM_FREELISTS 11
85 : : #define ALLOC_CHUNK_LIMIT (1 << (ALLOCSET_NUM_FREELISTS-1+ALLOC_MINBITS))
86 : : /* Size of largest chunk that we use a fixed size for */
87 : : #define ALLOC_CHUNK_FRACTION 4
88 : : /* We allow chunks to be at most 1/4 of maxBlockSize (less overhead) */
89 : :
90 : : /* ALLOC_CHUNK_LIMIT must be equal to ALLOCSET_SEPARATE_THRESHOLD */
91 : : StaticAssertDecl(ALLOC_CHUNK_LIMIT == ALLOCSET_SEPARATE_THRESHOLD,
92 : : "ALLOC_CHUNK_LIMIT != ALLOCSET_SEPARATE_THRESHOLD");
93 : :
94 : : /*--------------------
95 : : * The first block allocated for an allocset has size initBlockSize.
96 : : * Each time we have to allocate another block, we double the block size
97 : : * (if possible, and without exceeding maxBlockSize), so as to reduce
98 : : * the bookkeeping load on malloc().
99 : : *
100 : : * Blocks allocated to hold oversize chunks do not follow this rule, however;
101 : : * they are just however big they need to be to hold that single chunk.
102 : : *
103 : : * Also, if a minContextSize is specified, the first block has that size,
104 : : * and then initBlockSize is used for the next one.
105 : : *--------------------
106 : : */
107 : :
108 : : #define ALLOC_BLOCKHDRSZ MAXALIGN(sizeof(AllocBlockData))
109 : : #define ALLOC_CHUNKHDRSZ sizeof(MemoryChunk)
110 : : #define FIRST_BLOCKHDRSZ (MAXALIGN(sizeof(AllocSetContext)) + \
111 : : ALLOC_BLOCKHDRSZ)
112 : :
113 : : typedef struct AllocBlockData *AllocBlock; /* forward reference */
114 : :
115 : : /*
116 : : * AllocPointer
117 : : * Aligned pointer which may be a member of an allocation set.
118 : : */
119 : : typedef void *AllocPointer;
120 : :
121 : : /*
122 : : * AllocFreeListLink
123 : : * When pfreeing memory, if we maintain a freelist for the given chunk's
124 : : * size then we use a AllocFreeListLink to point to the current item in
125 : : * the AllocSetContext's freelist and then set the given freelist element
126 : : * to point to the chunk being freed.
127 : : */
128 : : typedef struct AllocFreeListLink
129 : : {
130 : : MemoryChunk *next;
131 : : } AllocFreeListLink;
132 : :
133 : : /*
134 : : * Obtain a AllocFreeListLink for the given chunk. Allocation sizes are
135 : : * always at least sizeof(AllocFreeListLink), so we reuse the pointer's memory
136 : : * itself to store the freelist link.
137 : : */
138 : : #define GetFreeListLink(chkptr) \
139 : : (AllocFreeListLink *) ((char *) (chkptr) + ALLOC_CHUNKHDRSZ)
140 : :
141 : : /* Validate a freelist index retrieved from a chunk header */
142 : : #define FreeListIdxIsValid(fidx) \
143 : : ((fidx) >= 0 && (fidx) < ALLOCSET_NUM_FREELISTS)
144 : :
145 : : /* Determine the size of the chunk based on the freelist index */
146 : : #define GetChunkSizeFromFreeListIdx(fidx) \
147 : : ((((Size) 1) << ALLOC_MINBITS) << (fidx))
148 : :
149 : : /*
150 : : * AllocSetContext is our standard implementation of MemoryContext.
151 : : *
152 : : * Note: header.isReset means there is nothing for AllocSetReset to do.
153 : : * This is different from the aset being physically empty (empty blocks list)
154 : : * because we will still have a keeper block. It's also different from the set
155 : : * being logically empty, because we don't attempt to detect pfree'ing the
156 : : * last active chunk.
157 : : */
158 : : typedef struct AllocSetContext
159 : : {
160 : : MemoryContextData header; /* Standard memory-context fields */
161 : : /* Info about storage allocated in this context: */
162 : : AllocBlock blocks; /* head of list of blocks in this set */
163 : : MemoryChunk *freelist[ALLOCSET_NUM_FREELISTS]; /* free chunk lists */
164 : : /* Allocation parameters for this context: */
165 : : uint32 initBlockSize; /* initial block size */
166 : : uint32 maxBlockSize; /* maximum block size */
167 : : uint32 nextBlockSize; /* next block size to allocate */
168 : : uint32 allocChunkLimit; /* effective chunk size limit */
169 : : /* freelist this context could be put in, or -1 if not a candidate: */
170 : : int freeListIndex; /* index in context_freelists[], or -1 */
171 : : } AllocSetContext;
172 : :
173 : : typedef AllocSetContext *AllocSet;
174 : :
175 : : /*
176 : : * AllocBlock
177 : : * An AllocBlock is the unit of memory that is obtained by aset.c
178 : : * from malloc(). It contains one or more MemoryChunks, which are
179 : : * the units requested by palloc() and freed by pfree(). MemoryChunks
180 : : * cannot be returned to malloc() individually, instead they are put
181 : : * on freelists by pfree() and re-used by the next palloc() that has
182 : : * a matching request size.
183 : : *
184 : : * AllocBlockData is the header data for a block --- the usable space
185 : : * within the block begins at the next alignment boundary.
186 : : */
187 : : typedef struct AllocBlockData
188 : : {
189 : : AllocSet aset; /* aset that owns this block */
190 : : AllocBlock prev; /* prev block in aset's blocks list, if any */
191 : : AllocBlock next; /* next block in aset's blocks list, if any */
192 : : char *freeptr; /* start of free space in this block */
193 : : char *endptr; /* end of space in this block */
194 : : } AllocBlockData;
195 : :
196 : : /*
197 : : * AllocSetIsValid
198 : : * True iff set is valid allocation set.
199 : : */
200 : : #define AllocSetIsValid(set) \
201 : : ((set) && IsA(set, AllocSetContext))
202 : :
203 : : /*
204 : : * AllocBlockIsValid
205 : : * True iff block is valid block of allocation set.
206 : : */
207 : : #define AllocBlockIsValid(block) \
208 : : ((block) && AllocSetIsValid((block)->aset))
209 : :
210 : : /*
211 : : * We always store external chunks on a dedicated block. This makes fetching
212 : : * the block from an external chunk easy since it's always the first and only
213 : : * chunk on the block.
214 : : */
215 : : #define ExternalChunkGetBlock(chunk) \
216 : : (AllocBlock) ((char *) chunk - ALLOC_BLOCKHDRSZ)
217 : :
218 : : /*
219 : : * Rather than repeatedly creating and deleting memory contexts, we keep some
220 : : * freed contexts in freelists so that we can hand them out again with little
221 : : * work. Before putting a context in a freelist, we reset it so that it has
222 : : * only its initial malloc chunk and no others. To be a candidate for a
223 : : * freelist, a context must have the same minContextSize/initBlockSize as
224 : : * other contexts in the list; but its maxBlockSize is irrelevant since that
225 : : * doesn't affect the size of the initial chunk.
226 : : *
227 : : * We currently provide one freelist for ALLOCSET_DEFAULT_SIZES contexts
228 : : * and one for ALLOCSET_SMALL_SIZES contexts; the latter works for
229 : : * ALLOCSET_START_SMALL_SIZES too, since only the maxBlockSize differs.
230 : : *
231 : : * Ordinarily, we re-use freelist contexts in last-in-first-out order, in
232 : : * hopes of improving locality of reference. But if there get to be too
233 : : * many contexts in the list, we'd prefer to drop the most-recently-created
234 : : * contexts in hopes of keeping the process memory map compact.
235 : : * We approximate that by simply deleting all existing entries when the list
236 : : * overflows, on the assumption that queries that allocate a lot of contexts
237 : : * will probably free them in more or less reverse order of allocation.
238 : : *
239 : : * Contexts in a freelist are chained via their nextchild pointers.
240 : : */
241 : : #define MAX_FREE_CONTEXTS 100 /* arbitrary limit on freelist length */
242 : :
243 : : /* Obtain the keeper block for an allocation set */
244 : : #define KeeperBlock(set) \
245 : : ((AllocBlock) (((char *) set) + MAXALIGN(sizeof(AllocSetContext))))
246 : :
247 : : /* Check if the block is the keeper block of the given allocation set */
248 : : #define IsKeeperBlock(set, block) ((block) == (KeeperBlock(set)))
249 : :
250 : : typedef struct AllocSetFreeList
251 : : {
252 : : int num_free; /* current list length */
253 : : AllocSetContext *first_free; /* list header */
254 : : } AllocSetFreeList;
255 : :
256 : : /* context_freelists[0] is for default params, [1] for small params */
257 : : static AllocSetFreeList context_freelists[2] =
258 : : {
259 : : {
260 : : 0, NULL
261 : : },
262 : : {
263 : : 0, NULL
264 : : }
265 : : };
266 : :
267 : :
268 : : /* ----------
269 : : * AllocSetFreeIndex -
270 : : *
271 : : * Depending on the size of an allocation compute which freechunk
272 : : * list of the alloc set it belongs to. Caller must have verified
273 : : * that size <= ALLOC_CHUNK_LIMIT.
274 : : * ----------
275 : : */
276 : : static inline int
277 : 880461212 : AllocSetFreeIndex(Size size)
278 : : {
279 : : int idx;
280 : :
281 [ + + ]: 880461212 : if (size > (1 << ALLOC_MINBITS))
282 : : {
283 : : /*----------
284 : : * At this point we must compute ceil(log2(size >> ALLOC_MINBITS)).
285 : : * This is the same as
286 : : * pg_leftmost_one_pos32((size - 1) >> ALLOC_MINBITS) + 1
287 : : * or equivalently
288 : : * pg_leftmost_one_pos32(size - 1) - ALLOC_MINBITS + 1
289 : : *
290 : : * However, for platforms without intrinsic support, we duplicate the
291 : : * logic here, allowing an additional optimization. It's reasonable
292 : : * to assume that ALLOC_CHUNK_LIMIT fits in 16 bits, so we can unroll
293 : : * the byte-at-a-time loop in pg_leftmost_one_pos32 and just handle
294 : : * the last two bytes.
295 : : *
296 : : * Yes, this function is enough of a hot-spot to make it worth this
297 : : * much trouble.
298 : : *----------
299 : : */
300 : : #ifdef HAVE_BITSCAN_REVERSE
301 : 759620322 : idx = pg_leftmost_one_pos32((uint32) size - 1) - ALLOC_MINBITS + 1;
302 : : #else
303 : : uint32 t,
304 : : tsize;
305 : :
306 : : /* Statically assert that we only have a 16-bit input value. */
307 : : StaticAssertDecl(ALLOC_CHUNK_LIMIT < (1 << 16),
308 : : "ALLOC_CHUNK_LIMIT must be less than 64kB");
309 : :
310 : : tsize = size - 1;
311 : : t = tsize >> 8;
312 : : idx = t ? pg_leftmost_one_pos[t] + 8 : pg_leftmost_one_pos[tsize];
313 : : idx -= ALLOC_MINBITS - 1;
314 : : #endif
315 : :
316 : : Assert(idx < ALLOCSET_NUM_FREELISTS);
317 : : }
318 : : else
319 : 120840890 : idx = 0;
320 : :
321 : 880461212 : return idx;
322 : : }
323 : :
324 : : /* ----------
325 : : * AllocSetFreeIndex -
326 : : *
327 : : * Compute and return the multi-chunk block chunk size limit for the
328 : : * given maximum block size.
329 : : * ----------
330 : : */
331 : : static inline uint32
332 : 2586155 : AllocSetComputeChunkLimit(Size maxBlockSize)
333 : : {
334 : : uint32 allocChunkLimit;
335 : :
336 : : /*
337 : : * Compute the allocation chunk size limit for this context. It can't be
338 : : * more than ALLOC_CHUNK_LIMIT because of the fixed number of freelists.
339 : : * If maxBlockSize is small then requests exceeding the maxBlockSize, or
340 : : * even a significant fraction of it, should be treated as large chunks
341 : : * too. For the typical case of maxBlockSize a power of 2, the chunk size
342 : : * limit will be at most 1/8th maxBlockSize, so that given a stream of
343 : : * requests that are all the maximum chunk size we will waste at most
344 : : * 1/8th of the allocated space.
345 : : *
346 : : * Determine the maximum size that a chunk can be before we allocate an
347 : : * entire AllocBlock dedicated for that chunk. We set the absolute limit
348 : : * of that size as ALLOC_CHUNK_LIMIT but we reduce it further so that we
349 : : * can fit about ALLOC_CHUNK_FRACTION chunks this size on a maximally
350 : : * sized block. (We opt to keep allocChunkLimit a power-of-2 value
351 : : * primarily for legacy reasons rather than calculating it so that exactly
352 : : * ALLOC_CHUNK_FRACTION chunks fit on a maximally sized block.)
353 : : */
354 : 2586155 : allocChunkLimit = ALLOC_CHUNK_LIMIT;
355 : 2586155 : while ((Size) (allocChunkLimit + ALLOC_CHUNKHDRSZ) >
356 [ + + ]: 8655108 : (Size) ((maxBlockSize - ALLOC_BLOCKHDRSZ) / ALLOC_CHUNK_FRACTION))
357 : 6068953 : allocChunkLimit >>= 1;
358 : :
359 : 2586155 : return allocChunkLimit;
360 : : }
361 : :
362 : :
363 : : /*
364 : : * Public routines
365 : : */
366 : :
367 : :
368 : : /*
369 : : * AllocSetContextCreateInternal
370 : : * Create a new AllocSet context.
371 : : *
372 : : * parent: parent context, or NULL if top-level context
373 : : * name: name of context (must be statically allocated)
374 : : * minContextSize: minimum context size
375 : : * initBlockSize: initial allocation block size
376 : : * maxBlockSize: maximum allocation block size
377 : : *
378 : : * Most callers should abstract the context size parameters using a macro
379 : : * such as ALLOCSET_DEFAULT_SIZES.
380 : : *
381 : : * Note: don't call this directly; go through the wrapper macro
382 : : * AllocSetContextCreate.
383 : : */
384 : : MemoryContext
385 : 8416110 : AllocSetContextCreateInternal(MemoryContext parent,
386 : : const char *name,
387 : : Size minContextSize,
388 : : Size initBlockSize,
389 : : Size maxBlockSize)
390 : : {
391 : : int freeListIndex;
392 : : Size firstBlockSize;
393 : : AllocSet set;
394 : : AllocBlock block;
395 : :
396 : : /* ensure MemoryChunk's size is properly maxaligned */
397 : : StaticAssertDecl(ALLOC_CHUNKHDRSZ == MAXALIGN(ALLOC_CHUNKHDRSZ),
398 : : "sizeof(MemoryChunk) is not maxaligned");
399 : : /* check we have enough space to store the freelist link */
400 : : StaticAssertDecl(sizeof(AllocFreeListLink) <= (1 << ALLOC_MINBITS),
401 : : "sizeof(AllocFreeListLink) larger than minimum allocation size");
402 : :
403 : : /*
404 : : * First, validate allocation parameters. Once these were regular runtime
405 : : * tests and elog's, but in practice Asserts seem sufficient because
406 : : * nobody varies their parameters at runtime. We somewhat arbitrarily
407 : : * enforce a minimum 1K block size. We restrict the maximum block size to
408 : : * MEMORYCHUNK_MAX_BLOCKOFFSET as MemoryChunks are limited to this in
409 : : * regards to addressing the offset between the chunk and the block that
410 : : * the chunk is stored on. We would be unable to store the offset between
411 : : * the chunk and block for any chunks that were beyond
412 : : * MEMORYCHUNK_MAX_BLOCKOFFSET bytes into the block if the block was to be
413 : : * larger than this.
414 : : */
415 : : Assert(initBlockSize == MAXALIGN(initBlockSize) &&
416 : : initBlockSize >= 1024);
417 : : Assert(maxBlockSize == MAXALIGN(maxBlockSize) &&
418 : : maxBlockSize >= initBlockSize &&
419 : : AllocHugeSizeIsValid(maxBlockSize)); /* must be safe to double */
420 : : Assert(minContextSize == 0 ||
421 : : (minContextSize == MAXALIGN(minContextSize) &&
422 : : minContextSize >= 1024 &&
423 : : minContextSize <= maxBlockSize));
424 : : Assert(maxBlockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET);
425 : :
426 : : /*
427 : : * Check whether the parameters match either available freelist. We do
428 : : * not need to demand a match of maxBlockSize.
429 : : */
430 [ + + + + ]: 8416110 : if (minContextSize == ALLOCSET_DEFAULT_MINSIZE &&
431 : : initBlockSize == ALLOCSET_DEFAULT_INITSIZE)
432 : 5439015 : freeListIndex = 0;
433 [ + + + - ]: 2977095 : else if (minContextSize == ALLOCSET_SMALL_MINSIZE &&
434 : : initBlockSize == ALLOCSET_SMALL_INITSIZE)
435 : 2953274 : freeListIndex = 1;
436 : : else
437 : 23821 : freeListIndex = -1;
438 : :
439 : : /*
440 : : * If a suitable freelist entry exists, just recycle that context.
441 : : */
442 [ + + ]: 8416110 : if (freeListIndex >= 0)
443 : : {
444 : 8392289 : AllocSetFreeList *freelist = &context_freelists[freeListIndex];
445 : :
446 [ + + ]: 8392289 : if (freelist->first_free != NULL)
447 : : {
448 : : /* Remove entry from freelist */
449 : 5908932 : set = freelist->first_free;
450 : 5908932 : freelist->first_free = (AllocSet) set->header.nextchild;
451 : 5908932 : freelist->num_free--;
452 : :
453 : : /* nextBlockSize should have been reset in AllocSetReset() */
454 : : Assert(set->nextBlockSize == set->initBlockSize);
455 : :
456 [ + + ]: 5908932 : if (set->maxBlockSize != maxBlockSize)
457 : : {
458 : : /*
459 : : * Update its maxBlockSize and allocChunkLimit; everything
460 : : * else should be OK.
461 : : */
462 : 78977 : set->maxBlockSize = maxBlockSize;
463 : 78977 : set->allocChunkLimit = AllocSetComputeChunkLimit(maxBlockSize);
464 : : }
465 : :
466 : : /* Reinitialize its header, installing correct name and parent */
467 : 5908932 : MemoryContextCreate((MemoryContext) set,
468 : : T_AllocSetContext,
469 : : MCTX_ASET_ID,
470 : : parent,
471 : : name);
472 : :
473 : 5908932 : ((MemoryContext) set)->mem_allocated =
474 : 5908932 : KeeperBlock(set)->endptr - ((char *) set);
475 : :
476 : 5908932 : return (MemoryContext) set;
477 : : }
478 : : }
479 : :
480 : : /* Determine size of initial block */
481 : 2507178 : firstBlockSize = MAXALIGN(sizeof(AllocSetContext)) +
482 : : ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
483 [ + + ]: 2507178 : if (minContextSize != 0)
484 : 23821 : firstBlockSize = Max(firstBlockSize, minContextSize);
485 : : else
486 : 2483357 : firstBlockSize = Max(firstBlockSize, initBlockSize);
487 : :
488 : : /*
489 : : * Allocate the initial block. Unlike other aset.c blocks, it starts with
490 : : * the context header and its block header follows that.
491 : : */
492 : 2507178 : set = (AllocSet) malloc(firstBlockSize);
493 [ - + ]: 2507178 : if (set == NULL)
494 : : {
495 [ # # ]: 0 : if (TopMemoryContext)
496 : 0 : MemoryContextStats(TopMemoryContext);
497 [ # # ]: 0 : ereport(ERROR,
498 : : (errcode(ERRCODE_OUT_OF_MEMORY),
499 : : errmsg("out of memory"),
500 : : errdetail("Failed while creating memory context \"%s\".",
501 : : name)));
502 : : }
503 : :
504 : : /*
505 : : * Avoid writing code that can fail between here and MemoryContextCreate;
506 : : * we'd leak the header/initial block if we ereport in this stretch.
507 : : */
508 : :
509 : : /* Create a vpool associated with the context */
510 : : VALGRIND_CREATE_MEMPOOL(set, 0, false);
511 : :
512 : : /*
513 : : * Create a vchunk covering both the AllocSetContext struct and the keeper
514 : : * block's header. (Perhaps it would be more sensible for these to be two
515 : : * separate vchunks, but doing that seems to tickle bugs in some versions
516 : : * of Valgrind.) We must have these vchunks, and also a vchunk for each
517 : : * subsequently-added block header, so that Valgrind considers the
518 : : * pointers within them while checking for leaked memory. Note that
519 : : * Valgrind doesn't distinguish between these vchunks and those created by
520 : : * mcxt.c for the user-accessible-data chunks we allocate.
521 : : */
522 : : VALGRIND_MEMPOOL_ALLOC(set, set, FIRST_BLOCKHDRSZ);
523 : :
524 : : /* Fill in the initial block's block header */
525 : 2507178 : block = KeeperBlock(set);
526 : 2507178 : block->aset = set;
527 : 2507178 : block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
528 : 2507178 : block->endptr = ((char *) set) + firstBlockSize;
529 : 2507178 : block->prev = NULL;
530 : 2507178 : block->next = NULL;
531 : :
532 : : /* Mark unallocated space NOACCESS; leave the block header alone. */
533 : : VALGRIND_MAKE_MEM_NOACCESS(block->freeptr, block->endptr - block->freeptr);
534 : :
535 : : /* Remember block as part of block list */
536 : 2507178 : set->blocks = block;
537 : :
538 : : /* Finish filling in aset-specific parts of the context header */
539 [ + - + - : 30086136 : MemSetAligned(set->freelist, 0, sizeof(set->freelist));
+ - + + ]
540 : :
541 : 2507178 : set->initBlockSize = (uint32) initBlockSize;
542 : 2507178 : set->maxBlockSize = (uint32) maxBlockSize;
543 : 2507178 : set->nextBlockSize = (uint32) initBlockSize;
544 : 2507178 : set->freeListIndex = freeListIndex;
545 : :
546 : 2507178 : set->allocChunkLimit = AllocSetComputeChunkLimit(maxBlockSize);
547 : :
548 : : /* Finally, do the type-independent part of context creation */
549 : 2507178 : MemoryContextCreate((MemoryContext) set,
550 : : T_AllocSetContext,
551 : : MCTX_ASET_ID,
552 : : parent,
553 : : name);
554 : :
555 : 2507178 : ((MemoryContext) set)->mem_allocated = firstBlockSize;
556 : :
557 : 2507178 : return (MemoryContext) set;
558 : : }
559 : :
560 : : /*
561 : : * AllocSetReset
562 : : * Frees all memory which is allocated in the given set.
563 : : *
564 : : * Actually, this routine has some discretion about what to do.
565 : : * It should mark all allocated chunks freed, but it need not necessarily
566 : : * give back all the resources the set owns. Our actual implementation is
567 : : * that we give back all but the "keeper" block (which we must keep, since
568 : : * it shares a malloc chunk with the context header). In this way, we don't
569 : : * thrash malloc() when a context is repeatedly reset after small allocations,
570 : : * which is typical behavior for per-tuple contexts.
571 : : */
572 : : void
573 : 29609655 : AllocSetReset(MemoryContext context)
574 : : {
575 : 29609655 : AllocSet set = (AllocSet) context;
576 : : AllocBlock block;
577 : : Size keepersize PG_USED_FOR_ASSERTS_ONLY;
578 : :
579 : : Assert(AllocSetIsValid(set));
580 : :
581 : : #ifdef MEMORY_CONTEXT_CHECKING
582 : : /* Check for corruption and leaks before freeing */
583 : : AllocSetCheck(context);
584 : : #endif
585 : :
586 : : /* Remember keeper block size for Assert below */
587 : 29609655 : keepersize = KeeperBlock(set)->endptr - ((char *) set);
588 : :
589 : : /* Clear chunk freelists */
590 [ + - + - : 355315860 : MemSetAligned(set->freelist, 0, sizeof(set->freelist));
+ - + + ]
591 : :
592 : 29609655 : block = set->blocks;
593 : :
594 : : /* New blocks list will be just the keeper block */
595 : 29609655 : set->blocks = KeeperBlock(set);
596 : :
597 [ + + ]: 63522532 : while (block != NULL)
598 : : {
599 : 33912877 : AllocBlock next = block->next;
600 : :
601 [ + + ]: 33912877 : if (IsKeeperBlock(set, block))
602 : : {
603 : : /* Reset the block, but don't return it to malloc */
604 : 29609655 : char *datastart = ((char *) block) + ALLOC_BLOCKHDRSZ;
605 : :
606 : : #ifdef CLOBBER_FREED_MEMORY
607 : : wipe_mem(datastart, block->freeptr - datastart);
608 : : #else
609 : : /* wipe_mem() would have done this */
610 : : VALGRIND_MAKE_MEM_NOACCESS(datastart, block->freeptr - datastart);
611 : : #endif
612 : 29609655 : block->freeptr = datastart;
613 : 29609655 : block->prev = NULL;
614 : 29609655 : block->next = NULL;
615 : : }
616 : : else
617 : : {
618 : : /* Normal case, release the block */
619 : 4303222 : context->mem_allocated -= block->endptr - ((char *) block);
620 : :
621 : : #ifdef CLOBBER_FREED_MEMORY
622 : : wipe_mem(block, block->freeptr - ((char *) block));
623 : : #endif
624 : :
625 : : /*
626 : : * We need to free the block header's vchunk explicitly, although
627 : : * the user-data vchunks within will go away in the TRIM below.
628 : : * Otherwise Valgrind complains about leaked allocations.
629 : : */
630 : : VALGRIND_MEMPOOL_FREE(set, block);
631 : :
632 : 4303222 : free(block);
633 : : }
634 : 33912877 : block = next;
635 : : }
636 : :
637 : : Assert(context->mem_allocated == keepersize);
638 : :
639 : : /*
640 : : * Instruct Valgrind to throw away all the vchunks associated with this
641 : : * context, except for the one covering the AllocSetContext and
642 : : * keeper-block header. This gets rid of the vchunks for whatever user
643 : : * data is getting discarded by the context reset.
644 : : */
645 : : VALGRIND_MEMPOOL_TRIM(set, set, FIRST_BLOCKHDRSZ);
646 : :
647 : : /* Reset block size allocation sequence, too */
648 : 29609655 : set->nextBlockSize = set->initBlockSize;
649 : 29609655 : }
650 : :
651 : : /*
652 : : * AllocSetDelete
653 : : * Frees all memory which is allocated in the given set,
654 : : * in preparation for deletion of the set.
655 : : *
656 : : * Unlike AllocSetReset, this *must* free all resources of the set.
657 : : */
658 : : void
659 : 6232122 : AllocSetDelete(MemoryContext context)
660 : : {
661 : 6232122 : AllocSet set = (AllocSet) context;
662 : 6232122 : AllocBlock block = set->blocks;
663 : : Size keepersize PG_USED_FOR_ASSERTS_ONLY;
664 : :
665 : : Assert(AllocSetIsValid(set));
666 : :
667 : : #ifdef MEMORY_CONTEXT_CHECKING
668 : : /* Check for corruption and leaks before freeing */
669 : : AllocSetCheck(context);
670 : : #endif
671 : :
672 : : /* Remember keeper block size for Assert below */
673 : 6232122 : keepersize = KeeperBlock(set)->endptr - ((char *) set);
674 : :
675 : : /*
676 : : * If the context is a candidate for a freelist, put it into that freelist
677 : : * instead of destroying it.
678 : : */
679 [ + - ]: 6232122 : if (set->freeListIndex >= 0)
680 : : {
681 : 6232122 : AllocSetFreeList *freelist = &context_freelists[set->freeListIndex];
682 : :
683 : : /*
684 : : * Reset the context, if it needs it, so that we aren't hanging on to
685 : : * more than the initial malloc chunk.
686 : : */
687 [ + + ]: 6232122 : if (!context->isReset)
688 : 3839876 : MemoryContextResetOnly(context);
689 : :
690 : : /*
691 : : * If the freelist is full, just discard what's already in it. See
692 : : * comments with context_freelists[].
693 : : */
694 [ + + ]: 6232122 : if (freelist->num_free >= MAX_FREE_CONTEXTS)
695 : : {
696 [ + + ]: 38784 : while (freelist->first_free != NULL)
697 : : {
698 : 38400 : AllocSetContext *oldset = freelist->first_free;
699 : :
700 : 38400 : freelist->first_free = (AllocSetContext *) oldset->header.nextchild;
701 : 38400 : freelist->num_free--;
702 : :
703 : : /* Destroy the context's vpool --- see notes below */
704 : : VALGRIND_DESTROY_MEMPOOL(oldset);
705 : :
706 : : /* All that remains is to free the header/initial block */
707 : 38400 : free(oldset);
708 : : }
709 : : Assert(freelist->num_free == 0);
710 : : }
711 : :
712 : : /* Now add the just-deleted context to the freelist. */
713 : 6232122 : set->header.nextchild = (MemoryContext) freelist->first_free;
714 : 6232122 : freelist->first_free = set;
715 : 6232122 : freelist->num_free++;
716 : :
717 : 6232122 : return;
718 : : }
719 : :
720 : : /* Free all blocks, except the keeper which is part of context header */
721 [ # # ]: 0 : while (block != NULL)
722 : : {
723 : 0 : AllocBlock next = block->next;
724 : :
725 [ # # ]: 0 : if (!IsKeeperBlock(set, block))
726 : 0 : context->mem_allocated -= block->endptr - ((char *) block);
727 : :
728 : : #ifdef CLOBBER_FREED_MEMORY
729 : : wipe_mem(block, block->freeptr - ((char *) block));
730 : : #endif
731 : :
732 [ # # ]: 0 : if (!IsKeeperBlock(set, block))
733 : : {
734 : : /* As in AllocSetReset, free block-header vchunks explicitly */
735 : : VALGRIND_MEMPOOL_FREE(set, block);
736 : 0 : free(block);
737 : : }
738 : :
739 : 0 : block = next;
740 : : }
741 : :
742 : : Assert(context->mem_allocated == keepersize);
743 : :
744 : : /*
745 : : * Destroy the vpool. We don't seem to need to explicitly free the
746 : : * initial block's header vchunk, nor any user-data vchunks that Valgrind
747 : : * still knows about; they'll all go away automatically.
748 : : */
749 : : VALGRIND_DESTROY_MEMPOOL(set);
750 : :
751 : : /* Finally, free the context header, including the keeper block */
752 : 0 : free(set);
753 : : }
754 : :
755 : : /*
756 : : * Helper for AllocSetAlloc() that allocates an entire block for the chunk.
757 : : *
758 : : * AllocSetAlloc()'s comment explains why this is separate.
759 : : */
760 : : pg_noinline
761 : : static void *
762 : 12709938 : AllocSetAllocLarge(MemoryContext context, Size size, int flags)
763 : : {
764 : 12709938 : AllocSet set = (AllocSet) context;
765 : : AllocBlock block;
766 : : MemoryChunk *chunk;
767 : : Size chunk_size;
768 : : Size blksize;
769 : :
770 : : /* validate 'size' is within the limits for the given 'flags' */
771 : 12709938 : MemoryContextCheckSize(context, size, flags);
772 : :
773 : : #ifdef MEMORY_CONTEXT_CHECKING
774 : : /* ensure there's always space for the sentinel byte */
775 : : chunk_size = MAXALIGN(size + 1);
776 : : #else
777 : 12709938 : chunk_size = MAXALIGN(size);
778 : : #endif
779 : :
780 : 12709938 : blksize = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
781 : 12709938 : block = (AllocBlock) malloc(blksize);
782 [ - + ]: 12709938 : if (block == NULL)
783 : 0 : return MemoryContextAllocationFailure(context, size, flags);
784 : :
785 : : /* Make a vchunk covering the new block's header */
786 : : VALGRIND_MEMPOOL_ALLOC(set, block, ALLOC_BLOCKHDRSZ);
787 : :
788 : 12709938 : context->mem_allocated += blksize;
789 : :
790 : 12709938 : block->aset = set;
791 : 12709938 : block->freeptr = block->endptr = ((char *) block) + blksize;
792 : :
793 : 12709938 : chunk = (MemoryChunk *) (((char *) block) + ALLOC_BLOCKHDRSZ);
794 : :
795 : : /* mark the MemoryChunk as externally managed */
796 : 12709938 : MemoryChunkSetHdrMaskExternal(chunk, MCTX_ASET_ID);
797 : :
798 : : #ifdef MEMORY_CONTEXT_CHECKING
799 : : chunk->requested_size = size;
800 : : /* set mark to catch clobber of "unused" space */
801 : : Assert(size < chunk_size);
802 : : set_sentinel(MemoryChunkGetPointer(chunk), size);
803 : : #endif
804 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
805 : : /* fill the allocated space with junk */
806 : : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
807 : : #endif
808 : :
809 : : /*
810 : : * Stick the new block underneath the active allocation block, if any, so
811 : : * that we don't lose the use of the space remaining therein.
812 : : */
813 [ + - ]: 12709938 : if (set->blocks != NULL)
814 : : {
815 : 12709938 : block->prev = set->blocks;
816 : 12709938 : block->next = set->blocks->next;
817 [ + + ]: 12709938 : if (block->next)
818 : 9444075 : block->next->prev = block;
819 : 12709938 : set->blocks->next = block;
820 : : }
821 : : else
822 : : {
823 : 0 : block->prev = NULL;
824 : 0 : block->next = NULL;
825 : 0 : set->blocks = block;
826 : : }
827 : :
828 : : /* Ensure any padding bytes are marked NOACCESS. */
829 : : VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
830 : : chunk_size - size);
831 : :
832 : : /* Disallow access to the chunk header. */
833 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
834 : :
835 : 12709938 : return MemoryChunkGetPointer(chunk);
836 : : }
837 : :
838 : : /*
839 : : * Small helper for allocating a new chunk from a chunk, to avoid duplicating
840 : : * the code between AllocSetAlloc() and AllocSetAllocFromNewBlock().
841 : : */
842 : : static inline void *
843 : 620553570 : AllocSetAllocChunkFromBlock(MemoryContext context, AllocBlock block,
844 : : Size size, Size chunk_size, int fidx)
845 : : {
846 : : MemoryChunk *chunk;
847 : :
848 : 620553570 : chunk = (MemoryChunk *) (block->freeptr);
849 : :
850 : : /* Prepare to initialize the chunk header. */
851 : : VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNKHDRSZ);
852 : :
853 : 620553570 : block->freeptr += (chunk_size + ALLOC_CHUNKHDRSZ);
854 : : Assert(block->freeptr <= block->endptr);
855 : :
856 : : /* store the free list index in the value field */
857 : 620553570 : MemoryChunkSetHdrMask(chunk, block, fidx, MCTX_ASET_ID);
858 : :
859 : : #ifdef MEMORY_CONTEXT_CHECKING
860 : : chunk->requested_size = size;
861 : : /* set mark to catch clobber of "unused" space */
862 : : if (size < chunk_size)
863 : : set_sentinel(MemoryChunkGetPointer(chunk), size);
864 : : #endif
865 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
866 : : /* fill the allocated space with junk */
867 : : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
868 : : #endif
869 : :
870 : : /* Ensure any padding bytes are marked NOACCESS. */
871 : : VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
872 : : chunk_size - size);
873 : :
874 : : /* Disallow access to the chunk header. */
875 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
876 : :
877 : 620553570 : return MemoryChunkGetPointer(chunk);
878 : : }
879 : :
880 : : /*
881 : : * Helper for AllocSetAlloc() that allocates a new block and returns a chunk
882 : : * allocated from it.
883 : : *
884 : : * AllocSetAlloc()'s comment explains why this is separate.
885 : : */
886 : : pg_noinline
887 : : static void *
888 : 4849285 : AllocSetAllocFromNewBlock(MemoryContext context, Size size, int flags,
889 : : int fidx)
890 : : {
891 : 4849285 : AllocSet set = (AllocSet) context;
892 : : AllocBlock block;
893 : : Size availspace;
894 : : Size blksize;
895 : : Size required_size;
896 : : Size chunk_size;
897 : :
898 : : /* due to the keeper block set->blocks should always be valid */
899 : : Assert(set->blocks != NULL);
900 : 4849285 : block = set->blocks;
901 : 4849285 : availspace = block->endptr - block->freeptr;
902 : :
903 : : /*
904 : : * The existing active (top) block does not have enough room for the
905 : : * requested allocation, but it might still have a useful amount of space
906 : : * in it. Once we push it down in the block list, we'll never try to
907 : : * allocate more space from it. So, before we do that, carve up its free
908 : : * space into chunks that we can put on the set's freelists.
909 : : *
910 : : * Because we can only get here when there's less than ALLOC_CHUNK_LIMIT
911 : : * left in the block, this loop cannot iterate more than
912 : : * ALLOCSET_NUM_FREELISTS-1 times.
913 : : */
914 [ + + ]: 15881638 : while (availspace >= ((1 << ALLOC_MINBITS) + ALLOC_CHUNKHDRSZ))
915 : : {
916 : : AllocFreeListLink *link;
917 : : MemoryChunk *chunk;
918 : 11032353 : Size availchunk = availspace - ALLOC_CHUNKHDRSZ;
919 : 11032353 : int a_fidx = AllocSetFreeIndex(availchunk);
920 : :
921 : : /*
922 : : * In most cases, we'll get back the index of the next larger freelist
923 : : * than the one we need to put this chunk on. The exception is when
924 : : * availchunk is exactly a power of 2.
925 : : */
926 [ + + ]: 11032353 : if (availchunk != GetChunkSizeFromFreeListIdx(a_fidx))
927 : : {
928 : 8545235 : a_fidx--;
929 : : Assert(a_fidx >= 0);
930 : 8545235 : availchunk = GetChunkSizeFromFreeListIdx(a_fidx);
931 : : }
932 : :
933 : 11032353 : chunk = (MemoryChunk *) (block->freeptr);
934 : :
935 : : /* Prepare to initialize the chunk header. */
936 : : VALGRIND_MAKE_MEM_UNDEFINED(chunk, ALLOC_CHUNKHDRSZ);
937 : 11032353 : block->freeptr += (availchunk + ALLOC_CHUNKHDRSZ);
938 : 11032353 : availspace -= (availchunk + ALLOC_CHUNKHDRSZ);
939 : :
940 : : /* store the freelist index in the value field */
941 : 11032353 : MemoryChunkSetHdrMask(chunk, block, a_fidx, MCTX_ASET_ID);
942 : : #ifdef MEMORY_CONTEXT_CHECKING
943 : : chunk->requested_size = InvalidAllocSize; /* mark it free */
944 : : #endif
945 : : /* push this chunk onto the free list */
946 : 11032353 : link = GetFreeListLink(chunk);
947 : :
948 : : VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink));
949 : 11032353 : link->next = set->freelist[a_fidx];
950 : : VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink));
951 : :
952 : 11032353 : set->freelist[a_fidx] = chunk;
953 : : }
954 : :
955 : : /*
956 : : * The first such block has size initBlockSize, and we double the space in
957 : : * each succeeding block, but not more than maxBlockSize.
958 : : */
959 : 4849285 : blksize = set->nextBlockSize;
960 : 4849285 : set->nextBlockSize <<= 1;
961 [ + + ]: 4849285 : if (set->nextBlockSize > set->maxBlockSize)
962 : 520673 : set->nextBlockSize = set->maxBlockSize;
963 : :
964 : : /* Choose the actual chunk size to allocate */
965 : 4849285 : chunk_size = GetChunkSizeFromFreeListIdx(fidx);
966 : : Assert(chunk_size >= size);
967 : :
968 : : /*
969 : : * If initBlockSize is less than ALLOC_CHUNK_LIMIT, we could need more
970 : : * space... but try to keep it a power of 2.
971 : : */
972 : 4849285 : required_size = chunk_size + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
973 [ + + ]: 5892583 : while (blksize < required_size)
974 : 1043298 : blksize <<= 1;
975 : :
976 : : /* Try to allocate it */
977 : 4849285 : block = (AllocBlock) malloc(blksize);
978 : :
979 : : /*
980 : : * We could be asking for pretty big blocks here, so cope if malloc fails.
981 : : * But give up if there's less than 1 MB or so available...
982 : : */
983 [ - + - - ]: 4849285 : while (block == NULL && blksize > 1024 * 1024)
984 : : {
985 : 0 : blksize >>= 1;
986 [ # # ]: 0 : if (blksize < required_size)
987 : 0 : break;
988 : 0 : block = (AllocBlock) malloc(blksize);
989 : : }
990 : :
991 [ - + ]: 4849285 : if (block == NULL)
992 : 0 : return MemoryContextAllocationFailure(context, size, flags);
993 : :
994 : : /* Make a vchunk covering the new block's header */
995 : : VALGRIND_MEMPOOL_ALLOC(set, block, ALLOC_BLOCKHDRSZ);
996 : :
997 : 4849285 : context->mem_allocated += blksize;
998 : :
999 : 4849285 : block->aset = set;
1000 : 4849285 : block->freeptr = ((char *) block) + ALLOC_BLOCKHDRSZ;
1001 : 4849285 : block->endptr = ((char *) block) + blksize;
1002 : :
1003 : : /* Mark unallocated space NOACCESS. */
1004 : : VALGRIND_MAKE_MEM_NOACCESS(block->freeptr,
1005 : : blksize - ALLOC_BLOCKHDRSZ);
1006 : :
1007 : 4849285 : block->prev = NULL;
1008 : 4849285 : block->next = set->blocks;
1009 [ + - ]: 4849285 : if (block->next)
1010 : 4849285 : block->next->prev = block;
1011 : 4849285 : set->blocks = block;
1012 : :
1013 : 4849285 : return AllocSetAllocChunkFromBlock(context, block, size, chunk_size, fidx);
1014 : : }
1015 : :
1016 : : /*
1017 : : * AllocSetAlloc
1018 : : * Returns a pointer to allocated memory of given size or raises an ERROR
1019 : : * on allocation failure, or returns NULL when flags contains
1020 : : * MCXT_ALLOC_NO_OOM.
1021 : : *
1022 : : * No request may exceed:
1023 : : * MAXALIGN_DOWN(SIZE_MAX) - ALLOC_BLOCKHDRSZ - ALLOC_CHUNKHDRSZ
1024 : : * All callers use a much-lower limit.
1025 : : *
1026 : : * Note: when using valgrind, it doesn't matter how the returned allocation
1027 : : * is marked, as mcxt.c will set it to UNDEFINED. In some paths we will
1028 : : * return space that is marked NOACCESS - AllocSetRealloc has to beware!
1029 : : *
1030 : : * This function should only contain the most common code paths. Everything
1031 : : * else should be in pg_noinline helper functions, thus avoiding the overhead
1032 : : * of creating a stack frame for the common cases. Allocating memory is often
1033 : : * a bottleneck in many workloads, so avoiding stack frame setup is
1034 : : * worthwhile. Helper functions should always directly return the newly
1035 : : * allocated memory so that we can just return that address directly as a tail
1036 : : * call.
1037 : : */
1038 : : void *
1039 : 882138797 : AllocSetAlloc(MemoryContext context, Size size, int flags)
1040 : : {
1041 : 882138797 : AllocSet set = (AllocSet) context;
1042 : : AllocBlock block;
1043 : : MemoryChunk *chunk;
1044 : : int fidx;
1045 : : Size chunk_size;
1046 : : Size availspace;
1047 : :
1048 : : Assert(AllocSetIsValid(set));
1049 : :
1050 : : /* due to the keeper block set->blocks should never be NULL */
1051 : : Assert(set->blocks != NULL);
1052 : :
1053 : : /*
1054 : : * If requested size exceeds maximum for chunks we hand the request off to
1055 : : * AllocSetAllocLarge().
1056 : : */
1057 [ + + ]: 882138797 : if (size > set->allocChunkLimit)
1058 : 12709938 : return AllocSetAllocLarge(context, size, flags);
1059 : :
1060 : : /*
1061 : : * Request is small enough to be treated as a chunk. Look in the
1062 : : * corresponding free list to see if there is a free chunk we could reuse.
1063 : : * If one is found, remove it from the free list, make it again a member
1064 : : * of the alloc set and return its data address.
1065 : : *
1066 : : * Note that we don't attempt to ensure there's space for the sentinel
1067 : : * byte here. We expect a large proportion of allocations to be for sizes
1068 : : * which are already a power of 2. If we were to always make space for a
1069 : : * sentinel byte in MEMORY_CONTEXT_CHECKING builds, then we'd end up
1070 : : * doubling the memory requirements for such allocations.
1071 : : */
1072 : 869428859 : fidx = AllocSetFreeIndex(size);
1073 : 869428859 : chunk = set->freelist[fidx];
1074 [ + + ]: 869428859 : if (chunk != NULL)
1075 : : {
1076 : 248875289 : AllocFreeListLink *link = GetFreeListLink(chunk);
1077 : :
1078 : : /* Allow access to the chunk header. */
1079 : : VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ);
1080 : :
1081 : : Assert(fidx == MemoryChunkGetValue(chunk));
1082 : :
1083 : : /* pop this chunk off the freelist */
1084 : : VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink));
1085 : 248875289 : set->freelist[fidx] = link->next;
1086 : : VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink));
1087 : :
1088 : : #ifdef MEMORY_CONTEXT_CHECKING
1089 : : chunk->requested_size = size;
1090 : : /* set mark to catch clobber of "unused" space */
1091 : : if (size < GetChunkSizeFromFreeListIdx(fidx))
1092 : : set_sentinel(MemoryChunkGetPointer(chunk), size);
1093 : : #endif
1094 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
1095 : : /* fill the allocated space with junk */
1096 : : randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
1097 : : #endif
1098 : :
1099 : : /* Ensure any padding bytes are marked NOACCESS. */
1100 : : VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
1101 : : GetChunkSizeFromFreeListIdx(fidx) - size);
1102 : :
1103 : : /* Disallow access to the chunk header. */
1104 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1105 : :
1106 : 248875289 : return MemoryChunkGetPointer(chunk);
1107 : : }
1108 : :
1109 : : /*
1110 : : * Choose the actual chunk size to allocate.
1111 : : */
1112 : 620553570 : chunk_size = GetChunkSizeFromFreeListIdx(fidx);
1113 : : Assert(chunk_size >= size);
1114 : :
1115 : 620553570 : block = set->blocks;
1116 : 620553570 : availspace = block->endptr - block->freeptr;
1117 : :
1118 : : /*
1119 : : * If there is enough room in the active allocation block, we will put the
1120 : : * chunk into that block. Else must start a new one.
1121 : : */
1122 [ + + ]: 620553570 : if (unlikely(availspace < (chunk_size + ALLOC_CHUNKHDRSZ)))
1123 : 4849285 : return AllocSetAllocFromNewBlock(context, size, flags, fidx);
1124 : :
1125 : : /* There's enough space on the current block, so allocate from that */
1126 : 615704285 : return AllocSetAllocChunkFromBlock(context, block, size, chunk_size, fidx);
1127 : : }
1128 : :
1129 : : /*
1130 : : * AllocSetFree
1131 : : * Frees allocated memory; memory is removed from the set.
1132 : : */
1133 : : void
1134 : 294230937 : AllocSetFree(void *pointer)
1135 : : {
1136 : : AllocSet set;
1137 : 294230937 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
1138 : :
1139 : : /* Allow access to the chunk header. */
1140 : : VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ);
1141 : :
1142 [ + + ]: 294230937 : if (MemoryChunkIsExternal(chunk))
1143 : : {
1144 : : /* Release single-chunk block. */
1145 : 11835574 : AllocBlock block = ExternalChunkGetBlock(chunk);
1146 : :
1147 : : /*
1148 : : * Try to verify that we have a sane block pointer: the block header
1149 : : * should reference an aset and the freeptr should match the endptr.
1150 : : */
1151 [ + - + - : 11835574 : if (!AllocBlockIsValid(block) || block->freeptr != block->endptr)
+ - - + ]
1152 [ # # ]: 0 : elog(ERROR, "could not find block containing chunk %p", chunk);
1153 : :
1154 : 11835574 : set = block->aset;
1155 : :
1156 : : #ifdef MEMORY_CONTEXT_CHECKING
1157 : : {
1158 : : /* Test for someone scribbling on unused space in chunk */
1159 : : Assert(chunk->requested_size < (block->endptr - (char *) pointer));
1160 : : if (!sentinel_ok(pointer, chunk->requested_size))
1161 : : elog(WARNING, "detected write past chunk end in %s %p",
1162 : : set->header.name, chunk);
1163 : : }
1164 : : #endif
1165 : :
1166 : : /* OK, remove block from aset's list and free it */
1167 [ + - ]: 11835574 : if (block->prev)
1168 : 11835574 : block->prev->next = block->next;
1169 : : else
1170 : 0 : set->blocks = block->next;
1171 [ + + ]: 11835574 : if (block->next)
1172 : 8928572 : block->next->prev = block->prev;
1173 : :
1174 : 11835574 : set->header.mem_allocated -= block->endptr - ((char *) block);
1175 : :
1176 : : #ifdef CLOBBER_FREED_MEMORY
1177 : : wipe_mem(block, block->freeptr - ((char *) block));
1178 : : #endif
1179 : :
1180 : : /* As in AllocSetReset, free block-header vchunks explicitly */
1181 : : VALGRIND_MEMPOOL_FREE(set, block);
1182 : :
1183 : 11835574 : free(block);
1184 : : }
1185 : : else
1186 : : {
1187 : 282395363 : AllocBlock block = MemoryChunkGetBlock(chunk);
1188 : : int fidx;
1189 : : AllocFreeListLink *link;
1190 : :
1191 : : /*
1192 : : * In this path, for speed reasons we just Assert that the referenced
1193 : : * block is good. We can also Assert that the value field is sane.
1194 : : * Future field experience may show that these Asserts had better
1195 : : * become regular runtime test-and-elog checks.
1196 : : */
1197 : : Assert(AllocBlockIsValid(block));
1198 : 282395363 : set = block->aset;
1199 : :
1200 : 282395363 : fidx = MemoryChunkGetValue(chunk);
1201 : : Assert(FreeListIdxIsValid(fidx));
1202 : 282395363 : link = GetFreeListLink(chunk);
1203 : :
1204 : : /*
1205 : : * It might seem odd that we use elevel ERROR for double-pfree but
1206 : : * only WARNING for write-past-chunk-end. But the two conditions are
1207 : : * not very comparable. In the double-pfree case we can prevent
1208 : : * corruption before it happens; while if we let it go through, the
1209 : : * result would be a corrupted freelist that allows this chunk to get
1210 : : * re-allocated twice. Thus the original bug could cascade into
1211 : : * hard-to-understand misbehavior that might manifest far away from
1212 : : * the actual source of the problem. On the other hand, a write past
1213 : : * chunk end can be relatively benign if just a few bytes too many
1214 : : * were written: often, only padding or unused space gets affected.
1215 : : * Moreover, whatever damage was done is already done, and we're just
1216 : : * reporting after the fact with no ability to clean it up. So just
1217 : : * warn, like AllocSetCheck would do if the chunk didn't get freed.
1218 : : */
1219 : : #ifdef MEMORY_CONTEXT_CHECKING
1220 : : /* Test for previously-freed chunk */
1221 : : if (unlikely(chunk->requested_size == InvalidAllocSize))
1222 : : elog(ERROR, "detected double pfree in %s %p",
1223 : : set->header.name, chunk);
1224 : : /* Test for someone scribbling on unused space in chunk */
1225 : : if (chunk->requested_size < GetChunkSizeFromFreeListIdx(fidx))
1226 : : if (!sentinel_ok(pointer, chunk->requested_size))
1227 : : elog(WARNING, "detected write past chunk end in %s %p",
1228 : : set->header.name, chunk);
1229 : : #endif
1230 : :
1231 : : #ifdef CLOBBER_FREED_MEMORY
1232 : : wipe_mem(pointer, GetChunkSizeFromFreeListIdx(fidx));
1233 : : #endif
1234 : : /* push this chunk onto the top of the free list */
1235 : : VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink));
1236 : 282395363 : link->next = set->freelist[fidx];
1237 : : VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink));
1238 : 282395363 : set->freelist[fidx] = chunk;
1239 : :
1240 : : #ifdef MEMORY_CONTEXT_CHECKING
1241 : :
1242 : : /*
1243 : : * Reset requested_size to InvalidAllocSize in chunks that are on free
1244 : : * list.
1245 : : */
1246 : : chunk->requested_size = InvalidAllocSize;
1247 : : #endif
1248 : : }
1249 : 294230937 : }
1250 : :
1251 : : /*
1252 : : * AllocSetRealloc
1253 : : * Returns new pointer to allocated memory of given size or NULL if
1254 : : * request could not be completed; this memory is added to the set.
1255 : : * Memory associated with given pointer is copied into the new memory,
1256 : : * and the old memory is freed.
1257 : : *
1258 : : * Without MEMORY_CONTEXT_CHECKING, we don't know the old request size. This
1259 : : * makes our Valgrind client requests less-precise, hazarding false negatives.
1260 : : * (In principle, we could use VALGRIND_GET_VBITS() to rediscover the old
1261 : : * request size.)
1262 : : */
1263 : : void *
1264 : 5609115 : AllocSetRealloc(void *pointer, Size size, int flags)
1265 : : {
1266 : : AllocBlock block;
1267 : : AllocSet set;
1268 : 5609115 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
1269 : : Size oldchksize;
1270 : : int fidx;
1271 : :
1272 : : /* Allow access to the chunk header. */
1273 : : VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ);
1274 : :
1275 [ + + ]: 5609115 : if (MemoryChunkIsExternal(chunk))
1276 : : {
1277 : : /*
1278 : : * The chunk must have been allocated as a single-chunk block. Use
1279 : : * realloc() to make the containing block bigger, or smaller, with
1280 : : * minimum space wastage.
1281 : : */
1282 : : AllocBlock newblock;
1283 : : Size chksize;
1284 : : Size blksize;
1285 : : Size oldblksize;
1286 : :
1287 : 194247 : block = ExternalChunkGetBlock(chunk);
1288 : :
1289 : : /*
1290 : : * Try to verify that we have a sane block pointer: the block header
1291 : : * should reference an aset and the freeptr should match the endptr.
1292 : : */
1293 [ + - + - : 194247 : if (!AllocBlockIsValid(block) || block->freeptr != block->endptr)
+ - - + ]
1294 [ # # ]: 0 : elog(ERROR, "could not find block containing chunk %p", chunk);
1295 : :
1296 : 194247 : set = block->aset;
1297 : :
1298 : : /* only check size in paths where the limits could be hit */
1299 : 194247 : MemoryContextCheckSize((MemoryContext) set, size, flags);
1300 : :
1301 : 194247 : oldchksize = block->endptr - (char *) pointer;
1302 : :
1303 : : #ifdef MEMORY_CONTEXT_CHECKING
1304 : : /* Test for someone scribbling on unused space in chunk */
1305 : : Assert(chunk->requested_size < oldchksize);
1306 : : if (!sentinel_ok(pointer, chunk->requested_size))
1307 : : elog(WARNING, "detected write past chunk end in %s %p",
1308 : : set->header.name, chunk);
1309 : : #endif
1310 : :
1311 : : #ifdef MEMORY_CONTEXT_CHECKING
1312 : : /* ensure there's always space for the sentinel byte */
1313 : : chksize = MAXALIGN(size + 1);
1314 : : #else
1315 : 194247 : chksize = MAXALIGN(size);
1316 : : #endif
1317 : :
1318 : : /* Do the realloc */
1319 : 194247 : blksize = chksize + ALLOC_BLOCKHDRSZ + ALLOC_CHUNKHDRSZ;
1320 : 194247 : oldblksize = block->endptr - ((char *) block);
1321 : :
1322 : 194247 : newblock = (AllocBlock) realloc(block, blksize);
1323 [ - + ]: 194247 : if (newblock == NULL)
1324 : : {
1325 : : /* Disallow access to the chunk header. */
1326 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1327 : 0 : return MemoryContextAllocationFailure(&set->header, size, flags);
1328 : : }
1329 : :
1330 : : /*
1331 : : * Move the block-header vchunk explicitly. (mcxt.c will take care of
1332 : : * moving the vchunk for the user data.)
1333 : : */
1334 : : VALGRIND_MEMPOOL_CHANGE(set, block, newblock, ALLOC_BLOCKHDRSZ);
1335 : 194247 : block = newblock;
1336 : :
1337 : : /* updated separately, not to underflow when (oldblksize > blksize) */
1338 : 194247 : set->header.mem_allocated -= oldblksize;
1339 : 194247 : set->header.mem_allocated += blksize;
1340 : :
1341 : 194247 : block->freeptr = block->endptr = ((char *) block) + blksize;
1342 : :
1343 : : /* Update pointers since block has likely been moved */
1344 : 194247 : chunk = (MemoryChunk *) (((char *) block) + ALLOC_BLOCKHDRSZ);
1345 : 194247 : pointer = MemoryChunkGetPointer(chunk);
1346 [ + - ]: 194247 : if (block->prev)
1347 : 194247 : block->prev->next = block;
1348 : : else
1349 : 0 : set->blocks = block;
1350 [ + + ]: 194247 : if (block->next)
1351 : 177775 : block->next->prev = block;
1352 : :
1353 : : #ifdef MEMORY_CONTEXT_CHECKING
1354 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
1355 : :
1356 : : /*
1357 : : * We can only randomize the extra space if we know the prior request.
1358 : : * When using Valgrind, randomize_mem() also marks memory UNDEFINED.
1359 : : */
1360 : : if (size > chunk->requested_size)
1361 : : randomize_mem((char *) pointer + chunk->requested_size,
1362 : : size - chunk->requested_size);
1363 : : #else
1364 : :
1365 : : /*
1366 : : * If this is an increase, realloc() will have marked any
1367 : : * newly-allocated part (from oldchksize to chksize) UNDEFINED, but we
1368 : : * also need to adjust trailing bytes from the old allocation (from
1369 : : * chunk->requested_size to oldchksize) as they are marked NOACCESS.
1370 : : * Make sure not to mark too many bytes in case chunk->requested_size
1371 : : * < size < oldchksize.
1372 : : */
1373 : : #ifdef USE_VALGRIND
1374 : : if (Min(size, oldchksize) > chunk->requested_size)
1375 : : VALGRIND_MAKE_MEM_UNDEFINED((char *) pointer + chunk->requested_size,
1376 : : Min(size, oldchksize) - chunk->requested_size);
1377 : : #endif
1378 : : #endif
1379 : :
1380 : : chunk->requested_size = size;
1381 : : /* set mark to catch clobber of "unused" space */
1382 : : Assert(size < chksize);
1383 : : set_sentinel(pointer, size);
1384 : : #else /* !MEMORY_CONTEXT_CHECKING */
1385 : :
1386 : : /*
1387 : : * We may need to adjust marking of bytes from the old allocation as
1388 : : * some of them may be marked NOACCESS. We don't know how much of the
1389 : : * old chunk size was the requested size; it could have been as small
1390 : : * as one byte. We have to be conservative and just mark the entire
1391 : : * old portion DEFINED. Make sure not to mark memory beyond the new
1392 : : * allocation in case it's smaller than the old one.
1393 : : */
1394 : : VALGRIND_MAKE_MEM_DEFINED(pointer, Min(size, oldchksize));
1395 : : #endif
1396 : :
1397 : : /* Ensure any padding bytes are marked NOACCESS. */
1398 : : VALGRIND_MAKE_MEM_NOACCESS((char *) pointer + size, chksize - size);
1399 : :
1400 : : /* Disallow access to the chunk header. */
1401 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1402 : :
1403 : 194247 : return pointer;
1404 : : }
1405 : :
1406 : 5414868 : block = MemoryChunkGetBlock(chunk);
1407 : :
1408 : : /*
1409 : : * In this path, for speed reasons we just Assert that the referenced
1410 : : * block is good. We can also Assert that the value field is sane. Future
1411 : : * field experience may show that these Asserts had better become regular
1412 : : * runtime test-and-elog checks.
1413 : : */
1414 : : Assert(AllocBlockIsValid(block));
1415 : 5414868 : set = block->aset;
1416 : :
1417 : 5414868 : fidx = MemoryChunkGetValue(chunk);
1418 : : Assert(FreeListIdxIsValid(fidx));
1419 : 5414868 : oldchksize = GetChunkSizeFromFreeListIdx(fidx);
1420 : :
1421 : : #ifdef MEMORY_CONTEXT_CHECKING
1422 : : /* See comments in AllocSetFree about uses of ERROR and WARNING here */
1423 : : /* Test for previously-freed chunk */
1424 : : if (unlikely(chunk->requested_size == InvalidAllocSize))
1425 : : elog(ERROR, "detected realloc of freed chunk in %s %p",
1426 : : set->header.name, chunk);
1427 : : /* Test for someone scribbling on unused space in chunk */
1428 : : if (chunk->requested_size < oldchksize)
1429 : : if (!sentinel_ok(pointer, chunk->requested_size))
1430 : : elog(WARNING, "detected write past chunk end in %s %p",
1431 : : set->header.name, chunk);
1432 : : #endif
1433 : :
1434 : : /*
1435 : : * Chunk sizes are aligned to power of 2 in AllocSetAlloc(). Maybe the
1436 : : * allocated area already is >= the new size. (In particular, we will
1437 : : * fall out here if the requested size is a decrease.)
1438 : : */
1439 [ + + ]: 5414868 : if (oldchksize >= size)
1440 : : {
1441 : : #ifdef MEMORY_CONTEXT_CHECKING
1442 : : Size oldrequest = chunk->requested_size;
1443 : :
1444 : : #ifdef RANDOMIZE_ALLOCATED_MEMORY
1445 : : /* We can only fill the extra space if we know the prior request */
1446 : : if (size > oldrequest)
1447 : : randomize_mem((char *) pointer + oldrequest,
1448 : : size - oldrequest);
1449 : : #endif
1450 : :
1451 : : chunk->requested_size = size;
1452 : :
1453 : : /*
1454 : : * If this is an increase, mark any newly-available part UNDEFINED.
1455 : : * Otherwise, mark the obsolete part NOACCESS.
1456 : : */
1457 : : if (size > oldrequest)
1458 : : VALGRIND_MAKE_MEM_UNDEFINED((char *) pointer + oldrequest,
1459 : : size - oldrequest);
1460 : : else
1461 : : VALGRIND_MAKE_MEM_NOACCESS((char *) pointer + size,
1462 : : oldchksize - size);
1463 : :
1464 : : /* set mark to catch clobber of "unused" space */
1465 : : if (size < oldchksize)
1466 : : set_sentinel(pointer, size);
1467 : : #else /* !MEMORY_CONTEXT_CHECKING */
1468 : :
1469 : : /*
1470 : : * We don't have the information to determine whether we're growing
1471 : : * the old request or shrinking it, so we conservatively mark the
1472 : : * entire new allocation DEFINED.
1473 : : */
1474 : : VALGRIND_MAKE_MEM_NOACCESS(pointer, oldchksize);
1475 : : VALGRIND_MAKE_MEM_DEFINED(pointer, size);
1476 : : #endif
1477 : :
1478 : : /* Disallow access to the chunk header. */
1479 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1480 : :
1481 : 1314262 : return pointer;
1482 : : }
1483 : : else
1484 : : {
1485 : : /*
1486 : : * Enlarge-a-small-chunk case. We just do this by brute force, ie,
1487 : : * allocate a new chunk and copy the data. Since we know the existing
1488 : : * data isn't huge, this won't involve any great memcpy expense, so
1489 : : * it's not worth being smarter. (At one time we tried to avoid
1490 : : * memcpy when it was possible to enlarge the chunk in-place, but that
1491 : : * turns out to misbehave unpleasantly for repeated cycles of
1492 : : * palloc/repalloc/pfree: the eventually freed chunks go into the
1493 : : * wrong freelist for the next initial palloc request, and so we leak
1494 : : * memory indefinitely. See pgsql-hackers archives for 2007-08-11.)
1495 : : */
1496 : : AllocPointer newPointer;
1497 : : Size oldsize;
1498 : :
1499 : : /* allocate new chunk (this also checks size is valid) */
1500 : 4100606 : newPointer = AllocSetAlloc((MemoryContext) set, size, flags);
1501 : :
1502 : : /* leave immediately if request was not completed */
1503 [ - + ]: 4100606 : if (newPointer == NULL)
1504 : : {
1505 : : /* Disallow access to the chunk header. */
1506 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1507 : 0 : return MemoryContextAllocationFailure((MemoryContext) set, size, flags);
1508 : : }
1509 : :
1510 : : /*
1511 : : * AllocSetAlloc() may have returned a region that is still NOACCESS.
1512 : : * Change it to UNDEFINED for the moment; memcpy() will then transfer
1513 : : * definedness from the old allocation to the new. If we know the old
1514 : : * allocation, copy just that much. Otherwise, make the entire old
1515 : : * chunk defined to avoid errors as we copy the currently-NOACCESS
1516 : : * trailing bytes.
1517 : : */
1518 : : VALGRIND_MAKE_MEM_UNDEFINED(newPointer, size);
1519 : : #ifdef MEMORY_CONTEXT_CHECKING
1520 : : oldsize = chunk->requested_size;
1521 : : #else
1522 : 4100606 : oldsize = oldchksize;
1523 : : VALGRIND_MAKE_MEM_DEFINED(pointer, oldsize);
1524 : : #endif
1525 : :
1526 : : /* transfer existing data (certain to fit) */
1527 : 4100606 : memcpy(newPointer, pointer, oldsize);
1528 : :
1529 : : /* free old chunk */
1530 : 4100606 : AllocSetFree(pointer);
1531 : :
1532 : 4100606 : return newPointer;
1533 : : }
1534 : : }
1535 : :
1536 : : /*
1537 : : * AllocSetGetChunkContext
1538 : : * Return the MemoryContext that 'pointer' belongs to.
1539 : : */
1540 : : MemoryContext
1541 : 2824824 : AllocSetGetChunkContext(void *pointer)
1542 : : {
1543 : 2824824 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
1544 : : AllocBlock block;
1545 : : AllocSet set;
1546 : :
1547 : : /* Allow access to the chunk header. */
1548 : : VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ);
1549 : :
1550 [ + + ]: 2824824 : if (MemoryChunkIsExternal(chunk))
1551 : 548 : block = ExternalChunkGetBlock(chunk);
1552 : : else
1553 : 2824276 : block = (AllocBlock) MemoryChunkGetBlock(chunk);
1554 : :
1555 : : /* Disallow access to the chunk header. */
1556 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1557 : :
1558 : : Assert(AllocBlockIsValid(block));
1559 : 2824824 : set = block->aset;
1560 : :
1561 : 2824824 : return &set->header;
1562 : : }
1563 : :
1564 : : /*
1565 : : * AllocSetGetChunkSpace
1566 : : * Given a currently-allocated chunk, determine the total space
1567 : : * it occupies (including all memory-allocation overhead).
1568 : : */
1569 : : Size
1570 : 5143358 : AllocSetGetChunkSpace(void *pointer)
1571 : : {
1572 : 5143358 : MemoryChunk *chunk = PointerGetMemoryChunk(pointer);
1573 : : int fidx;
1574 : :
1575 : : /* Allow access to the chunk header. */
1576 : : VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ);
1577 : :
1578 [ + + ]: 5143358 : if (MemoryChunkIsExternal(chunk))
1579 : : {
1580 : 558775 : AllocBlock block = ExternalChunkGetBlock(chunk);
1581 : :
1582 : : /* Disallow access to the chunk header. */
1583 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1584 : :
1585 : : Assert(AllocBlockIsValid(block));
1586 : :
1587 : 558775 : return block->endptr - (char *) chunk;
1588 : : }
1589 : :
1590 : 4584583 : fidx = MemoryChunkGetValue(chunk);
1591 : : Assert(FreeListIdxIsValid(fidx));
1592 : :
1593 : : /* Disallow access to the chunk header. */
1594 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1595 : :
1596 : 4584583 : return GetChunkSizeFromFreeListIdx(fidx) + ALLOC_CHUNKHDRSZ;
1597 : : }
1598 : :
1599 : : /*
1600 : : * AllocSetIsEmpty
1601 : : * Is an allocset empty of any allocated space?
1602 : : */
1603 : : bool
1604 : 17396 : AllocSetIsEmpty(MemoryContext context)
1605 : : {
1606 : : Assert(AllocSetIsValid(context));
1607 : :
1608 : : /*
1609 : : * For now, we say "empty" only if the context is new or just reset. We
1610 : : * could examine the freelists to determine if all space has been freed,
1611 : : * but it's not really worth the trouble for present uses of this
1612 : : * functionality.
1613 : : */
1614 [ + + ]: 17396 : if (context->isReset)
1615 : 17381 : return true;
1616 : 15 : return false;
1617 : : }
1618 : :
1619 : : /*
1620 : : * AllocSetStats
1621 : : * Compute stats about memory consumption of an allocset.
1622 : : *
1623 : : * printfunc: if not NULL, pass a human-readable stats string to this.
1624 : : * passthru: pass this pointer through to printfunc.
1625 : : * totals: if not NULL, add stats about this context into *totals.
1626 : : * print_to_stderr: print stats to stderr if true, elog otherwise.
1627 : : */
1628 : : void
1629 : 3234 : AllocSetStats(MemoryContext context,
1630 : : MemoryStatsPrintFunc printfunc, void *passthru,
1631 : : MemoryContextCounters *totals, bool print_to_stderr)
1632 : : {
1633 : 3234 : AllocSet set = (AllocSet) context;
1634 : 3234 : Size nblocks = 0;
1635 : 3234 : Size freechunks = 0;
1636 : : Size totalspace;
1637 : 3234 : Size freespace = 0;
1638 : : AllocBlock block;
1639 : : int fidx;
1640 : :
1641 : : Assert(AllocSetIsValid(set));
1642 : :
1643 : : /* Include context header in totalspace */
1644 : 3234 : totalspace = MAXALIGN(sizeof(AllocSetContext));
1645 : :
1646 [ + + ]: 9030 : for (block = set->blocks; block != NULL; block = block->next)
1647 : : {
1648 : 5796 : nblocks++;
1649 : 5796 : totalspace += block->endptr - ((char *) block);
1650 : 5796 : freespace += block->endptr - block->freeptr;
1651 : : }
1652 [ + + ]: 38808 : for (fidx = 0; fidx < ALLOCSET_NUM_FREELISTS; fidx++)
1653 : : {
1654 : 35574 : Size chksz = GetChunkSizeFromFreeListIdx(fidx);
1655 : 35574 : MemoryChunk *chunk = set->freelist[fidx];
1656 : :
1657 [ + + ]: 49078 : while (chunk != NULL)
1658 : : {
1659 : 13504 : AllocFreeListLink *link = GetFreeListLink(chunk);
1660 : :
1661 : : /* Allow access to the chunk header. */
1662 : : VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ);
1663 : : Assert(MemoryChunkGetValue(chunk) == fidx);
1664 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1665 : :
1666 : 13504 : freechunks++;
1667 : 13504 : freespace += chksz + ALLOC_CHUNKHDRSZ;
1668 : :
1669 : : VALGRIND_MAKE_MEM_DEFINED(link, sizeof(AllocFreeListLink));
1670 : 13504 : chunk = link->next;
1671 : : VALGRIND_MAKE_MEM_NOACCESS(link, sizeof(AllocFreeListLink));
1672 : : }
1673 : : }
1674 : :
1675 [ + + ]: 3234 : if (printfunc)
1676 : : {
1677 : : char stats_string[200];
1678 : :
1679 : 1150 : snprintf(stats_string, sizeof(stats_string),
1680 : : "%zu total in %zu blocks; %zu free (%zu chunks); %zu used",
1681 : : totalspace, nblocks, freespace, freechunks,
1682 : : totalspace - freespace);
1683 : 1150 : printfunc(context, passthru, stats_string, print_to_stderr);
1684 : : }
1685 : :
1686 [ + - ]: 3234 : if (totals)
1687 : : {
1688 : 3234 : totals->nblocks += nblocks;
1689 : 3234 : totals->freechunks += freechunks;
1690 : 3234 : totals->totalspace += totalspace;
1691 : 3234 : totals->freespace += freespace;
1692 : : }
1693 : 3234 : }
1694 : :
1695 : :
1696 : : #ifdef MEMORY_CONTEXT_CHECKING
1697 : :
1698 : : /*
1699 : : * AllocSetCheck
1700 : : * Walk through chunks and check consistency of memory.
1701 : : *
1702 : : * NOTE: report errors as WARNING, *not* ERROR or FATAL. Otherwise you'll
1703 : : * find yourself in an infinite loop when trouble occurs, because this
1704 : : * routine will be entered again when elog cleanup tries to release memory!
1705 : : */
1706 : : void
1707 : : AllocSetCheck(MemoryContext context)
1708 : : {
1709 : : AllocSet set = (AllocSet) context;
1710 : : const char *name = set->header.name;
1711 : : AllocBlock prevblock;
1712 : : AllocBlock block;
1713 : : Size total_allocated = 0;
1714 : :
1715 : : for (prevblock = NULL, block = set->blocks;
1716 : : block != NULL;
1717 : : prevblock = block, block = block->next)
1718 : : {
1719 : : char *bpoz = ((char *) block) + ALLOC_BLOCKHDRSZ;
1720 : : Size blk_used = block->freeptr - bpoz;
1721 : : Size blk_data = 0;
1722 : : Size nchunks = 0;
1723 : : bool has_external_chunk = false;
1724 : :
1725 : : if (IsKeeperBlock(set, block))
1726 : : total_allocated += block->endptr - ((char *) set);
1727 : : else
1728 : : total_allocated += block->endptr - ((char *) block);
1729 : :
1730 : : /*
1731 : : * Empty block - empty can be keeper-block only
1732 : : */
1733 : : if (!blk_used)
1734 : : {
1735 : : if (!IsKeeperBlock(set, block))
1736 : : elog(WARNING, "problem in alloc set %s: empty block %p",
1737 : : name, block);
1738 : : }
1739 : :
1740 : : /*
1741 : : * Check block header fields
1742 : : */
1743 : : if (block->aset != set ||
1744 : : block->prev != prevblock ||
1745 : : block->freeptr < bpoz ||
1746 : : block->freeptr > block->endptr)
1747 : : elog(WARNING, "problem in alloc set %s: corrupt header in block %p",
1748 : : name, block);
1749 : :
1750 : : /*
1751 : : * Chunk walker
1752 : : */
1753 : : while (bpoz < block->freeptr)
1754 : : {
1755 : : MemoryChunk *chunk = (MemoryChunk *) bpoz;
1756 : : Size chsize,
1757 : : dsize;
1758 : :
1759 : : /* Allow access to the chunk header. */
1760 : : VALGRIND_MAKE_MEM_DEFINED(chunk, ALLOC_CHUNKHDRSZ);
1761 : :
1762 : : if (MemoryChunkIsExternal(chunk))
1763 : : {
1764 : : chsize = block->endptr - (char *) MemoryChunkGetPointer(chunk); /* aligned chunk size */
1765 : : has_external_chunk = true;
1766 : :
1767 : : /* make sure this chunk consumes the entire block */
1768 : : if (chsize + ALLOC_CHUNKHDRSZ != blk_used)
1769 : : elog(WARNING, "problem in alloc set %s: bad single-chunk %p in block %p",
1770 : : name, chunk, block);
1771 : : }
1772 : : else
1773 : : {
1774 : : int fidx = MemoryChunkGetValue(chunk);
1775 : :
1776 : : if (!FreeListIdxIsValid(fidx))
1777 : : elog(WARNING, "problem in alloc set %s: bad chunk size for chunk %p in block %p",
1778 : : name, chunk, block);
1779 : :
1780 : : chsize = GetChunkSizeFromFreeListIdx(fidx); /* aligned chunk size */
1781 : :
1782 : : /*
1783 : : * Check the stored block offset correctly references this
1784 : : * block.
1785 : : */
1786 : : if (block != MemoryChunkGetBlock(chunk))
1787 : : elog(WARNING, "problem in alloc set %s: bad block offset for chunk %p in block %p",
1788 : : name, chunk, block);
1789 : : }
1790 : : dsize = chunk->requested_size; /* real data */
1791 : :
1792 : : /* an allocated chunk's requested size must be <= the chsize */
1793 : : if (dsize != InvalidAllocSize && dsize > chsize)
1794 : : elog(WARNING, "problem in alloc set %s: req size > alloc size for chunk %p in block %p",
1795 : : name, chunk, block);
1796 : :
1797 : : /* chsize must not be smaller than the first freelist's size */
1798 : : if (chsize < (1 << ALLOC_MINBITS))
1799 : : elog(WARNING, "problem in alloc set %s: bad size %zu for chunk %p in block %p",
1800 : : name, chsize, chunk, block);
1801 : :
1802 : : /*
1803 : : * Check for overwrite of padding space in an allocated chunk.
1804 : : */
1805 : : if (dsize != InvalidAllocSize && dsize < chsize &&
1806 : : !sentinel_ok(MemoryChunkGetPointer(chunk), dsize))
1807 : : elog(WARNING, "problem in alloc set %s: detected write past chunk end in block %p, chunk %p",
1808 : : name, block, chunk);
1809 : :
1810 : : /* if chunk is allocated, disallow access to the chunk header */
1811 : : if (dsize != InvalidAllocSize)
1812 : : VALGRIND_MAKE_MEM_NOACCESS(chunk, ALLOC_CHUNKHDRSZ);
1813 : :
1814 : : blk_data += chsize;
1815 : : nchunks++;
1816 : :
1817 : : bpoz += ALLOC_CHUNKHDRSZ + chsize;
1818 : : }
1819 : :
1820 : : if ((blk_data + (nchunks * ALLOC_CHUNKHDRSZ)) != blk_used)
1821 : : elog(WARNING, "problem in alloc set %s: found inconsistent memory block %p",
1822 : : name, block);
1823 : :
1824 : : if (has_external_chunk && nchunks > 1)
1825 : : elog(WARNING, "problem in alloc set %s: external chunk on non-dedicated block %p",
1826 : : name, block);
1827 : : }
1828 : :
1829 : : Assert(total_allocated == context->mem_allocated);
1830 : : }
1831 : :
1832 : : #endif /* MEMORY_CONTEXT_CHECKING */
|
