LCOV - code coverage report
Current view: top level - src/backend/utils/mmgr - bump.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 104 129 80.6 %
Date: 2024-05-19 07:11:14 Functions: 12 18 66.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * bump.c
       4             :  *    Bump allocator definitions.
       5             :  *
       6             :  * Bump is a MemoryContext implementation designed for memory usages which
       7             :  * require allocating a large number of chunks, none of which ever need to be
       8             :  * pfree'd or realloc'd.  Chunks allocated by this context have no chunk header
       9             :  * and operations which ordinarily require looking at the chunk header cannot
      10             :  * be performed.  For example, pfree, realloc, GetMemoryChunkSpace and
      11             :  * GetMemoryChunkContext are all not possible with bump allocated chunks.  The
      12             :  * only way to release memory allocated by this context type is to reset or
      13             :  * delete the context.
      14             :  *
      15             :  * Portions Copyright (c) 2024, PostgreSQL Global Development Group
      16             :  *
      17             :  * IDENTIFICATION
      18             :  *    src/backend/utils/mmgr/bump.c
      19             :  *
      20             :  *
      21             :  *  Bump is best suited to cases which require a large number of short-lived
      22             :  *  chunks where performance matters.  Because bump allocated chunks don't
      23             :  *  have a chunk header, it can fit more chunks on each block.  This means we
      24             :  *  can do more with less memory and fewer cache lines.  The reason it's best
      25             :  *  suited for short-lived usages of memory is that ideally, pointers to bump
      26             :  *  allocated chunks won't be visible to a large amount of code.  The more
      27             :  *  code that operates on memory allocated by this allocator, the more chances
      28             :  *  that some code will try to perform a pfree or one of the other operations
      29             :  *  which are made impossible due to the lack of chunk header.  In order to
      30             :  *  detect accidental usage of the various disallowed operations, we do add a
      31             :  *  MemoryChunk chunk header in MEMORY_CONTEXT_CHECKING builds and have the
      32             :  *  various disallowed functions raise an ERROR.
      33             :  *
      34             :  *  Allocations are MAXALIGNed.
      35             :  *
      36             :  *-------------------------------------------------------------------------
      37             :  */
      38             : 
      39             : #include "postgres.h"
      40             : 
      41             : #include "lib/ilist.h"
      42             : #include "port/pg_bitutils.h"
      43             : #include "utils/memdebug.h"
      44             : #include "utils/memutils.h"
      45             : #include "utils/memutils_memorychunk.h"
      46             : #include "utils/memutils_internal.h"
      47             : 
      48             : #define Bump_BLOCKHDRSZ MAXALIGN(sizeof(BumpBlock))
      49             : 
      50             : /* No chunk header unless built with MEMORY_CONTEXT_CHECKING */
      51             : #ifdef MEMORY_CONTEXT_CHECKING
      52             : #define Bump_CHUNKHDRSZ sizeof(MemoryChunk)
      53             : #else
      54             : #define Bump_CHUNKHDRSZ 0
      55             : #endif
      56             : 
      57             : #define Bump_CHUNK_FRACTION 8
      58             : 
      59             : /* The keeper block is allocated in the same allocation as the set */
      60             : #define KeeperBlock(set) ((BumpBlock *) ((char *) (set) + \
      61             :             MAXALIGN(sizeof(BumpContext))))
      62             : #define IsKeeperBlock(set, blk) (KeeperBlock(set) == (blk))
      63             : 
      64             : typedef struct BumpBlock BumpBlock; /* forward reference */
      65             : 
      66             : typedef struct BumpContext
      67             : {
      68             :     MemoryContextData header;   /* Standard memory-context fields */
      69             : 
      70             :     /* Bump context parameters */
      71             :     uint32      initBlockSize;  /* initial block size */
      72             :     uint32      maxBlockSize;   /* maximum block size */
      73             :     uint32      nextBlockSize;  /* next block size to allocate */
      74             :     uint32      allocChunkLimit;    /* effective chunk size limit */
      75             : 
      76             :     dlist_head  blocks;         /* list of blocks with the block currently
      77             :                                  * being filled at the head */
      78             : } BumpContext;
      79             : 
      80             : /*
      81             :  * BumpBlock
      82             :  *      BumpBlock is the unit of memory that is obtained by bump.c from
      83             :  *      malloc().  It contains zero or more allocations, which are the
      84             :  *      units requested by palloc().
      85             :  */
      86             : struct BumpBlock
      87             : {
      88             :     dlist_node  node;           /* doubly-linked list of blocks */
      89             : #ifdef MEMORY_CONTEXT_CHECKING
      90             :     BumpContext *context;       /* pointer back to the owning context */
      91             : #endif
      92             :     char       *freeptr;        /* start of free space in this block */
      93             :     char       *endptr;         /* end of space in this block */
      94             : };
      95             : 
      96             : /*
      97             :  * BumpIsValid
      98             :  *      True iff set is valid bump context.
      99             :  */
     100             : #define BumpIsValid(set) \
     101             :     (PointerIsValid(set) && IsA(set, BumpContext))
     102             : 
     103             : /*
     104             :  * We always store external chunks on a dedicated block.  This makes fetching
     105             :  * the block from an external chunk easy since it's always the first and only
     106             :  * chunk on the block.
     107             :  */
     108             : #define ExternalChunkGetBlock(chunk) \
     109             :     (BumpBlock *) ((char *) chunk - Bump_BLOCKHDRSZ)
     110             : 
     111             : /* Inlined helper functions */
     112             : static inline void BumpBlockInit(BumpContext *context, BumpBlock *block,
     113             :                                  Size blksize);
     114             : static inline bool BumpBlockIsEmpty(BumpBlock *block);
     115             : static inline void BumpBlockMarkEmpty(BumpBlock *block);
     116             : static inline Size BumpBlockFreeBytes(BumpBlock *block);
     117             : static inline void BumpBlockFree(BumpContext *set, BumpBlock *block);
     118             : 
     119             : 
     120             : /*
     121             : * BumpContextCreate
     122             : *       Create a new Bump context.
     123             : *
     124             : * parent: parent context, or NULL if top-level context
     125             : * name: name of context (must be statically allocated)
     126             : * minContextSize: minimum context size
     127             : * initBlockSize: initial allocation block size
     128             : * maxBlockSize: maximum allocation block size
     129             : */
     130             : MemoryContext
     131      249488 : BumpContextCreate(MemoryContext parent, const char *name, Size minContextSize,
     132             :                   Size initBlockSize, Size maxBlockSize)
     133             : {
     134             :     Size        firstBlockSize;
     135             :     Size        allocSize;
     136             :     BumpContext *set;
     137             :     BumpBlock  *block;
     138             : 
     139             :     /* ensure MemoryChunk's size is properly maxaligned */
     140             :     StaticAssertDecl(Bump_CHUNKHDRSZ == MAXALIGN(Bump_CHUNKHDRSZ),
     141             :                      "sizeof(MemoryChunk) is not maxaligned");
     142             : 
     143             :     /*
     144             :      * First, validate allocation parameters.  Asserts seem sufficient because
     145             :      * nobody varies their parameters at runtime.  We somewhat arbitrarily
     146             :      * enforce a minimum 1K block size.  We restrict the maximum block size to
     147             :      * MEMORYCHUNK_MAX_BLOCKOFFSET as MemoryChunks are limited to this in
     148             :      * regards to addressing the offset between the chunk and the block that
     149             :      * the chunk is stored on.  We would be unable to store the offset between
     150             :      * the chunk and block for any chunks that were beyond
     151             :      * MEMORYCHUNK_MAX_BLOCKOFFSET bytes into the block if the block was to be
     152             :      * larger than this.
     153             :      */
     154             :     Assert(initBlockSize == MAXALIGN(initBlockSize) &&
     155             :            initBlockSize >= 1024);
     156             :     Assert(maxBlockSize == MAXALIGN(maxBlockSize) &&
     157             :            maxBlockSize >= initBlockSize &&
     158             :            AllocHugeSizeIsValid(maxBlockSize)); /* must be safe to double */
     159             :     Assert(minContextSize == 0 ||
     160             :            (minContextSize == MAXALIGN(minContextSize) &&
     161             :             minContextSize >= 1024 &&
     162             :             minContextSize <= maxBlockSize));
     163             :     Assert(maxBlockSize <= MEMORYCHUNK_MAX_BLOCKOFFSET);
     164             : 
     165             :     /* Determine size of initial block */
     166      249488 :     allocSize = MAXALIGN(sizeof(BumpContext)) + Bump_BLOCKHDRSZ +
     167             :         Bump_CHUNKHDRSZ;
     168      249488 :     if (minContextSize != 0)
     169           0 :         allocSize = Max(allocSize, minContextSize);
     170             :     else
     171      249488 :         allocSize = Max(allocSize, initBlockSize);
     172             : 
     173             :     /*
     174             :      * Allocate the initial block.  Unlike other bump.c blocks, it starts with
     175             :      * the context header and its block header follows that.
     176             :      */
     177      249488 :     set = (BumpContext *) malloc(allocSize);
     178      249488 :     if (set == NULL)
     179             :     {
     180           0 :         MemoryContextStats(TopMemoryContext);
     181           0 :         ereport(ERROR,
     182             :                 (errcode(ERRCODE_OUT_OF_MEMORY),
     183             :                  errmsg("out of memory"),
     184             :                  errdetail("Failed while creating memory context \"%s\".",
     185             :                            name)));
     186             :     }
     187             : 
     188             :     /*
     189             :      * Avoid writing code that can fail between here and MemoryContextCreate;
     190             :      * we'd leak the header and initial block if we ereport in this stretch.
     191             :      */
     192      249488 :     dlist_init(&set->blocks);
     193             : 
     194             :     /* Fill in the initial block's block header */
     195      249488 :     block = KeeperBlock(set);
     196             :     /* determine the block size and initialize it */
     197      249488 :     firstBlockSize = allocSize - MAXALIGN(sizeof(BumpContext));
     198      249488 :     BumpBlockInit(set, block, firstBlockSize);
     199             : 
     200             :     /* add it to the doubly-linked list of blocks */
     201      249488 :     dlist_push_head(&set->blocks, &block->node);
     202             : 
     203             :     /*
     204             :      * Fill in BumpContext-specific header fields.  The Asserts above should
     205             :      * ensure that these all fit inside a uint32.
     206             :      */
     207      249488 :     set->initBlockSize = (uint32) initBlockSize;
     208      249488 :     set->maxBlockSize = (uint32) maxBlockSize;
     209      249488 :     set->nextBlockSize = (uint32) initBlockSize;
     210             : 
     211             :     /*
     212             :      * Compute the allocation chunk size limit for this context.
     213             :      *
     214             :      * Limit the maximum size a non-dedicated chunk can be so that we can fit
     215             :      * at least Bump_CHUNK_FRACTION of chunks this big onto the maximum sized
     216             :      * block.  We must further limit this value so that it's no more than
     217             :      * MEMORYCHUNK_MAX_VALUE.  We're unable to have non-external chunks larger
     218             :      * than that value as we store the chunk size in the MemoryChunk 'value'
     219             :      * field in the call to MemoryChunkSetHdrMask().
     220             :      */
     221      249488 :     set->allocChunkLimit = Min(maxBlockSize, MEMORYCHUNK_MAX_VALUE);
     222      249488 :     while ((Size) (set->allocChunkLimit + Bump_CHUNKHDRSZ) >
     223     1247440 :            (Size) ((Size) (maxBlockSize - Bump_BLOCKHDRSZ) / Bump_CHUNK_FRACTION))
     224      997952 :         set->allocChunkLimit >>= 1;
     225             : 
     226             :     /* Finally, do the type-independent part of context creation */
     227      249488 :     MemoryContextCreate((MemoryContext) set, T_BumpContext, MCTX_BUMP_ID,
     228             :                         parent, name);
     229             : 
     230      249488 :     ((MemoryContext) set)->mem_allocated = allocSize;
     231             : 
     232      249488 :     return (MemoryContext) set;
     233             : }
     234             : 
     235             : /*
     236             :  * BumpReset
     237             :  *      Frees all memory which is allocated in the given set.
     238             :  *
     239             :  * The code simply frees all the blocks in the context apart from the keeper
     240             :  * block.
     241             :  */
     242             : void
     243      251376 : BumpReset(MemoryContext context)
     244             : {
     245      251376 :     BumpContext *set = (BumpContext *) context;
     246             :     dlist_mutable_iter miter;
     247             : 
     248             :     Assert(BumpIsValid(set));
     249             : 
     250             : #ifdef MEMORY_CONTEXT_CHECKING
     251             :     /* Check for corruption and leaks before freeing */
     252             :     BumpCheck(context);
     253             : #endif
     254             : 
     255      522088 :     dlist_foreach_modify(miter, &set->blocks)
     256             :     {
     257      270712 :         BumpBlock  *block = dlist_container(BumpBlock, node, miter.cur);
     258             : 
     259      270712 :         if (IsKeeperBlock(set, block))
     260      251376 :             BumpBlockMarkEmpty(block);
     261             :         else
     262       19336 :             BumpBlockFree(set, block);
     263             :     }
     264             : 
     265             :     /* Reset block size allocation sequence, too */
     266      251376 :     set->nextBlockSize = set->initBlockSize;
     267             : 
     268             :     /* Ensure there is only 1 item in the dlist */
     269             :     Assert(!dlist_is_empty(&set->blocks));
     270             :     Assert(!dlist_has_next(&set->blocks, dlist_head_node(&set->blocks)));
     271      251376 : }
     272             : 
     273             : /*
     274             :  * BumpDelete
     275             :  *      Free all memory which is allocated in the given context.
     276             :  */
     277             : void
     278      249488 : BumpDelete(MemoryContext context)
     279             : {
     280             :     /* Reset to release all releasable BumpBlocks */
     281      249488 :     BumpReset(context);
     282             :     /* And free the context header and keeper block */
     283      249488 :     free(context);
     284      249488 : }
     285             : 
     286             : /*
     287             :  * Helper for BumpAlloc() that allocates an entire block for the chunk.
     288             :  *
     289             :  * BumpAlloc()'s comment explains why this is separate.
     290             :  */
     291             : pg_noinline
     292             : static void *
     293          18 : BumpAllocLarge(MemoryContext context, Size size, int flags)
     294             : {
     295          18 :     BumpContext *set = (BumpContext *) context;
     296             :     BumpBlock  *block;
     297             : #ifdef MEMORY_CONTEXT_CHECKING
     298             :     MemoryChunk *chunk;
     299             : #endif
     300             :     Size        chunk_size;
     301             :     Size        required_size;
     302             :     Size        blksize;
     303             : 
     304             :     /* validate 'size' is within the limits for the given 'flags' */
     305          18 :     MemoryContextCheckSize(context, size, flags);
     306             : 
     307             : #ifdef MEMORY_CONTEXT_CHECKING
     308             :     /* ensure there's always space for the sentinel byte */
     309             :     chunk_size = MAXALIGN(size + 1);
     310             : #else
     311          18 :     chunk_size = MAXALIGN(size);
     312             : #endif
     313             : 
     314          18 :     required_size = chunk_size + Bump_CHUNKHDRSZ;
     315          18 :     blksize = required_size + Bump_BLOCKHDRSZ;
     316             : 
     317          18 :     block = (BumpBlock *) malloc(blksize);
     318          18 :     if (block == NULL)
     319           0 :         return NULL;
     320             : 
     321          18 :     context->mem_allocated += blksize;
     322             : 
     323             :     /* the block is completely full */
     324          18 :     block->freeptr = block->endptr = ((char *) block) + blksize;
     325             : 
     326             : #ifdef MEMORY_CONTEXT_CHECKING
     327             :     /* block with a single (used) chunk */
     328             :     block->context = set;
     329             : 
     330             :     chunk = (MemoryChunk *) (((char *) block) + Bump_BLOCKHDRSZ);
     331             : 
     332             :     /* mark the MemoryChunk as externally managed */
     333             :     MemoryChunkSetHdrMaskExternal(chunk, MCTX_BUMP_ID);
     334             : 
     335             :     chunk->requested_size = size;
     336             :     /* set mark to catch clobber of "unused" space */
     337             :     Assert(size < chunk_size);
     338             :     set_sentinel(MemoryChunkGetPointer(chunk), size);
     339             : #endif
     340             : #ifdef RANDOMIZE_ALLOCATED_MEMORY
     341             :     /* fill the allocated space with junk */
     342             :     randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
     343             : #endif
     344             : 
     345             :     /*
     346             :      * Add the block to the tail of allocated blocks list.  The current block
     347             :      * is left at the head of the list as it may still have space for
     348             :      * non-large allocations.
     349             :      */
     350          18 :     dlist_push_tail(&set->blocks, &block->node);
     351             : 
     352             : #ifdef MEMORY_CONTEXT_CHECKING
     353             :     /* Ensure any padding bytes are marked NOACCESS. */
     354             :     VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
     355             :                                chunk_size - size);
     356             : 
     357             :     /* Disallow access to the chunk header. */
     358             :     VALGRIND_MAKE_MEM_NOACCESS(chunk, Bump_CHUNKHDRSZ);
     359             : 
     360             :     return MemoryChunkGetPointer(chunk);
     361             : #else
     362          18 :     return (void *) (((char *) block) + Bump_BLOCKHDRSZ);
     363             : #endif
     364             : }
     365             : 
     366             : /*
     367             :  * Small helper for allocating a new chunk from a chunk, to avoid duplicating
     368             :  * the code between BumpAlloc() and BumpAllocFromNewBlock().
     369             :  */
     370             : static inline void *
     371    22452456 : BumpAllocChunkFromBlock(MemoryContext context, BumpBlock *block, Size size,
     372             :                         Size chunk_size)
     373             : {
     374             : #ifdef MEMORY_CONTEXT_CHECKING
     375             :     MemoryChunk *chunk;
     376             : #else
     377             :     void       *ptr;
     378             : #endif
     379             : 
     380             :     /* validate we've been given a block with enough free space */
     381             :     Assert(block != NULL);
     382             :     Assert((block->endptr - block->freeptr) >= Bump_CHUNKHDRSZ + chunk_size);
     383             : 
     384             : #ifdef MEMORY_CONTEXT_CHECKING
     385             :     chunk = (MemoryChunk *) block->freeptr;
     386             : #else
     387    22452456 :     ptr = (void *) block->freeptr;
     388             : #endif
     389             : 
     390             :     /* point the freeptr beyond this chunk */
     391    22452456 :     block->freeptr += (Bump_CHUNKHDRSZ + chunk_size);
     392             :     Assert(block->freeptr <= block->endptr);
     393             : 
     394             : #ifdef MEMORY_CONTEXT_CHECKING
     395             :     /* Prepare to initialize the chunk header. */
     396             :     VALGRIND_MAKE_MEM_UNDEFINED(chunk, Bump_CHUNKHDRSZ);
     397             : 
     398             :     MemoryChunkSetHdrMask(chunk, block, chunk_size, MCTX_BUMP_ID);
     399             :     chunk->requested_size = size;
     400             :     /* set mark to catch clobber of "unused" space */
     401             :     Assert(size < chunk_size);
     402             :     set_sentinel(MemoryChunkGetPointer(chunk), size);
     403             : 
     404             : #ifdef RANDOMIZE_ALLOCATED_MEMORY
     405             :     /* fill the allocated space with junk */
     406             :     randomize_mem((char *) MemoryChunkGetPointer(chunk), size);
     407             : #endif
     408             : 
     409             :     /* Ensure any padding bytes are marked NOACCESS. */
     410             :     VALGRIND_MAKE_MEM_NOACCESS((char *) MemoryChunkGetPointer(chunk) + size,
     411             :                                chunk_size - size);
     412             : 
     413             :     /* Disallow access to the chunk header. */
     414             :     VALGRIND_MAKE_MEM_NOACCESS(chunk, Bump_CHUNKHDRSZ);
     415             : 
     416             :     return MemoryChunkGetPointer(chunk);
     417             : #else
     418    22452456 :     return ptr;
     419             : #endif                          /* MEMORY_CONTEXT_CHECKING */
     420             : }
     421             : 
     422             : /*
     423             :  * Helper for BumpAlloc() that allocates a new block and returns a chunk
     424             :  * allocated from it.
     425             :  *
     426             :  * BumpAlloc()'s comment explains why this is separate.
     427             :  */
     428             : pg_noinline
     429             : static void *
     430       19318 : BumpAllocFromNewBlock(MemoryContext context, Size size, int flags,
     431             :                       Size chunk_size)
     432             : {
     433       19318 :     BumpContext *set = (BumpContext *) context;
     434             :     BumpBlock  *block;
     435             :     Size        blksize;
     436             :     Size        required_size;
     437             : 
     438             :     /*
     439             :      * The first such block has size initBlockSize, and we double the space in
     440             :      * each succeeding block, but not more than maxBlockSize.
     441             :      */
     442       19318 :     blksize = set->nextBlockSize;
     443       19318 :     set->nextBlockSize <<= 1;
     444       19318 :     if (set->nextBlockSize > set->maxBlockSize)
     445           2 :         set->nextBlockSize = set->maxBlockSize;
     446             : 
     447             :     /* we'll need space for the chunk, chunk hdr and block hdr */
     448       19318 :     required_size = chunk_size + Bump_CHUNKHDRSZ + Bump_BLOCKHDRSZ;
     449             :     /* round the size up to the next power of 2 */
     450       19318 :     if (blksize < required_size)
     451           0 :         blksize = pg_nextpower2_size_t(required_size);
     452             : 
     453       19318 :     block = (BumpBlock *) malloc(blksize);
     454             : 
     455       19318 :     if (block == NULL)
     456           0 :         return MemoryContextAllocationFailure(context, size, flags);
     457             : 
     458       19318 :     context->mem_allocated += blksize;
     459             : 
     460             :     /* initialize the new block */
     461       19318 :     BumpBlockInit(set, block, blksize);
     462             : 
     463             :     /* add it to the doubly-linked list of blocks */
     464       19318 :     dlist_push_head(&set->blocks, &block->node);
     465             : 
     466       19318 :     return BumpAllocChunkFromBlock(context, block, size, chunk_size);
     467             : }
     468             : 
     469             : /*
     470             :  * BumpAlloc
     471             :  *      Returns a pointer to allocated memory of given size or raises an ERROR
     472             :  *      on allocation failure, or returns NULL when flags contains
     473             :  *      MCXT_ALLOC_NO_OOM.
     474             :  *
     475             :  * No request may exceed:
     476             :  *      MAXALIGN_DOWN(SIZE_MAX) - Bump_BLOCKHDRSZ - Bump_CHUNKHDRSZ
     477             :  * All callers use a much-lower limit.
     478             :  *
     479             :  *
     480             :  * Note: when using valgrind, it doesn't matter how the returned allocation
     481             :  * is marked, as mcxt.c will set it to UNDEFINED.
     482             :  * This function should only contain the most common code paths.  Everything
     483             :  * else should be in pg_noinline helper functions, thus avoiding the overhead
     484             :  * of creating a stack frame for the common cases.  Allocating memory is often
     485             :  * a bottleneck in many workloads, so avoiding stack frame setup is
     486             :  * worthwhile.  Helper functions should always directly return the newly
     487             :  * allocated memory so that we can just return that address directly as a tail
     488             :  * call.
     489             :  */
     490             : void *
     491    22452474 : BumpAlloc(MemoryContext context, Size size, int flags)
     492             : {
     493    22452474 :     BumpContext *set = (BumpContext *) context;
     494             :     BumpBlock  *block;
     495             :     Size        chunk_size;
     496             :     Size        required_size;
     497             : 
     498             :     Assert(BumpIsValid(set));
     499             : 
     500             : #ifdef MEMORY_CONTEXT_CHECKING
     501             :     /* ensure there's always space for the sentinel byte */
     502             :     chunk_size = MAXALIGN(size + 1);
     503             : #else
     504    22452474 :     chunk_size = MAXALIGN(size);
     505             : #endif
     506             : 
     507             :     /*
     508             :      * If requested size exceeds maximum for chunks we hand the request off to
     509             :      * BumpAllocLarge().
     510             :      */
     511    22452474 :     if (chunk_size > set->allocChunkLimit)
     512          18 :         return BumpAllocLarge(context, size, flags);
     513             : 
     514    22452456 :     required_size = chunk_size + Bump_CHUNKHDRSZ;
     515             : 
     516             :     /*
     517             :      * Not an oversized chunk.  We try to first make use of the latest block,
     518             :      * but if there's not enough space in it we must allocate a new block.
     519             :      */
     520    22452456 :     block = dlist_container(BumpBlock, node, dlist_head_node(&set->blocks));
     521             : 
     522    22452456 :     if (BumpBlockFreeBytes(block) < required_size)
     523       19318 :         return BumpAllocFromNewBlock(context, size, flags, chunk_size);
     524             : 
     525             :     /* The current block has space, so just allocate chunk there. */
     526    22433138 :     return BumpAllocChunkFromBlock(context, block, size, chunk_size);
     527             : }
     528             : 
     529             : /*
     530             :  * BumpBlockInit
     531             :  *      Initializes 'block' assuming 'blksize'.  Does not update the context's
     532             :  *      mem_allocated field.
     533             :  */
     534             : static inline void
     535      268806 : BumpBlockInit(BumpContext *context, BumpBlock *block, Size blksize)
     536             : {
     537             : #ifdef MEMORY_CONTEXT_CHECKING
     538             :     block->context = context;
     539             : #endif
     540      268806 :     block->freeptr = ((char *) block) + Bump_BLOCKHDRSZ;
     541      268806 :     block->endptr = ((char *) block) + blksize;
     542             : 
     543             :     /* Mark unallocated space NOACCESS. */
     544             :     VALGRIND_MAKE_MEM_NOACCESS(block->freeptr, blksize - Bump_BLOCKHDRSZ);
     545      268806 : }
     546             : 
     547             : /*
     548             :  * BumpBlockIsEmpty
     549             :  *      Returns true iff 'block' contains no chunks
     550             :  */
     551             : static inline bool
     552           0 : BumpBlockIsEmpty(BumpBlock *block)
     553             : {
     554             :     /* it's empty if the freeptr has not moved */
     555           0 :     return (block->freeptr == ((char *) block + Bump_BLOCKHDRSZ));
     556             : }
     557             : 
     558             : /*
     559             :  * BumpBlockMarkEmpty
     560             :  *      Set a block as empty.  Does not free the block.
     561             :  */
     562             : static inline void
     563      251376 : BumpBlockMarkEmpty(BumpBlock *block)
     564             : {
     565             : #if defined(USE_VALGRIND) || defined(CLOBBER_FREED_MEMORY)
     566             :     char       *datastart = ((char *) block) + Bump_BLOCKHDRSZ;
     567             : #endif
     568             : 
     569             : #ifdef CLOBBER_FREED_MEMORY
     570             :     wipe_mem(datastart, block->freeptr - datastart);
     571             : #else
     572             :     /* wipe_mem() would have done this */
     573             :     VALGRIND_MAKE_MEM_NOACCESS(datastart, block->freeptr - datastart);
     574             : #endif
     575             : 
     576             :     /* Reset the block, but don't return it to malloc */
     577      251376 :     block->freeptr = ((char *) block) + Bump_BLOCKHDRSZ;
     578      251376 : }
     579             : 
     580             : /*
     581             :  * BumpBlockFreeBytes
     582             :  *      Returns the number of bytes free in 'block'
     583             :  */
     584             : static inline Size
     585    22452456 : BumpBlockFreeBytes(BumpBlock *block)
     586             : {
     587    22452456 :     return (block->endptr - block->freeptr);
     588             : }
     589             : 
     590             : /*
     591             :  * BumpBlockFree
     592             :  *      Remove 'block' from 'set' and release the memory consumed by it.
     593             :  */
     594             : static inline void
     595       19336 : BumpBlockFree(BumpContext *set, BumpBlock *block)
     596             : {
     597             :     /* Make sure nobody tries to free the keeper block */
     598             :     Assert(!IsKeeperBlock(set, block));
     599             : 
     600             :     /* release the block from the list of blocks */
     601       19336 :     dlist_delete(&block->node);
     602             : 
     603       19336 :     ((MemoryContext) set)->mem_allocated -= ((char *) block->endptr - (char *) block);
     604             : 
     605             : #ifdef CLOBBER_FREED_MEMORY
     606             :     wipe_mem(block, ((char *) block->endptr - (char *) block));
     607             : #endif
     608             : 
     609       19336 :     free(block);
     610       19336 : }
     611             : 
     612             : /*
     613             :  * BumpFree
     614             :  *      Unsupported.
     615             :  */
     616             : void
     617           0 : BumpFree(void *pointer)
     618             : {
     619           0 :     elog(ERROR, "%s is not supported by the bump memory allocator", "pfree");
     620             : }
     621             : 
     622             : /*
     623             :  * BumpRealloc
     624             :  *      Unsupported.
     625             :  */
     626             : void *
     627           0 : BumpRealloc(void *pointer, Size size, int flags)
     628             : {
     629           0 :     elog(ERROR, "%s is not supported by the bump memory allocator", "realloc");
     630             :     return NULL;                /* keep compiler quiet */
     631             : }
     632             : 
     633             : /*
     634             :  * BumpGetChunkContext
     635             :  *      Unsupported.
     636             :  */
     637             : MemoryContext
     638           0 : BumpGetChunkContext(void *pointer)
     639             : {
     640           0 :     elog(ERROR, "%s is not supported by the bump memory allocator", "GetMemoryChunkContext");
     641             :     return NULL;                /* keep compiler quiet */
     642             : }
     643             : 
     644             : /*
     645             :  * BumpGetChunkSpace
     646             :  *      Unsupported.
     647             :  */
     648             : Size
     649           0 : BumpGetChunkSpace(void *pointer)
     650             : {
     651           0 :     elog(ERROR, "%s is not supported by the bump memory allocator", "GetMemoryChunkSpace");
     652             :     return 0;                   /* keep compiler quiet */
     653             : }
     654             : 
     655             : /*
     656             :  * BumpIsEmpty
     657             :  *      Is a BumpContext empty of any allocated space?
     658             :  */
     659             : bool
     660           0 : BumpIsEmpty(MemoryContext context)
     661             : {
     662           0 :     BumpContext *set = (BumpContext *) context;
     663             :     dlist_iter  iter;
     664             : 
     665             :     Assert(BumpIsValid(set));
     666             : 
     667           0 :     dlist_foreach(iter, &set->blocks)
     668             :     {
     669           0 :         BumpBlock  *block = dlist_container(BumpBlock, node, iter.cur);
     670             : 
     671           0 :         if (!BumpBlockIsEmpty(block))
     672           0 :             return false;
     673             :     }
     674             : 
     675           0 :     return true;
     676             : }
     677             : 
     678             : /*
     679             :  * BumpStats
     680             :  *      Compute stats about memory consumption of a Bump context.
     681             :  *
     682             :  * printfunc: if not NULL, pass a human-readable stats string to this.
     683             :  * passthru: pass this pointer through to printfunc.
     684             :  * totals: if not NULL, add stats about this context into *totals.
     685             :  * print_to_stderr: print stats to stderr if true, elog otherwise.
     686             :  */
     687             : void
     688           6 : BumpStats(MemoryContext context, MemoryStatsPrintFunc printfunc,
     689             :           void *passthru, MemoryContextCounters *totals, bool print_to_stderr)
     690             : {
     691           6 :     BumpContext *set = (BumpContext *) context;
     692           6 :     Size        nblocks = 0;
     693           6 :     Size        totalspace = 0;
     694           6 :     Size        freespace = 0;
     695             :     dlist_iter  iter;
     696             : 
     697             :     Assert(BumpIsValid(set));
     698             : 
     699          18 :     dlist_foreach(iter, &set->blocks)
     700             :     {
     701          12 :         BumpBlock  *block = dlist_container(BumpBlock, node, iter.cur);
     702             : 
     703          12 :         nblocks++;
     704          12 :         totalspace += (block->endptr - (char *) block);
     705          12 :         freespace += (block->endptr - block->freeptr);
     706             :     }
     707             : 
     708           6 :     if (printfunc)
     709             :     {
     710             :         char        stats_string[200];
     711             : 
     712           0 :         snprintf(stats_string, sizeof(stats_string),
     713             :                  "%zu total in %zu blocks; %zu free; %zu used",
     714             :                  totalspace, nblocks, freespace, totalspace - freespace);
     715           0 :         printfunc(context, passthru, stats_string, print_to_stderr);
     716             :     }
     717             : 
     718           6 :     if (totals)
     719             :     {
     720           6 :         totals->nblocks += nblocks;
     721           6 :         totals->totalspace += totalspace;
     722           6 :         totals->freespace += freespace;
     723             :     }
     724           6 : }
     725             : 
     726             : 
     727             : #ifdef MEMORY_CONTEXT_CHECKING
     728             : 
     729             : /*
     730             :  * BumpCheck
     731             :  *      Walk through chunks and check consistency of memory.
     732             :  *
     733             :  * NOTE: report errors as WARNING, *not* ERROR or FATAL.  Otherwise you'll
     734             :  * find yourself in an infinite loop when trouble occurs, because this
     735             :  * routine will be entered again when elog cleanup tries to release memory!
     736             :  */
     737             : void
     738             : BumpCheck(MemoryContext context)
     739             : {
     740             :     BumpContext *bump = (BumpContext *) context;
     741             :     const char *name = context->name;
     742             :     dlist_iter  iter;
     743             :     Size        total_allocated = 0;
     744             : 
     745             :     /* walk all blocks in this context */
     746             :     dlist_foreach(iter, &bump->blocks)
     747             :     {
     748             :         BumpBlock  *block = dlist_container(BumpBlock, node, iter.cur);
     749             :         int         nchunks;
     750             :         char       *ptr;
     751             :         bool        has_external_chunk = false;
     752             : 
     753             :         if (IsKeeperBlock(bump, block))
     754             :             total_allocated += block->endptr - (char *) bump;
     755             :         else
     756             :             total_allocated += block->endptr - (char *) block;
     757             : 
     758             :         /* check block belongs to the correct context */
     759             :         if (block->context != bump)
     760             :             elog(WARNING, "problem in Bump %s: bogus context link in block %p",
     761             :                  name, block);
     762             : 
     763             :         /* now walk through the chunks and count them */
     764             :         nchunks = 0;
     765             :         ptr = ((char *) block) + Bump_BLOCKHDRSZ;
     766             : 
     767             :         while (ptr < block->freeptr)
     768             :         {
     769             :             MemoryChunk *chunk = (MemoryChunk *) ptr;
     770             :             BumpBlock  *chunkblock;
     771             :             Size        chunksize;
     772             : 
     773             :             /* allow access to the chunk header */
     774             :             VALGRIND_MAKE_MEM_DEFINED(chunk, Bump_CHUNKHDRSZ);
     775             : 
     776             :             if (MemoryChunkIsExternal(chunk))
     777             :             {
     778             :                 chunkblock = ExternalChunkGetBlock(chunk);
     779             :                 chunksize = block->endptr - (char *) MemoryChunkGetPointer(chunk);
     780             :                 has_external_chunk = true;
     781             :             }
     782             :             else
     783             :             {
     784             :                 chunkblock = MemoryChunkGetBlock(chunk);
     785             :                 chunksize = MemoryChunkGetValue(chunk);
     786             :             }
     787             : 
     788             :             /* move to the next chunk */
     789             :             ptr += (chunksize + Bump_CHUNKHDRSZ);
     790             : 
     791             :             nchunks += 1;
     792             : 
     793             :             /* chunks have both block and context pointers, so check both */
     794             :             if (chunkblock != block)
     795             :                 elog(WARNING, "problem in Bump %s: bogus block link in block %p, chunk %p",
     796             :                      name, block, chunk);
     797             :         }
     798             : 
     799             :         if (has_external_chunk && nchunks > 1)
     800             :             elog(WARNING, "problem in Bump %s: external chunk on non-dedicated block %p",
     801             :                  name, block);
     802             : 
     803             :     }
     804             : 
     805             :     Assert(total_allocated == context->mem_allocated);
     806             : }
     807             : 
     808             : #endif                          /* MEMORY_CONTEXT_CHECKING */

Generated by: LCOV version 1.14