LCOV - code coverage report
Current view: top level - src/backend/utils/mmgr - dsa.c (source / functions) Hit Total Coverage
Test: PostgreSQL 12devel Lines: 396 650 60.9 %
Date: 2018-11-13 18:21:17 Functions: 23 34 67.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * dsa.c
       4             :  *    Dynamic shared memory areas.
       5             :  *
       6             :  * This module provides dynamic shared memory areas which are built on top of
       7             :  * DSM segments.  While dsm.c allows segments of memory of shared memory to be
       8             :  * created and shared between backends, it isn't designed to deal with small
       9             :  * objects.  A DSA area is a shared memory heap usually backed by one or more
      10             :  * DSM segments which can allocate memory using dsa_allocate() and dsa_free().
      11             :  * Alternatively, it can be created in pre-existing shared memory, including a
      12             :  * DSM segment, and then create extra DSM segments as required.  Unlike the
      13             :  * regular system heap, it deals in pseudo-pointers which must be converted to
      14             :  * backend-local pointers before they are dereferenced.  These pseudo-pointers
      15             :  * can however be shared with other backends, and can be used to construct
      16             :  * shared data structures.
      17             :  *
      18             :  * Each DSA area manages a set of DSM segments, adding new segments as
      19             :  * required and detaching them when they are no longer needed.  Each segment
      20             :  * contains a number of 4KB pages, a free page manager for tracking
      21             :  * consecutive runs of free pages, and a page map for tracking the source of
      22             :  * objects allocated on each page.  Allocation requests above 8KB are handled
      23             :  * by choosing a segment and finding consecutive free pages in its free page
      24             :  * manager.  Allocation requests for smaller sizes are handled using pools of
      25             :  * objects of a selection of sizes.  Each pool consists of a number of 16 page
      26             :  * (64KB) superblocks allocated in the same way as large objects.  Allocation
      27             :  * of large objects and new superblocks is serialized by a single LWLock, but
      28             :  * allocation of small objects from pre-existing superblocks uses one LWLock
      29             :  * per pool.  Currently there is one pool, and therefore one lock, per size
      30             :  * class.  Per-core pools to increase concurrency and strategies for reducing
      31             :  * the resulting fragmentation are areas for future research.  Each superblock
      32             :  * is managed with a 'span', which tracks the superblock's freelist.  Free
      33             :  * requests are handled by looking in the page map to find which span an
      34             :  * address was allocated from, so that small objects can be returned to the
      35             :  * appropriate free list, and large object pages can be returned directly to
      36             :  * the free page map.  When allocating, simple heuristics for selecting
      37             :  * segments and superblocks try to encourage occupied memory to be
      38             :  * concentrated, increasing the likelihood that whole superblocks can become
      39             :  * empty and be returned to the free page manager, and whole segments can
      40             :  * become empty and be returned to the operating system.
      41             :  *
      42             :  * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group
      43             :  * Portions Copyright (c) 1994, Regents of the University of California
      44             :  *
      45             :  * IDENTIFICATION
      46             :  *    src/backend/utils/mmgr/dsa.c
      47             :  *
      48             :  *-------------------------------------------------------------------------
      49             :  */
      50             : 
      51             : #include "postgres.h"
      52             : 
      53             : #include "port/atomics.h"
      54             : #include "storage/dsm.h"
      55             : #include "storage/ipc.h"
      56             : #include "storage/lwlock.h"
      57             : #include "storage/shmem.h"
      58             : #include "utils/dsa.h"
      59             : #include "utils/freepage.h"
      60             : #include "utils/memutils.h"
      61             : 
      62             : /*
      63             :  * The size of the initial DSM segment that backs a dsa_area created by
      64             :  * dsa_create.  After creating some number of segments of this size we'll
      65             :  * double this size, and so on.  Larger segments may be created if necessary
      66             :  * to satisfy large requests.
      67             :  */
      68             : #define DSA_INITIAL_SEGMENT_SIZE ((size_t) (1 * 1024 * 1024))
      69             : 
      70             : /*
      71             :  * How many segments to create before we double the segment size.  If this is
      72             :  * low, then there is likely to be a lot of wasted space in the largest
      73             :  * segment.  If it is high, then we risk running out of segment slots (see
      74             :  * dsm.c's limits on total number of segments), or limiting the total size
      75             :  * an area can manage when using small pointers.
      76             :  */
      77             : #define DSA_NUM_SEGMENTS_AT_EACH_SIZE 4
      78             : 
      79             : /*
      80             :  * The number of bits used to represent the offset part of a dsa_pointer.
      81             :  * This controls the maximum size of a segment, the maximum possible
      82             :  * allocation size and also the maximum number of segments per area.
      83             :  */
      84             : #if SIZEOF_DSA_POINTER == 4
      85             : #define DSA_OFFSET_WIDTH 27     /* 32 segments of size up to 128MB */
      86             : #else
      87             : #define DSA_OFFSET_WIDTH 40     /* 1024 segments of size up to 1TB */
      88             : #endif
      89             : 
      90             : /*
      91             :  * The maximum number of DSM segments that an area can own, determined by
      92             :  * the number of bits remaining (but capped at 1024).
      93             :  */
      94             : #define DSA_MAX_SEGMENTS \
      95             :     Min(1024, (1 << ((SIZEOF_DSA_POINTER * 8) - DSA_OFFSET_WIDTH)))
      96             : 
      97             : /* The bitmask for extracting the offset from a dsa_pointer. */
      98             : #define DSA_OFFSET_BITMASK (((dsa_pointer) 1 << DSA_OFFSET_WIDTH) - 1)
      99             : 
     100             : /* The maximum size of a DSM segment. */
     101             : #define DSA_MAX_SEGMENT_SIZE ((size_t) 1 << DSA_OFFSET_WIDTH)
     102             : 
     103             : /* Number of pages (see FPM_PAGE_SIZE) per regular superblock. */
     104             : #define DSA_PAGES_PER_SUPERBLOCK        16
     105             : 
     106             : /*
     107             :  * A magic number used as a sanity check for following DSM segments belonging
     108             :  * to a DSA area (this number will be XORed with the area handle and
     109             :  * the segment index).
     110             :  */
     111             : #define DSA_SEGMENT_HEADER_MAGIC 0x0ce26608
     112             : 
     113             : /* Build a dsa_pointer given a segment number and offset. */
     114             : #define DSA_MAKE_POINTER(segment_number, offset) \
     115             :     (((dsa_pointer) (segment_number) << DSA_OFFSET_WIDTH) | (offset))
     116             : 
     117             : /* Extract the segment number from a dsa_pointer. */
     118             : #define DSA_EXTRACT_SEGMENT_NUMBER(dp) ((dp) >> DSA_OFFSET_WIDTH)
     119             : 
     120             : /* Extract the offset from a dsa_pointer. */
     121             : #define DSA_EXTRACT_OFFSET(dp) ((dp) & DSA_OFFSET_BITMASK)
     122             : 
     123             : /* The type used for index segment indexes (zero based). */
     124             : typedef size_t dsa_segment_index;
     125             : 
     126             : /* Sentinel value for dsa_segment_index indicating 'none' or 'end'. */
     127             : #define DSA_SEGMENT_INDEX_NONE (~(dsa_segment_index)0)
     128             : 
     129             : /*
     130             :  * How many bins of segments do we have?  The bins are used to categorize
     131             :  * segments by their largest contiguous run of free pages.
     132             :  */
     133             : #define DSA_NUM_SEGMENT_BINS 16
     134             : 
     135             : /*
     136             :  * What is the lowest bin that holds segments that *might* have n contiguous
     137             :  * free pages?  There is no point in looking in segments in lower bins; they
     138             :  * definitely can't service a request for n free pages.
     139             :  */
     140             : #define contiguous_pages_to_segment_bin(n) Min(fls(n), DSA_NUM_SEGMENT_BINS - 1)
     141             : 
     142             : /* Macros for access to locks. */
     143             : #define DSA_AREA_LOCK(area) (&area->control->lock)
     144             : #define DSA_SCLASS_LOCK(area, sclass) (&area->control->pools[sclass].lock)
     145             : 
     146             : /*
     147             :  * The header for an individual segment.  This lives at the start of each DSM
     148             :  * segment owned by a DSA area including the first segment (where it appears
     149             :  * as part of the dsa_area_control struct).
     150             :  */
     151             : typedef struct
     152             : {
     153             :     /* Sanity check magic value. */
     154             :     uint32      magic;
     155             :     /* Total number of pages in this segment (excluding metadata area). */
     156             :     size_t      usable_pages;
     157             :     /* Total size of this segment in bytes. */
     158             :     size_t      size;
     159             : 
     160             :     /*
     161             :      * Index of the segment that precedes this one in the same segment bin, or
     162             :      * DSA_SEGMENT_INDEX_NONE if this is the first one.
     163             :      */
     164             :     dsa_segment_index prev;
     165             : 
     166             :     /*
     167             :      * Index of the segment that follows this one in the same segment bin, or
     168             :      * DSA_SEGMENT_INDEX_NONE if this is the last one.
     169             :      */
     170             :     dsa_segment_index next;
     171             :     /* The index of the bin that contains this segment. */
     172             :     size_t      bin;
     173             : 
     174             :     /*
     175             :      * A flag raised to indicate that this segment is being returned to the
     176             :      * operating system and has been unpinned.
     177             :      */
     178             :     bool        freed;
     179             : } dsa_segment_header;
     180             : 
     181             : /*
     182             :  * Metadata for one superblock.
     183             :  *
     184             :  * For most blocks, span objects are stored out-of-line; that is, the span
     185             :  * object is not stored within the block itself.  But, as an exception, for a
     186             :  * "span of spans", the span object is stored "inline".  The allocation is
     187             :  * always exactly one page, and the dsa_area_span object is located at
     188             :  * the beginning of that page.  The size class is DSA_SCLASS_BLOCK_OF_SPANS,
     189             :  * and the remaining fields are used just as they would be in an ordinary
     190             :  * block.  We can't allocate spans out of ordinary superblocks because
     191             :  * creating an ordinary superblock requires us to be able to allocate a span
     192             :  * *first*.  Doing it this way avoids that circularity.
     193             :  */
     194             : typedef struct
     195             : {
     196             :     dsa_pointer pool;           /* Containing pool. */
     197             :     dsa_pointer prevspan;       /* Previous span. */
     198             :     dsa_pointer nextspan;       /* Next span. */
     199             :     dsa_pointer start;          /* Starting address. */
     200             :     size_t      npages;         /* Length of span in pages. */
     201             :     uint16      size_class;     /* Size class. */
     202             :     uint16      ninitialized;   /* Maximum number of objects ever allocated. */
     203             :     uint16      nallocatable;   /* Number of objects currently allocatable. */
     204             :     uint16      firstfree;      /* First object on free list. */
     205             :     uint16      nmax;           /* Maximum number of objects ever possible. */
     206             :     uint16      fclass;         /* Current fullness class. */
     207             : } dsa_area_span;
     208             : 
     209             : /*
     210             :  * Given a pointer to an object in a span, access the index of the next free
     211             :  * object in the same span (ie in the span's freelist) as an L-value.
     212             :  */
     213             : #define NextFreeObjectIndex(object) (* (uint16 *) (object))
     214             : 
     215             : /*
     216             :  * Small allocations are handled by dividing a single block of memory into
     217             :  * many small objects of equal size.  The possible allocation sizes are
     218             :  * defined by the following array.  Larger size classes are spaced more widely
     219             :  * than smaller size classes.  We fudge the spacing for size classes >1kB to
     220             :  * avoid space wastage: based on the knowledge that we plan to allocate 64kB
     221             :  * blocks, we bump the maximum object size up to the largest multiple of
     222             :  * 8 bytes that still lets us fit the same number of objects into one block.
     223             :  *
     224             :  * NB: Because of this fudging, if we were ever to use differently-sized blocks
     225             :  * for small allocations, these size classes would need to be reworked to be
     226             :  * optimal for the new size.
     227             :  *
     228             :  * NB: The optimal spacing for size classes, as well as the size of the blocks
     229             :  * out of which small objects are allocated, is not a question that has one
     230             :  * right answer.  Some allocators (such as tcmalloc) use more closely-spaced
     231             :  * size classes than we do here, while others (like aset.c) use more
     232             :  * widely-spaced classes.  Spacing the classes more closely avoids wasting
     233             :  * memory within individual chunks, but also means a larger number of
     234             :  * potentially-unfilled blocks.
     235             :  */
     236             : static const uint16 dsa_size_classes[] = {
     237             :     sizeof(dsa_area_span), 0,   /* special size classes */
     238             :     8, 16, 24, 32, 40, 48, 56, 64,  /* 8 classes separated by 8 bytes */
     239             :     80, 96, 112, 128,           /* 4 classes separated by 16 bytes */
     240             :     160, 192, 224, 256,         /* 4 classes separated by 32 bytes */
     241             :     320, 384, 448, 512,         /* 4 classes separated by 64 bytes */
     242             :     640, 768, 896, 1024,        /* 4 classes separated by 128 bytes */
     243             :     1280, 1560, 1816, 2048,     /* 4 classes separated by ~256 bytes */
     244             :     2616, 3120, 3640, 4096,     /* 4 classes separated by ~512 bytes */
     245             :     5456, 6552, 7280, 8192      /* 4 classes separated by ~1024 bytes */
     246             : };
     247             : #define DSA_NUM_SIZE_CLASSES                lengthof(dsa_size_classes)
     248             : 
     249             : /* Special size classes. */
     250             : #define DSA_SCLASS_BLOCK_OF_SPANS       0
     251             : #define DSA_SCLASS_SPAN_LARGE           1
     252             : 
     253             : /*
     254             :  * The following lookup table is used to map the size of small objects
     255             :  * (less than 1kB) onto the corresponding size class.  To use this table,
     256             :  * round the size of the object up to the next multiple of 8 bytes, and then
     257             :  * index into this array.
     258             :  */
     259             : static const uint8 dsa_size_class_map[] = {
     260             :     2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 11, 11, 12, 12, 13, 13,
     261             :     14, 14, 14, 14, 15, 15, 15, 15, 16, 16, 16, 16, 17, 17, 17, 17,
     262             :     18, 18, 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19,
     263             :     20, 20, 20, 20, 20, 20, 20, 20, 21, 21, 21, 21, 21, 21, 21, 21,
     264             :     22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
     265             :     23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23, 23,
     266             :     24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
     267             :     25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25
     268             : };
     269             : #define DSA_SIZE_CLASS_MAP_QUANTUM  8
     270             : 
     271             : /*
     272             :  * Superblocks are binned by how full they are.  Generally, each fullness
     273             :  * class corresponds to one quartile, but the block being used for
     274             :  * allocations is always at the head of the list for fullness class 1,
     275             :  * regardless of how full it really is.
     276             :  */
     277             : #define DSA_FULLNESS_CLASSES        4
     278             : 
     279             : /*
     280             :  * A dsa_area_pool represents a set of objects of a given size class.
     281             :  *
     282             :  * Perhaps there should be multiple pools for the same size class for
     283             :  * contention avoidance, but for now there is just one!
     284             :  */
     285             : typedef struct
     286             : {
     287             :     /* A lock protecting access to this pool. */
     288             :     LWLock      lock;
     289             :     /* A set of linked lists of spans, arranged by fullness. */
     290             :     dsa_pointer spans[DSA_FULLNESS_CLASSES];
     291             :     /* Should we pad this out to a cacheline boundary? */
     292             : } dsa_area_pool;
     293             : 
     294             : /*
     295             :  * The control block for an area.  This lives in shared memory, at the start of
     296             :  * the first DSM segment controlled by this area.
     297             :  */
     298             : typedef struct
     299             : {
     300             :     /* The segment header for the first segment. */
     301             :     dsa_segment_header segment_header;
     302             :     /* The handle for this area. */
     303             :     dsa_handle  handle;
     304             :     /* The handles of the segments owned by this area. */
     305             :     dsm_handle  segment_handles[DSA_MAX_SEGMENTS];
     306             :     /* Lists of segments, binned by maximum contiguous run of free pages. */
     307             :     dsa_segment_index segment_bins[DSA_NUM_SEGMENT_BINS];
     308             :     /* The object pools for each size class. */
     309             :     dsa_area_pool pools[DSA_NUM_SIZE_CLASSES];
     310             :     /* The total size of all active segments. */
     311             :     size_t      total_segment_size;
     312             :     /* The maximum total size of backing storage we are allowed. */
     313             :     size_t      max_total_segment_size;
     314             :     /* Highest used segment index in the history of this area. */
     315             :     dsa_segment_index high_segment_index;
     316             :     /* The reference count for this area. */
     317             :     int         refcnt;
     318             :     /* A flag indicating that this area has been pinned. */
     319             :     bool        pinned;
     320             :     /* The number of times that segments have been freed. */
     321             :     size_t      freed_segment_counter;
     322             :     /* The LWLock tranche ID. */
     323             :     int         lwlock_tranche_id;
     324             :     /* The general lock (protects everything except object pools). */
     325             :     LWLock      lock;
     326             : } dsa_area_control;
     327             : 
     328             : /* Given a pointer to a pool, find a dsa_pointer. */
     329             : #define DsaAreaPoolToDsaPointer(area, p)    \
     330             :     DSA_MAKE_POINTER(0, (char *) p - (char *) area->control)
     331             : 
     332             : /*
     333             :  * A dsa_segment_map is stored within the backend-private memory of each
     334             :  * individual backend.  It holds the base address of the segment within that
     335             :  * backend, plus the addresses of key objects within the segment.  Those
     336             :  * could instead be derived from the base address but it's handy to have them
     337             :  * around.
     338             :  */
     339             : typedef struct
     340             : {
     341             :     dsm_segment *segment;       /* DSM segment */
     342             :     char       *mapped_address; /* Address at which segment is mapped */
     343             :     dsa_segment_header *header; /* Header (same as mapped_address) */
     344             :     FreePageManager *fpm;       /* Free page manager within segment. */
     345             :     dsa_pointer *pagemap;       /* Page map within segment. */
     346             : } dsa_segment_map;
     347             : 
     348             : /*
     349             :  * Per-backend state for a storage area.  Backends obtain one of these by
     350             :  * creating an area or attaching to an existing one using a handle.  Each
     351             :  * process that needs to use an area uses its own object to track where the
     352             :  * segments are mapped.
     353             :  */
     354             : struct dsa_area
     355             : {
     356             :     /* Pointer to the control object in shared memory. */
     357             :     dsa_area_control *control;
     358             : 
     359             :     /* Has the mapping been pinned? */
     360             :     bool        mapping_pinned;
     361             : 
     362             :     /*
     363             :      * This backend's array of segment maps, ordered by segment index
     364             :      * corresponding to control->segment_handles.  Some of the area's segments
     365             :      * may not be mapped in this backend yet, and some slots may have been
     366             :      * freed and need to be detached; these operations happen on demand.
     367             :      */
     368             :     dsa_segment_map segment_maps[DSA_MAX_SEGMENTS];
     369             : 
     370             :     /* The highest segment index this backend has ever mapped. */
     371             :     dsa_segment_index high_segment_index;
     372             : 
     373             :     /* The last observed freed_segment_counter. */
     374             :     size_t      freed_segment_counter;
     375             : };
     376             : 
     377             : #define DSA_SPAN_NOTHING_FREE   ((uint16) -1)
     378             : #define DSA_SUPERBLOCK_SIZE (DSA_PAGES_PER_SUPERBLOCK * FPM_PAGE_SIZE)
     379             : 
     380             : /* Given a pointer to a segment_map, obtain a segment index number. */
     381             : #define get_segment_index(area, segment_map_ptr) \
     382             :     (segment_map_ptr - &area->segment_maps[0])
     383             : 
     384             : static void init_span(dsa_area *area, dsa_pointer span_pointer,
     385             :           dsa_area_pool *pool, dsa_pointer start, size_t npages,
     386             :           uint16 size_class);
     387             : static bool transfer_first_span(dsa_area *area, dsa_area_pool *pool,
     388             :                     int fromclass, int toclass);
     389             : static inline dsa_pointer alloc_object(dsa_area *area, int size_class);
     390             : static bool ensure_active_superblock(dsa_area *area, dsa_area_pool *pool,
     391             :                          int size_class);
     392             : static dsa_segment_map *get_segment_by_index(dsa_area *area,
     393             :                      dsa_segment_index index);
     394             : static void destroy_superblock(dsa_area *area, dsa_pointer span_pointer);
     395             : static void unlink_span(dsa_area *area, dsa_area_span *span);
     396             : static void add_span_to_fullness_class(dsa_area *area, dsa_area_span *span,
     397             :                            dsa_pointer span_pointer, int fclass);
     398             : static void unlink_segment(dsa_area *area, dsa_segment_map *segment_map);
     399             : static dsa_segment_map *get_best_segment(dsa_area *area, size_t npages);
     400             : static dsa_segment_map *make_new_segment(dsa_area *area, size_t requested_pages);
     401             : static dsa_area *create_internal(void *place, size_t size,
     402             :                 int tranche_id,
     403             :                 dsm_handle control_handle,
     404             :                 dsm_segment *control_segment);
     405             : static dsa_area *attach_internal(void *place, dsm_segment *segment,
     406             :                 dsa_handle handle);
     407             : static void check_for_freed_segments(dsa_area *area);
     408             : static void check_for_freed_segments_locked(dsa_area *area);
     409             : 
     410             : /*
     411             :  * Create a new shared area in a new DSM segment.  Further DSM segments will
     412             :  * be allocated as required to extend the available space.
     413             :  *
     414             :  * We can't allocate a LWLock tranche_id within this function, because tranche
     415             :  * IDs are a scarce resource; there are only 64k available, using low numbers
     416             :  * when possible matters, and we have no provision for recycling them.  So,
     417             :  * we require the caller to provide one.
     418             :  */
     419             : dsa_area *
     420           0 : dsa_create(int tranche_id)
     421             : {
     422             :     dsm_segment *segment;
     423             :     dsa_area   *area;
     424             : 
     425             :     /*
     426             :      * Create the DSM segment that will hold the shared control object and the
     427             :      * first segment of usable space.
     428             :      */
     429           0 :     segment = dsm_create(DSA_INITIAL_SEGMENT_SIZE, 0);
     430             : 
     431             :     /*
     432             :      * All segments backing this area are pinned, so that DSA can explicitly
     433             :      * control their lifetime (otherwise a newly created segment belonging to
     434             :      * this area might be freed when the only backend that happens to have it
     435             :      * mapped in ends, corrupting the area).
     436             :      */
     437           0 :     dsm_pin_segment(segment);
     438             : 
     439             :     /* Create a new DSA area with the control object in this segment. */
     440           0 :     area = create_internal(dsm_segment_address(segment),
     441             :                            DSA_INITIAL_SEGMENT_SIZE,
     442             :                            tranche_id,
     443             :                            dsm_segment_handle(segment), segment);
     444             : 
     445             :     /* Clean up when the control segment detaches. */
     446           0 :     on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
     447           0 :                   PointerGetDatum(dsm_segment_address(segment)));
     448             : 
     449           0 :     return area;
     450             : }
     451             : 
     452             : /*
     453             :  * Create a new shared area in an existing shared memory space, which may be
     454             :  * either DSM or Postmaster-initialized memory.  DSM segments will be
     455             :  * allocated as required to extend the available space, though that can be
     456             :  * prevented with dsa_set_size_limit(area, size) using the same size provided
     457             :  * to dsa_create_in_place.
     458             :  *
     459             :  * Areas created in-place must eventually be released by the backend that
     460             :  * created them and all backends that attach to them.  This can be done
     461             :  * explicitly with dsa_release_in_place, or, in the special case that 'place'
     462             :  * happens to be in a pre-existing DSM segment, by passing in a pointer to the
     463             :  * segment so that a detach hook can be registered with the containing DSM
     464             :  * segment.
     465             :  *
     466             :  * See dsa_create() for a note about the tranche arguments.
     467             :  */
     468             : dsa_area *
     469         420 : dsa_create_in_place(void *place, size_t size,
     470             :                     int tranche_id, dsm_segment *segment)
     471             : {
     472             :     dsa_area   *area;
     473             : 
     474         420 :     area = create_internal(place, size, tranche_id,
     475             :                            DSM_HANDLE_INVALID, NULL);
     476             : 
     477             :     /*
     478             :      * Clean up when the control segment detaches, if a containing DSM segment
     479             :      * was provided.
     480             :      */
     481         420 :     if (segment != NULL)
     482         420 :         on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
     483             :                       PointerGetDatum(place));
     484             : 
     485         420 :     return area;
     486             : }
     487             : 
     488             : /*
     489             :  * Obtain a handle that can be passed to other processes so that they can
     490             :  * attach to the given area.  Cannot be called for areas created with
     491             :  * dsa_create_in_place.
     492             :  */
     493             : dsa_handle
     494           0 : dsa_get_handle(dsa_area *area)
     495             : {
     496             :     Assert(area->control->handle != DSM_HANDLE_INVALID);
     497           0 :     return area->control->handle;
     498             : }
     499             : 
     500             : /*
     501             :  * Attach to an area given a handle generated (possibly in another process) by
     502             :  * dsa_get_handle.  The area must have been created with dsa_create (not
     503             :  * dsa_create_in_place).
     504             :  */
     505             : dsa_area *
     506           0 : dsa_attach(dsa_handle handle)
     507             : {
     508             :     dsm_segment *segment;
     509             :     dsa_area   *area;
     510             : 
     511             :     /*
     512             :      * An area handle is really a DSM segment handle for the first segment, so
     513             :      * we go ahead and attach to that.
     514             :      */
     515           0 :     segment = dsm_attach(handle);
     516           0 :     if (segment == NULL)
     517           0 :         ereport(ERROR,
     518             :                 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
     519             :                  errmsg("could not attach to dynamic shared area")));
     520             : 
     521           0 :     area = attach_internal(dsm_segment_address(segment), segment, handle);
     522             : 
     523             :     /* Clean up when the control segment detaches. */
     524           0 :     on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
     525           0 :                   PointerGetDatum(dsm_segment_address(segment)));
     526             : 
     527           0 :     return area;
     528             : }
     529             : 
     530             : /*
     531             :  * Attach to an area that was created with dsa_create_in_place.  The caller
     532             :  * must somehow know the location in memory that was used when the area was
     533             :  * created, though it may be mapped at a different virtual address in this
     534             :  * process.
     535             :  *
     536             :  * See dsa_create_in_place for note about releasing in-place areas, and the
     537             :  * optional 'segment' argument which can be provided to allow automatic
     538             :  * release if the containing memory happens to be a DSM segment.
     539             :  */
     540             : dsa_area *
     541        3068 : dsa_attach_in_place(void *place, dsm_segment *segment)
     542             : {
     543             :     dsa_area   *area;
     544             : 
     545        3068 :     area = attach_internal(place, NULL, DSM_HANDLE_INVALID);
     546             : 
     547             :     /*
     548             :      * Clean up when the control segment detaches, if a containing DSM segment
     549             :      * was provided.
     550             :      */
     551        3068 :     if (segment != NULL)
     552        3068 :         on_dsm_detach(segment, &dsa_on_dsm_detach_release_in_place,
     553             :                       PointerGetDatum(place));
     554             : 
     555        3068 :     return area;
     556             : }
     557             : 
     558             : /*
     559             :  * Release a DSA area that was produced by dsa_create_in_place or
     560             :  * dsa_attach_in_place.  The 'segment' argument is ignored but provides an
     561             :  * interface suitable for on_dsm_detach, for the convenience of users who want
     562             :  * to create a DSA segment inside an existing DSM segment and have it
     563             :  * automatically released when the containing DSM segment is detached.
     564             :  * 'place' should be the address of the place where the area was created.
     565             :  *
     566             :  * This callback is automatically registered for the DSM segment containing
     567             :  * the control object of in-place areas when a segment is provided to
     568             :  * dsa_create_in_place or dsa_attach_in_place, and also for all areas created
     569             :  * with dsa_create.
     570             :  */
     571             : void
     572        3488 : dsa_on_dsm_detach_release_in_place(dsm_segment *segment, Datum place)
     573             : {
     574        3488 :     dsa_release_in_place(DatumGetPointer(place));
     575        3488 : }
     576             : 
     577             : /*
     578             :  * Release a DSA area that was produced by dsa_create_in_place or
     579             :  * dsa_attach_in_place.  The 'code' argument is ignored but provides an
     580             :  * interface suitable for on_shmem_exit or before_shmem_exit, for the
     581             :  * convenience of users who want to create a DSA segment inside shared memory
     582             :  * other than a DSM segment and have it automatically release at backend exit.
     583             :  * 'place' should be the address of the place where the area was created.
     584             :  */
     585             : void
     586           0 : dsa_on_shmem_exit_release_in_place(int code, Datum place)
     587             : {
     588           0 :     dsa_release_in_place(DatumGetPointer(place));
     589           0 : }
     590             : 
     591             : /*
     592             :  * Release a DSA area that was produced by dsa_create_in_place or
     593             :  * dsa_attach_in_place.  It is preferable to use one of the 'dsa_on_XXX'
     594             :  * callbacks so that this is managed automatically, because failure to release
     595             :  * an area created in-place leaks its segments permanently.
     596             :  *
     597             :  * This is also called automatically for areas produced by dsa_create or
     598             :  * dsa_attach as an implementation detail.
     599             :  */
     600             : void
     601        3488 : dsa_release_in_place(void *place)
     602             : {
     603        3488 :     dsa_area_control *control = (dsa_area_control *) place;
     604             :     int         i;
     605             : 
     606        3488 :     LWLockAcquire(&control->lock, LW_EXCLUSIVE);
     607             :     Assert(control->segment_header.magic ==
     608             :            (DSA_SEGMENT_HEADER_MAGIC ^ control->handle ^ 0));
     609             :     Assert(control->refcnt > 0);
     610        3488 :     if (--control->refcnt == 0)
     611             :     {
     612         948 :         for (i = 0; i <= control->high_segment_index; ++i)
     613             :         {
     614             :             dsm_handle  handle;
     615             : 
     616         528 :             handle = control->segment_handles[i];
     617         528 :             if (handle != DSM_HANDLE_INVALID)
     618         108 :                 dsm_unpin_segment(handle);
     619             :         }
     620             :     }
     621        3488 :     LWLockRelease(&control->lock);
     622        3488 : }
     623             : 
     624             : /*
     625             :  * Keep a DSA area attached until end of session or explicit detach.
     626             :  *
     627             :  * By default, areas are owned by the current resource owner, which means they
     628             :  * are detached automatically when that scope ends.
     629             :  */
     630             : void
     631        1608 : dsa_pin_mapping(dsa_area *area)
     632             : {
     633             :     int         i;
     634             : 
     635             :     Assert(!area->mapping_pinned);
     636        1608 :     area->mapping_pinned = true;
     637             : 
     638        3220 :     for (i = 0; i <= area->high_segment_index; ++i)
     639        1612 :         if (area->segment_maps[i].segment != NULL)
     640           4 :             dsm_pin_mapping(area->segment_maps[i].segment);
     641        1608 : }
     642             : 
     643             : /*
     644             :  * Allocate memory in this storage area.  The return value is a dsa_pointer
     645             :  * that can be passed to other processes, and converted to a local pointer
     646             :  * with dsa_get_address.  'flags' is a bitmap which should be constructed
     647             :  * from the following values:
     648             :  *
     649             :  * DSA_ALLOC_HUGE allows allocations >= 1GB.  Otherwise, such allocations
     650             :  * will result in an ERROR.
     651             :  *
     652             :  * DSA_ALLOC_NO_OOM causes this function to return InvalidDsaPointer when
     653             :  * no memory is available or a size limit established by set_dsa_size_limit
     654             :  * would be exceeded.  Otherwise, such allocations will result in an ERROR.
     655             :  *
     656             :  * DSA_ALLOC_ZERO causes the allocated memory to be zeroed.  Otherwise, the
     657             :  * contents of newly-allocated memory are indeterminate.
     658             :  *
     659             :  * These flags correspond to similarly named flags used by
     660             :  * MemoryContextAllocExtended().  See also the macros dsa_allocate and
     661             :  * dsa_allocate0 which expand to a call to this function with commonly used
     662             :  * flags.
     663             :  */
     664             : dsa_pointer
     665        3090 : dsa_allocate_extended(dsa_area *area, size_t size, int flags)
     666             : {
     667             :     uint16      size_class;
     668             :     dsa_pointer start_pointer;
     669             :     dsa_segment_map *segment_map;
     670             :     dsa_pointer result;
     671             : 
     672             :     Assert(size > 0);
     673             : 
     674             :     /* Sanity check on huge individual allocation size. */
     675        6180 :     if (((flags & DSA_ALLOC_HUGE) != 0 && !AllocHugeSizeIsValid(size)) ||
     676        6084 :         ((flags & DSA_ALLOC_HUGE) == 0 && !AllocSizeIsValid(size)))
     677           0 :         elog(ERROR, "invalid DSA memory alloc request size %zu", size);
     678             : 
     679             :     /*
     680             :      * If bigger than the largest size class, just grab a run of pages from
     681             :      * the free page manager, instead of allocating an object from a pool.
     682             :      * There will still be a span, but it's a special class of span that
     683             :      * manages this whole allocation and simply gives all pages back to the
     684             :      * free page manager when dsa_free is called.
     685             :      */
     686        3090 :     if (size > dsa_size_classes[lengthof(dsa_size_classes) - 1])
     687             :     {
     688        2494 :         size_t      npages = fpm_size_to_pages(size);
     689             :         size_t      first_page;
     690             :         dsa_pointer span_pointer;
     691        2494 :         dsa_area_pool *pool = &area->control->pools[DSA_SCLASS_SPAN_LARGE];
     692             : 
     693             :         /* Obtain a span object. */
     694        2494 :         span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
     695        2494 :         if (!DsaPointerIsValid(span_pointer))
     696           0 :             return InvalidDsaPointer;
     697             : 
     698        2494 :         LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     699             : 
     700             :         /* Find a segment from which to allocate. */
     701        2494 :         segment_map = get_best_segment(area, npages);
     702        2494 :         if (segment_map == NULL)
     703          20 :             segment_map = make_new_segment(area, npages);
     704        2494 :         if (segment_map == NULL)
     705             :         {
     706             :             /* Can't make any more segments: game over. */
     707           0 :             LWLockRelease(DSA_AREA_LOCK(area));
     708           0 :             dsa_free(area, span_pointer);
     709             : 
     710             :             /* Raise error unless asked not to. */
     711           0 :             if ((flags & DSA_ALLOC_NO_OOM) == 0)
     712           0 :                 ereport(ERROR,
     713             :                         (errcode(ERRCODE_OUT_OF_MEMORY),
     714             :                          errmsg("out of memory"),
     715             :                          errdetail("Failed on DSA request of size %zu.",
     716             :                                    size)));
     717           0 :             return InvalidDsaPointer;
     718             :         }
     719             : 
     720             :         /*
     721             :          * Ask the free page manager for a run of pages.  This should always
     722             :          * succeed, since both get_best_segment and make_new_segment should
     723             :          * only return a non-NULL pointer if it actually contains enough
     724             :          * contiguous freespace.  If it does fail, something in our backend
     725             :          * private state is out of whack, so use FATAL to kill the process.
     726             :          */
     727        2494 :         if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
     728           0 :             elog(FATAL,
     729             :                  "dsa_allocate could not find %zu free pages", npages);
     730        2494 :         LWLockRelease(DSA_AREA_LOCK(area));
     731             : 
     732        2494 :         start_pointer = DSA_MAKE_POINTER(get_segment_index(area, segment_map),
     733             :                                          first_page * FPM_PAGE_SIZE);
     734             : 
     735             :         /* Initialize span and pagemap. */
     736        2494 :         LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
     737             :                       LW_EXCLUSIVE);
     738        2494 :         init_span(area, span_pointer, pool, start_pointer, npages,
     739             :                   DSA_SCLASS_SPAN_LARGE);
     740        2494 :         segment_map->pagemap[first_page] = span_pointer;
     741        2494 :         LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
     742             : 
     743             :         /* Zero-initialize the memory if requested. */
     744        2494 :         if ((flags & DSA_ALLOC_ZERO) != 0)
     745          96 :             memset(dsa_get_address(area, start_pointer), 0, size);
     746             : 
     747        2494 :         return start_pointer;
     748             :     }
     749             : 
     750             :     /* Map allocation to a size class. */
     751         596 :     if (size < lengthof(dsa_size_class_map) * DSA_SIZE_CLASS_MAP_QUANTUM)
     752             :     {
     753             :         int         mapidx;
     754             : 
     755             :         /* For smaller sizes we have a lookup table... */
     756         544 :         mapidx = ((size + DSA_SIZE_CLASS_MAP_QUANTUM - 1) /
     757         272 :                   DSA_SIZE_CLASS_MAP_QUANTUM) - 1;
     758         272 :         size_class = dsa_size_class_map[mapidx];
     759             :     }
     760             :     else
     761             :     {
     762             :         uint16      min;
     763             :         uint16      max;
     764             : 
     765             :         /* ... and for the rest we search by binary chop. */
     766         324 :         min = dsa_size_class_map[lengthof(dsa_size_class_map) - 1];
     767         324 :         max = lengthof(dsa_size_classes) - 1;
     768             : 
     769        1784 :         while (min < max)
     770             :         {
     771        1136 :             uint16      mid = (min + max) / 2;
     772        1136 :             uint16      class_size = dsa_size_classes[mid];
     773             : 
     774        1136 :             if (class_size < size)
     775         504 :                 min = mid + 1;
     776             :             else
     777         632 :                 max = mid;
     778             :         }
     779             : 
     780         324 :         size_class = min;
     781             :     }
     782             :     Assert(size <= dsa_size_classes[size_class]);
     783             :     Assert(size_class == 0 || size > dsa_size_classes[size_class - 1]);
     784             : 
     785             :     /* Attempt to allocate an object from the appropriate pool. */
     786         596 :     result = alloc_object(area, size_class);
     787             : 
     788             :     /* Check for failure to allocate. */
     789         596 :     if (!DsaPointerIsValid(result))
     790             :     {
     791             :         /* Raise error unless asked not to. */
     792           0 :         if ((flags & DSA_ALLOC_NO_OOM) == 0)
     793             :         {
     794           0 :             ereport(ERROR,
     795             :                     (errcode(ERRCODE_OUT_OF_MEMORY),
     796             :                      errmsg("out of memory"),
     797             :                      errdetail("Failed on DSA request of size %zu.", size)));
     798             :         }
     799           0 :         return InvalidDsaPointer;
     800             :     }
     801             : 
     802             :     /* Zero-initialize the memory if requested. */
     803         596 :     if ((flags & DSA_ALLOC_ZERO) != 0)
     804         284 :         memset(dsa_get_address(area, result), 0, size);
     805             : 
     806         596 :     return result;
     807             : }
     808             : 
     809             : /*
     810             :  * Free memory obtained with dsa_allocate.
     811             :  */
     812             : void
     813        5304 : dsa_free(dsa_area *area, dsa_pointer dp)
     814             : {
     815             :     dsa_segment_map *segment_map;
     816             :     int         pageno;
     817             :     dsa_pointer span_pointer;
     818             :     dsa_area_span *span;
     819             :     char       *superblock;
     820             :     char       *object;
     821             :     size_t      size;
     822             :     int         size_class;
     823             : 
     824             :     /* Make sure we don't have a stale segment in the slot 'dp' refers to. */
     825        5304 :     check_for_freed_segments(area);
     826             : 
     827             :     /* Locate the object, span and pool. */
     828        5304 :     segment_map = get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(dp));
     829        5304 :     pageno = DSA_EXTRACT_OFFSET(dp) / FPM_PAGE_SIZE;
     830        5304 :     span_pointer = segment_map->pagemap[pageno];
     831        5304 :     span = dsa_get_address(area, span_pointer);
     832        5304 :     superblock = dsa_get_address(area, span->start);
     833        5304 :     object = dsa_get_address(area, dp);
     834        5304 :     size_class = span->size_class;
     835        5304 :     size = dsa_size_classes[size_class];
     836             : 
     837             :     /*
     838             :      * Special case for large objects that live in a special span: we return
     839             :      * those pages directly to the free page manager and free the span.
     840             :      */
     841        5304 :     if (span->size_class == DSA_SCLASS_SPAN_LARGE)
     842             :     {
     843             : 
     844             : #ifdef CLOBBER_FREED_MEMORY
     845             :         memset(object, 0x7f, span->npages * FPM_PAGE_SIZE);
     846             : #endif
     847             : 
     848             :         /* Give pages back to free page manager. */
     849        2486 :         LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     850        4972 :         FreePageManagerPut(segment_map->fpm,
     851        2486 :                            DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
     852             :                            span->npages);
     853        2486 :         LWLockRelease(DSA_AREA_LOCK(area));
     854             :         /* Unlink span. */
     855        2486 :         LWLockAcquire(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE),
     856             :                       LW_EXCLUSIVE);
     857        2486 :         unlink_span(area, span);
     858        2486 :         LWLockRelease(DSA_SCLASS_LOCK(area, DSA_SCLASS_SPAN_LARGE));
     859             :         /* Free the span object so it can be reused. */
     860        2486 :         dsa_free(area, span_pointer);
     861        2486 :         return;
     862             :     }
     863             : 
     864             : #ifdef CLOBBER_FREED_MEMORY
     865             :     memset(object, 0x7f, size);
     866             : #endif
     867             : 
     868        2818 :     LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
     869             : 
     870             :     /* Put the object on the span's freelist. */
     871             :     Assert(object >= superblock);
     872             :     Assert(object < superblock + DSA_SUPERBLOCK_SIZE);
     873             :     Assert((object - superblock) % size == 0);
     874        2818 :     NextFreeObjectIndex(object) = span->firstfree;
     875        2818 :     span->firstfree = (object - superblock) / size;
     876        2818 :     ++span->nallocatable;
     877             : 
     878             :     /*
     879             :      * See if the span needs to moved to a different fullness class, or be
     880             :      * freed so its pages can be given back to the segment.
     881             :      */
     882        2818 :     if (span->nallocatable == 1 && span->fclass == DSA_FULLNESS_CLASSES - 1)
     883             :     {
     884             :         /*
     885             :          * The block was completely full and is located in the
     886             :          * highest-numbered fullness class, which is never scanned for free
     887             :          * chunks.  We must move it to the next-lower fullness class.
     888             :          */
     889           0 :         unlink_span(area, span);
     890           0 :         add_span_to_fullness_class(area, span, span_pointer,
     891             :                                    DSA_FULLNESS_CLASSES - 2);
     892             : 
     893             :         /*
     894             :          * If this is the only span, and there is no active span, then we
     895             :          * should probably move this span to fullness class 1.  (Otherwise if
     896             :          * you allocate exactly all the objects in the only span, it moves to
     897             :          * class 3, then you free them all, it moves to 2, and then is given
     898             :          * back, leaving no active span).
     899             :          */
     900             :     }
     901        3110 :     else if (span->nallocatable == span->nmax &&
     902         584 :              (span->fclass != 1 || span->prevspan != InvalidDsaPointer))
     903             :     {
     904             :         /*
     905             :          * This entire block is free, and it's not the active block for this
     906             :          * size class.  Return the memory to the free page manager. We don't
     907             :          * do this for the active block to prevent hysteresis: if we
     908             :          * repeatedly allocate and free the only chunk in the active block, it
     909             :          * will be very inefficient if we deallocate and reallocate the block
     910             :          * every time.
     911             :          */
     912           0 :         destroy_superblock(area, span_pointer);
     913             :     }
     914             : 
     915        2818 :     LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
     916             : }
     917             : 
     918             : /*
     919             :  * Obtain a backend-local address for a dsa_pointer.  'dp' must point to
     920             :  * memory allocated by the given area (possibly in another process) that
     921             :  * hasn't yet been freed.  This may cause a segment to be mapped into the
     922             :  * current process if required, and may cause freed segments to be unmapped.
     923             :  */
     924             : void *
     925     3236174 : dsa_get_address(dsa_area *area, dsa_pointer dp)
     926             : {
     927             :     dsa_segment_index index;
     928             :     size_t      offset;
     929             : 
     930             :     /* Convert InvalidDsaPointer to NULL. */
     931     3236174 :     if (!DsaPointerIsValid(dp))
     932     1200096 :         return NULL;
     933             : 
     934             :     /* Process any requests to detach from freed segments. */
     935     2036078 :     check_for_freed_segments(area);
     936             : 
     937             :     /* Break the dsa_pointer into its components. */
     938     2036078 :     index = DSA_EXTRACT_SEGMENT_NUMBER(dp);
     939     2036078 :     offset = DSA_EXTRACT_OFFSET(dp);
     940             :     Assert(index < DSA_MAX_SEGMENTS);
     941             : 
     942             :     /* Check if we need to cause this segment to be mapped in. */
     943     2036078 :     if (unlikely(area->segment_maps[index].mapped_address == NULL))
     944             :     {
     945             :         /* Call for effect (we don't need the result). */
     946         364 :         get_segment_by_index(area, index);
     947             :     }
     948             : 
     949     2036078 :     return area->segment_maps[index].mapped_address + offset;
     950             : }
     951             : 
     952             : /*
     953             :  * Pin this area, so that it will continue to exist even if all backends
     954             :  * detach from it.  In that case, the area can still be reattached to if a
     955             :  * handle has been recorded somewhere.
     956             :  */
     957             : void
     958           0 : dsa_pin(dsa_area *area)
     959             : {
     960           0 :     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     961           0 :     if (area->control->pinned)
     962             :     {
     963           0 :         LWLockRelease(DSA_AREA_LOCK(area));
     964           0 :         elog(ERROR, "dsa_area already pinned");
     965             :     }
     966           0 :     area->control->pinned = true;
     967           0 :     ++area->control->refcnt;
     968           0 :     LWLockRelease(DSA_AREA_LOCK(area));
     969           0 : }
     970             : 
     971             : /*
     972             :  * Undo the effects of dsa_pin, so that the given area can be freed when no
     973             :  * backends are attached to it.  May be called only if dsa_pin has been
     974             :  * called.
     975             :  */
     976             : void
     977           0 : dsa_unpin(dsa_area *area)
     978             : {
     979           0 :     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
     980             :     Assert(area->control->refcnt > 1);
     981           0 :     if (!area->control->pinned)
     982             :     {
     983           0 :         LWLockRelease(DSA_AREA_LOCK(area));
     984           0 :         elog(ERROR, "dsa_area not pinned");
     985             :     }
     986           0 :     area->control->pinned = false;
     987           0 :     --area->control->refcnt;
     988           0 :     LWLockRelease(DSA_AREA_LOCK(area));
     989           0 : }
     990             : 
     991             : /*
     992             :  * Set the total size limit for this area.  This limit is checked whenever new
     993             :  * segments need to be allocated from the operating system.  If the new size
     994             :  * limit is already exceeded, this has no immediate effect.
     995             :  *
     996             :  * Note that the total virtual memory usage may be temporarily larger than
     997             :  * this limit when segments have been freed, but not yet detached by all
     998             :  * backends that have attached to them.
     999             :  */
    1000             : void
    1001           0 : dsa_set_size_limit(dsa_area *area, size_t limit)
    1002             : {
    1003           0 :     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    1004           0 :     area->control->max_total_segment_size = limit;
    1005           0 :     LWLockRelease(DSA_AREA_LOCK(area));
    1006           0 : }
    1007             : 
    1008             : /*
    1009             :  * Aggressively free all spare memory in the hope of returning DSM segments to
    1010             :  * the operating system.
    1011             :  */
    1012             : void
    1013           0 : dsa_trim(dsa_area *area)
    1014             : {
    1015             :     int         size_class;
    1016             : 
    1017             :     /*
    1018             :      * Trim in reverse pool order so we get to the spans-of-spans last, just
    1019             :      * in case any become entirely free while processing all the other pools.
    1020             :      */
    1021           0 :     for (size_class = DSA_NUM_SIZE_CLASSES - 1; size_class >= 0; --size_class)
    1022             :     {
    1023           0 :         dsa_area_pool *pool = &area->control->pools[size_class];
    1024             :         dsa_pointer span_pointer;
    1025             : 
    1026           0 :         if (size_class == DSA_SCLASS_SPAN_LARGE)
    1027             :         {
    1028             :             /* Large object frees give back segments aggressively already. */
    1029           0 :             continue;
    1030             :         }
    1031             : 
    1032             :         /*
    1033             :          * Search fullness class 1 only.  That is where we expect to find an
    1034             :          * entirely empty superblock (entirely empty superblocks in other
    1035             :          * fullness classes are returned to the free page map by dsa_free).
    1036             :          */
    1037           0 :         LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
    1038           0 :         span_pointer = pool->spans[1];
    1039           0 :         while (DsaPointerIsValid(span_pointer))
    1040             :         {
    1041           0 :             dsa_area_span *span = dsa_get_address(area, span_pointer);
    1042           0 :             dsa_pointer next = span->nextspan;
    1043             : 
    1044           0 :             if (span->nallocatable == span->nmax)
    1045           0 :                 destroy_superblock(area, span_pointer);
    1046             : 
    1047           0 :             span_pointer = next;
    1048             :         }
    1049           0 :         LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
    1050             :     }
    1051           0 : }
    1052             : 
    1053             : /*
    1054             :  * Print out debugging information about the internal state of the shared
    1055             :  * memory area.
    1056             :  */
    1057             : void
    1058           0 : dsa_dump(dsa_area *area)
    1059             : {
    1060             :     size_t      i,
    1061             :                 j;
    1062             : 
    1063             :     /*
    1064             :      * Note: This gives an inconsistent snapshot as it acquires and releases
    1065             :      * individual locks as it goes...
    1066             :      */
    1067             : 
    1068           0 :     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    1069           0 :     check_for_freed_segments_locked(area);
    1070           0 :     fprintf(stderr, "dsa_area handle %x:\n", area->control->handle);
    1071           0 :     fprintf(stderr, "  max_total_segment_size: %zu\n",
    1072           0 :             area->control->max_total_segment_size);
    1073           0 :     fprintf(stderr, "  total_segment_size: %zu\n",
    1074           0 :             area->control->total_segment_size);
    1075           0 :     fprintf(stderr, "  refcnt: %d\n", area->control->refcnt);
    1076           0 :     fprintf(stderr, "  pinned: %c\n", area->control->pinned ? 't' : 'f');
    1077           0 :     fprintf(stderr, "  segment bins:\n");
    1078           0 :     for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
    1079             :     {
    1080           0 :         if (area->control->segment_bins[i] != DSA_SEGMENT_INDEX_NONE)
    1081             :         {
    1082             :             dsa_segment_index segment_index;
    1083             : 
    1084           0 :             fprintf(stderr,
    1085             :                     "    segment bin %zu (at least %d contiguous pages free):\n",
    1086           0 :                     i, 1 << (i - 1));
    1087           0 :             segment_index = area->control->segment_bins[i];
    1088           0 :             while (segment_index != DSA_SEGMENT_INDEX_NONE)
    1089             :             {
    1090             :                 dsa_segment_map *segment_map;
    1091             : 
    1092           0 :                 segment_map =
    1093             :                     get_segment_by_index(area, segment_index);
    1094             : 
    1095           0 :                 fprintf(stderr,
    1096             :                         "      segment index %zu, usable_pages = %zu, "
    1097             :                         "contiguous_pages = %zu, mapped at %p\n",
    1098             :                         segment_index,
    1099           0 :                         segment_map->header->usable_pages,
    1100           0 :                         fpm_largest(segment_map->fpm),
    1101             :                         segment_map->mapped_address);
    1102           0 :                 segment_index = segment_map->header->next;
    1103             :             }
    1104             :         }
    1105             :     }
    1106           0 :     LWLockRelease(DSA_AREA_LOCK(area));
    1107             : 
    1108           0 :     fprintf(stderr, "  pools:\n");
    1109           0 :     for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
    1110             :     {
    1111           0 :         bool        found = false;
    1112             : 
    1113           0 :         LWLockAcquire(DSA_SCLASS_LOCK(area, i), LW_EXCLUSIVE);
    1114           0 :         for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
    1115           0 :             if (DsaPointerIsValid(area->control->pools[i].spans[j]))
    1116           0 :                 found = true;
    1117           0 :         if (found)
    1118             :         {
    1119           0 :             if (i == DSA_SCLASS_BLOCK_OF_SPANS)
    1120           0 :                 fprintf(stderr, "    pool for blocks of span objects:\n");
    1121           0 :             else if (i == DSA_SCLASS_SPAN_LARGE)
    1122           0 :                 fprintf(stderr, "    pool for large object spans:\n");
    1123             :             else
    1124           0 :                 fprintf(stderr,
    1125             :                         "    pool for size class %zu (object size %hu bytes):\n",
    1126           0 :                         i, dsa_size_classes[i]);
    1127           0 :             for (j = 0; j < DSA_FULLNESS_CLASSES; ++j)
    1128             :             {
    1129           0 :                 if (!DsaPointerIsValid(area->control->pools[i].spans[j]))
    1130           0 :                     fprintf(stderr, "      fullness class %zu is empty\n", j);
    1131             :                 else
    1132             :                 {
    1133           0 :                     dsa_pointer span_pointer = area->control->pools[i].spans[j];
    1134             : 
    1135           0 :                     fprintf(stderr, "      fullness class %zu:\n", j);
    1136           0 :                     while (DsaPointerIsValid(span_pointer))
    1137             :                     {
    1138             :                         dsa_area_span *span;
    1139             : 
    1140           0 :                         span = dsa_get_address(area, span_pointer);
    1141           0 :                         fprintf(stderr,
    1142             :                                 "        span descriptor at "
    1143             :                                 DSA_POINTER_FORMAT ", superblock at "
    1144             :                                 DSA_POINTER_FORMAT
    1145             :                                 ", pages = %zu, objects free = %hu/%hu\n",
    1146             :                                 span_pointer, span->start, span->npages,
    1147           0 :                                 span->nallocatable, span->nmax);
    1148           0 :                         span_pointer = span->nextspan;
    1149             :                     }
    1150             :                 }
    1151             :             }
    1152             :         }
    1153           0 :         LWLockRelease(DSA_SCLASS_LOCK(area, i));
    1154             :     }
    1155           0 : }
    1156             : 
    1157             : /*
    1158             :  * Return the smallest size that you can successfully provide to
    1159             :  * dsa_create_in_place.
    1160             :  */
    1161             : size_t
    1162         808 : dsa_minimum_size(void)
    1163             : {
    1164             :     size_t      size;
    1165         808 :     int         pages = 0;
    1166             : 
    1167         808 :     size = MAXALIGN(sizeof(dsa_area_control)) +
    1168             :         MAXALIGN(sizeof(FreePageManager));
    1169             : 
    1170             :     /* Figure out how many pages we need, including the page map... */
    1171        3232 :     while (((size + FPM_PAGE_SIZE - 1) / FPM_PAGE_SIZE) > pages)
    1172             :     {
    1173        1616 :         ++pages;
    1174        1616 :         size += sizeof(dsa_pointer);
    1175             :     }
    1176             : 
    1177         808 :     return pages * FPM_PAGE_SIZE;
    1178             : }
    1179             : 
    1180             : /*
    1181             :  * Workhorse function for dsa_create and dsa_create_in_place.
    1182             :  */
    1183             : static dsa_area *
    1184         420 : create_internal(void *place, size_t size,
    1185             :                 int tranche_id,
    1186             :                 dsm_handle control_handle,
    1187             :                 dsm_segment *control_segment)
    1188             : {
    1189             :     dsa_area_control *control;
    1190             :     dsa_area   *area;
    1191             :     dsa_segment_map *segment_map;
    1192             :     size_t      usable_pages;
    1193             :     size_t      total_pages;
    1194             :     size_t      metadata_bytes;
    1195             :     int         i;
    1196             : 
    1197             :     /* Sanity check on the space we have to work in. */
    1198         420 :     if (size < dsa_minimum_size())
    1199           0 :         elog(ERROR, "dsa_area space must be at least %zu, but %zu provided",
    1200             :              dsa_minimum_size(), size);
    1201             : 
    1202             :     /* Now figure out how much space is usable */
    1203         420 :     total_pages = size / FPM_PAGE_SIZE;
    1204         420 :     metadata_bytes =
    1205             :         MAXALIGN(sizeof(dsa_area_control)) +
    1206         420 :         MAXALIGN(sizeof(FreePageManager)) +
    1207             :         total_pages * sizeof(dsa_pointer);
    1208             :     /* Add padding up to next page boundary. */
    1209         420 :     if (metadata_bytes % FPM_PAGE_SIZE != 0)
    1210         420 :         metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
    1211             :     Assert(metadata_bytes <= size);
    1212         420 :     usable_pages = (size - metadata_bytes) / FPM_PAGE_SIZE;
    1213             : 
    1214             :     /*
    1215             :      * Initialize the dsa_area_control object located at the start of the
    1216             :      * space.
    1217             :      */
    1218         420 :     control = (dsa_area_control *) place;
    1219         420 :     control->segment_header.magic =
    1220         420 :         DSA_SEGMENT_HEADER_MAGIC ^ control_handle ^ 0;
    1221         420 :     control->segment_header.next = DSA_SEGMENT_INDEX_NONE;
    1222         420 :     control->segment_header.prev = DSA_SEGMENT_INDEX_NONE;
    1223         420 :     control->segment_header.usable_pages = usable_pages;
    1224         420 :     control->segment_header.freed = false;
    1225         420 :     control->segment_header.size = DSA_INITIAL_SEGMENT_SIZE;
    1226         420 :     control->handle = control_handle;
    1227         420 :     control->max_total_segment_size = (size_t) -1;
    1228         420 :     control->total_segment_size = size;
    1229         420 :     memset(&control->segment_handles[0], 0,
    1230             :            sizeof(dsm_handle) * DSA_MAX_SEGMENTS);
    1231         420 :     control->segment_handles[0] = control_handle;
    1232        7140 :     for (i = 0; i < DSA_NUM_SEGMENT_BINS; ++i)
    1233        6720 :         control->segment_bins[i] = DSA_SEGMENT_INDEX_NONE;
    1234         420 :     control->high_segment_index = 0;
    1235         420 :     control->refcnt = 1;
    1236         420 :     control->freed_segment_counter = 0;
    1237         420 :     control->lwlock_tranche_id = tranche_id;
    1238             : 
    1239             :     /*
    1240             :      * Create the dsa_area object that this backend will use to access the
    1241             :      * area.  Other backends will need to obtain their own dsa_area object by
    1242             :      * attaching.
    1243             :      */
    1244         420 :     area = palloc(sizeof(dsa_area));
    1245         420 :     area->control = control;
    1246         420 :     area->mapping_pinned = false;
    1247         420 :     memset(area->segment_maps, 0, sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
    1248         420 :     area->high_segment_index = 0;
    1249         420 :     area->freed_segment_counter = 0;
    1250         420 :     LWLockInitialize(&control->lock, control->lwlock_tranche_id);
    1251       16380 :     for (i = 0; i < DSA_NUM_SIZE_CLASSES; ++i)
    1252       15960 :         LWLockInitialize(DSA_SCLASS_LOCK(area, i),
    1253             :                          control->lwlock_tranche_id);
    1254             : 
    1255             :     /* Set up the segment map for this process's mapping. */
    1256         420 :     segment_map = &area->segment_maps[0];
    1257         420 :     segment_map->segment = control_segment;
    1258         420 :     segment_map->mapped_address = place;
    1259         420 :     segment_map->header = (dsa_segment_header *) place;
    1260         420 :     segment_map->fpm = (FreePageManager *)
    1261         420 :         (segment_map->mapped_address +
    1262             :          MAXALIGN(sizeof(dsa_area_control)));
    1263         420 :     segment_map->pagemap = (dsa_pointer *)
    1264         420 :         (segment_map->mapped_address +
    1265         420 :          MAXALIGN(sizeof(dsa_area_control)) +
    1266             :          MAXALIGN(sizeof(FreePageManager)));
    1267             : 
    1268             :     /* Set up the free page map. */
    1269         420 :     FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
    1270             :     /* There can be 0 usable pages if size is dsa_minimum_size(). */
    1271             : 
    1272         420 :     if (usable_pages > 0)
    1273          32 :         FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
    1274             :                            usable_pages);
    1275             : 
    1276             :     /* Put this segment into the appropriate bin. */
    1277         420 :     control->segment_bins[contiguous_pages_to_segment_bin(usable_pages)] = 0;
    1278         420 :     segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
    1279             : 
    1280         420 :     return area;
    1281             : }
    1282             : 
    1283             : /*
    1284             :  * Workhorse function for dsa_attach and dsa_attach_in_place.
    1285             :  */
    1286             : static dsa_area *
    1287        3068 : attach_internal(void *place, dsm_segment *segment, dsa_handle handle)
    1288             : {
    1289             :     dsa_area_control *control;
    1290             :     dsa_area   *area;
    1291             :     dsa_segment_map *segment_map;
    1292             : 
    1293        3068 :     control = (dsa_area_control *) place;
    1294             :     Assert(control->handle == handle);
    1295             :     Assert(control->segment_handles[0] == handle);
    1296             :     Assert(control->segment_header.magic ==
    1297             :            (DSA_SEGMENT_HEADER_MAGIC ^ handle ^ 0));
    1298             : 
    1299             :     /* Build the backend-local area object. */
    1300        3068 :     area = palloc(sizeof(dsa_area));
    1301        3068 :     area->control = control;
    1302        3068 :     area->mapping_pinned = false;
    1303        3068 :     memset(&area->segment_maps[0], 0,
    1304             :            sizeof(dsa_segment_map) * DSA_MAX_SEGMENTS);
    1305        3068 :     area->high_segment_index = 0;
    1306             : 
    1307             :     /* Set up the segment map for this process's mapping. */
    1308        3068 :     segment_map = &area->segment_maps[0];
    1309        3068 :     segment_map->segment = segment; /* NULL for in-place */
    1310        3068 :     segment_map->mapped_address = place;
    1311        3068 :     segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
    1312        3068 :     segment_map->fpm = (FreePageManager *)
    1313        3068 :         (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)));
    1314        3068 :     segment_map->pagemap = (dsa_pointer *)
    1315        3068 :         (segment_map->mapped_address + MAXALIGN(sizeof(dsa_area_control)) +
    1316             :          MAXALIGN(sizeof(FreePageManager)));
    1317             : 
    1318             :     /* Bump the reference count. */
    1319        3068 :     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    1320        3068 :     if (control->refcnt == 0)
    1321             :     {
    1322             :         /* We can't attach to a DSA area that has already been destroyed. */
    1323           0 :         ereport(ERROR,
    1324             :                 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
    1325             :                  errmsg("could not attach to dynamic shared area")));
    1326             :     }
    1327        3068 :     ++control->refcnt;
    1328        3068 :     area->freed_segment_counter = area->control->freed_segment_counter;
    1329        3068 :     LWLockRelease(DSA_AREA_LOCK(area));
    1330             : 
    1331        3068 :     return area;
    1332             : }
    1333             : 
    1334             : /*
    1335             :  * Add a new span to fullness class 1 of the indicated pool.
    1336             :  */
    1337             : static void
    1338        2862 : init_span(dsa_area *area,
    1339             :           dsa_pointer span_pointer,
    1340             :           dsa_area_pool *pool, dsa_pointer start, size_t npages,
    1341             :           uint16 size_class)
    1342             : {
    1343        2862 :     dsa_area_span *span = dsa_get_address(area, span_pointer);
    1344        2862 :     size_t      obsize = dsa_size_classes[size_class];
    1345             : 
    1346             :     /*
    1347             :      * The per-pool lock must be held because we manipulate the span list for
    1348             :      * this pool.
    1349             :      */
    1350             :     Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
    1351             : 
    1352             :     /* Push this span onto the front of the span list for fullness class 1. */
    1353        2862 :     if (DsaPointerIsValid(pool->spans[1]))
    1354             :     {
    1355        2262 :         dsa_area_span *head = (dsa_area_span *)
    1356        2262 :         dsa_get_address(area, pool->spans[1]);
    1357             : 
    1358        2262 :         head->prevspan = span_pointer;
    1359             :     }
    1360        2862 :     span->pool = DsaAreaPoolToDsaPointer(area, pool);
    1361        2862 :     span->nextspan = pool->spans[1];
    1362        2862 :     span->prevspan = InvalidDsaPointer;
    1363        2862 :     pool->spans[1] = span_pointer;
    1364             : 
    1365        2862 :     span->start = start;
    1366        2862 :     span->npages = npages;
    1367        2862 :     span->size_class = size_class;
    1368        2862 :     span->ninitialized = 0;
    1369        2862 :     if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
    1370             :     {
    1371             :         /*
    1372             :          * A block-of-spans contains its own descriptor, so mark one object as
    1373             :          * initialized and reduce the count of allocatable objects by one.
    1374             :          * Doing this here has the side effect of also reducing nmax by one,
    1375             :          * which is important to make sure we free this object at the correct
    1376             :          * time.
    1377             :          */
    1378         112 :         span->ninitialized = 1;
    1379         112 :         span->nallocatable = FPM_PAGE_SIZE / obsize - 1;
    1380             :     }
    1381        2750 :     else if (size_class != DSA_SCLASS_SPAN_LARGE)
    1382         256 :         span->nallocatable = DSA_SUPERBLOCK_SIZE / obsize;
    1383        2862 :     span->firstfree = DSA_SPAN_NOTHING_FREE;
    1384        2862 :     span->nmax = span->nallocatable;
    1385        2862 :     span->fclass = 1;
    1386        2862 : }
    1387             : 
    1388             : /*
    1389             :  * Transfer the first span in one fullness class to the head of another
    1390             :  * fullness class.
    1391             :  */
    1392             : static bool
    1393         736 : transfer_first_span(dsa_area *area,
    1394             :                     dsa_area_pool *pool, int fromclass, int toclass)
    1395             : {
    1396             :     dsa_pointer span_pointer;
    1397             :     dsa_area_span *span;
    1398             :     dsa_area_span *nextspan;
    1399             : 
    1400             :     /* Can't do it if source list is empty. */
    1401         736 :     span_pointer = pool->spans[fromclass];
    1402         736 :     if (!DsaPointerIsValid(span_pointer))
    1403         736 :         return false;
    1404             : 
    1405             :     /* Remove span from head of source list. */
    1406           0 :     span = dsa_get_address(area, span_pointer);
    1407           0 :     pool->spans[fromclass] = span->nextspan;
    1408           0 :     if (DsaPointerIsValid(span->nextspan))
    1409             :     {
    1410           0 :         nextspan = (dsa_area_span *)
    1411           0 :             dsa_get_address(area, span->nextspan);
    1412           0 :         nextspan->prevspan = InvalidDsaPointer;
    1413             :     }
    1414             : 
    1415             :     /* Add span to head of target list. */
    1416           0 :     span->nextspan = pool->spans[toclass];
    1417           0 :     pool->spans[toclass] = span_pointer;
    1418           0 :     if (DsaPointerIsValid(span->nextspan))
    1419             :     {
    1420           0 :         nextspan = (dsa_area_span *)
    1421           0 :             dsa_get_address(area, span->nextspan);
    1422           0 :         nextspan->prevspan = span_pointer;
    1423             :     }
    1424           0 :     span->fclass = toclass;
    1425             : 
    1426           0 :     return true;
    1427             : }
    1428             : 
    1429             : /*
    1430             :  * Allocate one object of the requested size class from the given area.
    1431             :  */
    1432             : static inline dsa_pointer
    1433        3346 : alloc_object(dsa_area *area, int size_class)
    1434             : {
    1435        3346 :     dsa_area_pool *pool = &area->control->pools[size_class];
    1436             :     dsa_area_span *span;
    1437             :     dsa_pointer block;
    1438             :     dsa_pointer result;
    1439             :     char       *object;
    1440             :     size_t      size;
    1441             : 
    1442             :     /*
    1443             :      * Even though ensure_active_superblock can in turn call alloc_object if
    1444             :      * it needs to allocate a new span, that's always from a different pool,
    1445             :      * and the order of lock acquisition is always the same, so it's OK that
    1446             :      * we hold this lock for the duration of this function.
    1447             :      */
    1448             :     Assert(!LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
    1449        3346 :     LWLockAcquire(DSA_SCLASS_LOCK(area, size_class), LW_EXCLUSIVE);
    1450             : 
    1451             :     /*
    1452             :      * If there's no active superblock, we must successfully obtain one or
    1453             :      * fail the request.
    1454             :      */
    1455        3714 :     if (!DsaPointerIsValid(pool->spans[1]) &&
    1456         368 :         !ensure_active_superblock(area, pool, size_class))
    1457             :     {
    1458           0 :         result = InvalidDsaPointer;
    1459             :     }
    1460             :     else
    1461             :     {
    1462             :         /*
    1463             :          * There should be a block in fullness class 1 at this point, and it
    1464             :          * should never be completely full.  Thus we can either pop an object
    1465             :          * from the free list or, failing that, initialize a new object.
    1466             :          */
    1467             :         Assert(DsaPointerIsValid(pool->spans[1]));
    1468        3346 :         span = (dsa_area_span *)
    1469        3346 :             dsa_get_address(area, pool->spans[1]);
    1470             :         Assert(span->nallocatable > 0);
    1471        3346 :         block = span->start;
    1472             :         Assert(size_class < DSA_NUM_SIZE_CLASSES);
    1473        3346 :         size = dsa_size_classes[size_class];
    1474        3346 :         if (span->firstfree != DSA_SPAN_NOTHING_FREE)
    1475             :         {
    1476        1780 :             result = block + span->firstfree * size;
    1477        1780 :             object = dsa_get_address(area, result);
    1478        1780 :             span->firstfree = NextFreeObjectIndex(object);
    1479             :         }
    1480             :         else
    1481             :         {
    1482        1566 :             result = block + span->ninitialized * size;
    1483        1566 :             ++span->ninitialized;
    1484             :         }
    1485        3346 :         --span->nallocatable;
    1486             : 
    1487             :         /* If it's now full, move it to the highest-numbered fullness class. */
    1488        3346 :         if (span->nallocatable == 0)
    1489           0 :             transfer_first_span(area, pool, 1, DSA_FULLNESS_CLASSES - 1);
    1490             :     }
    1491             : 
    1492             :     Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
    1493        3346 :     LWLockRelease(DSA_SCLASS_LOCK(area, size_class));
    1494             : 
    1495        3346 :     return result;
    1496             : }
    1497             : 
    1498             : /*
    1499             :  * Ensure an active (i.e. fullness class 1) superblock, unless all existing
    1500             :  * superblocks are completely full and no more can be allocated.
    1501             :  *
    1502             :  * Fullness classes K of 0..N are loosely intended to represent blocks whose
    1503             :  * utilization percentage is at least K/N, but we only enforce this rigorously
    1504             :  * for the highest-numbered fullness class, which always contains exactly
    1505             :  * those blocks that are completely full.  It's otherwise acceptable for a
    1506             :  * block to be in a higher-numbered fullness class than the one to which it
    1507             :  * logically belongs.  In addition, the active block, which is always the
    1508             :  * first block in fullness class 1, is permitted to have a higher allocation
    1509             :  * percentage than would normally be allowable for that fullness class; we
    1510             :  * don't move it until it's completely full, and then it goes to the
    1511             :  * highest-numbered fullness class.
    1512             :  *
    1513             :  * It might seem odd that the active block is the head of fullness class 1
    1514             :  * rather than fullness class 0, but experience with other allocators has
    1515             :  * shown that it's usually better to allocate from a block that's moderately
    1516             :  * full rather than one that's nearly empty.  Insofar as is reasonably
    1517             :  * possible, we want to avoid performing new allocations in a block that would
    1518             :  * otherwise become empty soon.
    1519             :  */
    1520             : static bool
    1521         368 : ensure_active_superblock(dsa_area *area, dsa_area_pool *pool,
    1522             :                          int size_class)
    1523             : {
    1524             :     dsa_pointer span_pointer;
    1525             :     dsa_pointer start_pointer;
    1526         368 :     size_t      obsize = dsa_size_classes[size_class];
    1527             :     size_t      nmax;
    1528             :     int         fclass;
    1529         368 :     size_t      npages = 1;
    1530             :     size_t      first_page;
    1531             :     size_t      i;
    1532             :     dsa_segment_map *segment_map;
    1533             : 
    1534             :     Assert(LWLockHeldByMe(DSA_SCLASS_LOCK(area, size_class)));
    1535             : 
    1536             :     /*
    1537             :      * Compute the number of objects that will fit in a block of this size
    1538             :      * class.  Span-of-spans blocks are just a single page, and the first
    1539             :      * object isn't available for use because it describes the block-of-spans
    1540             :      * itself.
    1541             :      */
    1542         368 :     if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
    1543         112 :         nmax = FPM_PAGE_SIZE / obsize - 1;
    1544             :     else
    1545         256 :         nmax = DSA_SUPERBLOCK_SIZE / obsize;
    1546             : 
    1547             :     /*
    1548             :      * If fullness class 1 is empty, try to find a span to put in it by
    1549             :      * scanning higher-numbered fullness classes (excluding the last one,
    1550             :      * whose blocks are certain to all be completely full).
    1551             :      */
    1552         736 :     for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
    1553             :     {
    1554         368 :         span_pointer = pool->spans[fclass];
    1555             : 
    1556         736 :         while (DsaPointerIsValid(span_pointer))
    1557             :         {
    1558             :             int         tfclass;
    1559             :             dsa_area_span *span;
    1560             :             dsa_area_span *nextspan;
    1561             :             dsa_area_span *prevspan;
    1562             :             dsa_pointer next_span_pointer;
    1563             : 
    1564           0 :             span = (dsa_area_span *)
    1565             :                 dsa_get_address(area, span_pointer);
    1566           0 :             next_span_pointer = span->nextspan;
    1567             : 
    1568             :             /* Figure out what fullness class should contain this span. */
    1569           0 :             tfclass = (nmax - span->nallocatable)
    1570           0 :                 * (DSA_FULLNESS_CLASSES - 1) / nmax;
    1571             : 
    1572             :             /* Look up next span. */
    1573           0 :             if (DsaPointerIsValid(span->nextspan))
    1574           0 :                 nextspan = (dsa_area_span *)
    1575           0 :                     dsa_get_address(area, span->nextspan);
    1576             :             else
    1577           0 :                 nextspan = NULL;
    1578             : 
    1579             :             /*
    1580             :              * If utilization has dropped enough that this now belongs in some
    1581             :              * other fullness class, move it there.
    1582             :              */
    1583           0 :             if (tfclass < fclass)
    1584             :             {
    1585             :                 /* Remove from the current fullness class list. */
    1586           0 :                 if (pool->spans[fclass] == span_pointer)
    1587             :                 {
    1588             :                     /* It was the head; remove it. */
    1589             :                     Assert(!DsaPointerIsValid(span->prevspan));
    1590           0 :                     pool->spans[fclass] = span->nextspan;
    1591           0 :                     if (nextspan != NULL)
    1592           0 :                         nextspan->prevspan = InvalidDsaPointer;
    1593             :                 }
    1594             :                 else
    1595             :                 {
    1596             :                     /* It was not the head. */
    1597             :                     Assert(DsaPointerIsValid(span->prevspan));
    1598           0 :                     prevspan = (dsa_area_span *)
    1599           0 :                         dsa_get_address(area, span->prevspan);
    1600           0 :                     prevspan->nextspan = span->nextspan;
    1601             :                 }
    1602           0 :                 if (nextspan != NULL)
    1603           0 :                     nextspan->prevspan = span->prevspan;
    1604             : 
    1605             :                 /* Push onto the head of the new fullness class list. */
    1606           0 :                 span->nextspan = pool->spans[tfclass];
    1607           0 :                 pool->spans[tfclass] = span_pointer;
    1608           0 :                 span->prevspan = InvalidDsaPointer;
    1609           0 :                 if (DsaPointerIsValid(span->nextspan))
    1610             :                 {
    1611           0 :                     nextspan = (dsa_area_span *)
    1612           0 :                         dsa_get_address(area, span->nextspan);
    1613           0 :                     nextspan->prevspan = span_pointer;
    1614             :                 }
    1615           0 :                 span->fclass = tfclass;
    1616             :             }
    1617             : 
    1618             :             /* Advance to next span on list. */
    1619           0 :             span_pointer = next_span_pointer;
    1620             :         }
    1621             : 
    1622             :         /* Stop now if we found a suitable block. */
    1623         368 :         if (DsaPointerIsValid(pool->spans[1]))
    1624           0 :             return true;
    1625             :     }
    1626             : 
    1627             :     /*
    1628             :      * If there are no blocks that properly belong in fullness class 1, pick
    1629             :      * one from some other fullness class and move it there anyway, so that we
    1630             :      * have an allocation target.  Our last choice is to transfer a block
    1631             :      * that's almost empty (and might become completely empty soon if left
    1632             :      * alone), but even that is better than failing, which is what we must do
    1633             :      * if there are no blocks at all with freespace.
    1634             :      */
    1635             :     Assert(!DsaPointerIsValid(pool->spans[1]));
    1636         736 :     for (fclass = 2; fclass < DSA_FULLNESS_CLASSES - 1; ++fclass)
    1637         368 :         if (transfer_first_span(area, pool, fclass, 1))
    1638           0 :             return true;
    1639         736 :     if (!DsaPointerIsValid(pool->spans[1]) &&
    1640         368 :         transfer_first_span(area, pool, 0, 1))
    1641           0 :         return true;
    1642             : 
    1643             :     /*
    1644             :      * We failed to find an existing span with free objects, so we need to
    1645             :      * allocate a new superblock and construct a new span to manage it.
    1646             :      *
    1647             :      * First, get a dsa_area_span object to describe the new superblock block
    1648             :      * ... unless this allocation is for a dsa_area_span object, in which case
    1649             :      * that's surely not going to work.  We handle that case by storing the
    1650             :      * span describing a block-of-spans inline.
    1651             :      */
    1652         368 :     if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
    1653             :     {
    1654         256 :         span_pointer = alloc_object(area, DSA_SCLASS_BLOCK_OF_SPANS);
    1655         256 :         if (!DsaPointerIsValid(span_pointer))
    1656           0 :             return false;
    1657         256 :         npages = DSA_PAGES_PER_SUPERBLOCK;
    1658             :     }
    1659             : 
    1660             :     /* Find or create a segment and allocate the superblock. */
    1661         368 :     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    1662         368 :     segment_map = get_best_segment(area, npages);
    1663         368 :     if (segment_map == NULL)
    1664             :     {
    1665          88 :         segment_map = make_new_segment(area, npages);
    1666          88 :         if (segment_map == NULL)
    1667             :         {
    1668           0 :             LWLockRelease(DSA_AREA_LOCK(area));
    1669           0 :             return false;
    1670             :         }
    1671             :     }
    1672         368 :     if (!FreePageManagerGet(segment_map->fpm, npages, &first_page))
    1673             :     {
    1674           0 :         LWLockRelease(DSA_AREA_LOCK(area));
    1675           0 :         if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
    1676           0 :             dsa_free(area, span_pointer);
    1677           0 :         return false;
    1678             :     }
    1679         368 :     LWLockRelease(DSA_AREA_LOCK(area));
    1680             : 
    1681             :     /* Compute the start of the superblock. */
    1682         368 :     start_pointer =
    1683         368 :         DSA_MAKE_POINTER(get_segment_index(area, segment_map),
    1684             :                          first_page * FPM_PAGE_SIZE);
    1685             : 
    1686             :     /*
    1687             :      * If this is a block-of-spans, carve the descriptor right out of the
    1688             :      * allocated space.
    1689             :      */
    1690         368 :     if (size_class == DSA_SCLASS_BLOCK_OF_SPANS)
    1691             :     {
    1692             :         /*
    1693             :          * We have a pointer into the segment.  We need to build a dsa_pointer
    1694             :          * from the segment index and offset into the segment.
    1695             :          */
    1696         112 :         span_pointer = start_pointer;
    1697             :     }
    1698             : 
    1699             :     /* Initialize span and pagemap. */
    1700         368 :     init_span(area, span_pointer, pool, start_pointer, npages, size_class);
    1701        4576 :     for (i = 0; i < npages; ++i)
    1702        4208 :         segment_map->pagemap[first_page + i] = span_pointer;
    1703             : 
    1704         368 :     return true;
    1705             : }
    1706             : 
    1707             : /*
    1708             :  * Return the segment map corresponding to a given segment index, mapping the
    1709             :  * segment in if necessary.  For internal segment book-keeping, this is called
    1710             :  * with the area lock held.  It is also called by dsa_free and dsa_get_address
    1711             :  * without any locking, relying on the fact they have a known live segment
    1712             :  * index and they always call check_for_freed_segments to ensures that any
    1713             :  * freed segment occupying the same slot is detached first.
    1714             :  */
    1715             : static dsa_segment_map *
    1716        8450 : get_segment_by_index(dsa_area *area, dsa_segment_index index)
    1717             : {
    1718        8450 :     if (unlikely(area->segment_maps[index].mapped_address == NULL))
    1719             :     {
    1720             :         dsm_handle  handle;
    1721             :         dsm_segment *segment;
    1722             :         dsa_segment_map *segment_map;
    1723             : 
    1724             :         /*
    1725             :          * If we are reached by dsa_free or dsa_get_address, there must be at
    1726             :          * least one object allocated in the referenced segment.  Otherwise,
    1727             :          * their caller has a double-free or access-after-free bug, which we
    1728             :          * have no hope of detecting.  So we know it's safe to access this
    1729             :          * array slot without holding a lock; it won't change underneath us.
    1730             :          * Furthermore, we know that we can see the latest contents of the
    1731             :          * slot, as explained in check_for_freed_segments, which those
    1732             :          * functions call before arriving here.
    1733             :          */
    1734         370 :         handle = area->control->segment_handles[index];
    1735             : 
    1736             :         /* It's an error to try to access an unused slot. */
    1737         370 :         if (handle == DSM_HANDLE_INVALID)
    1738           0 :             elog(ERROR,
    1739             :                  "dsa_area could not attach to a segment that has been freed");
    1740             : 
    1741         370 :         segment = dsm_attach(handle);
    1742         370 :         if (segment == NULL)
    1743           0 :             elog(ERROR, "dsa_area could not attach to segment");
    1744         370 :         if (area->mapping_pinned)
    1745           4 :             dsm_pin_mapping(segment);
    1746         370 :         segment_map = &area->segment_maps[index];
    1747         370 :         segment_map->segment = segment;
    1748         370 :         segment_map->mapped_address = dsm_segment_address(segment);
    1749         370 :         segment_map->header =
    1750         370 :             (dsa_segment_header *) segment_map->mapped_address;
    1751         370 :         segment_map->fpm = (FreePageManager *)
    1752         370 :             (segment_map->mapped_address +
    1753             :              MAXALIGN(sizeof(dsa_segment_header)));
    1754         370 :         segment_map->pagemap = (dsa_pointer *)
    1755         370 :             (segment_map->mapped_address +
    1756         370 :              MAXALIGN(sizeof(dsa_segment_header)) +
    1757             :              MAXALIGN(sizeof(FreePageManager)));
    1758             : 
    1759             :         /* Remember the highest index this backend has ever mapped. */
    1760         370 :         if (area->high_segment_index < index)
    1761         370 :             area->high_segment_index = index;
    1762             : 
    1763             :         Assert(segment_map->header->magic ==
    1764             :                (DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ index));
    1765             :     }
    1766             : 
    1767             :     /*
    1768             :      * Callers of dsa_get_address() and dsa_free() don't hold the area lock,
    1769             :      * but it's a bug in the calling code and undefined behavior if the
    1770             :      * address is not live (ie if the segment might possibly have been freed,
    1771             :      * they're trying to use a dangling pointer).
    1772             :      *
    1773             :      * For dsa.c code that holds the area lock to manipulate segment_bins
    1774             :      * lists, it would be a bug if we ever reach a freed segment here.  After
    1775             :      * it's marked as freed, the only thing any backend should do with it is
    1776             :      * unmap it, and it should always have done that in
    1777             :      * check_for_freed_segments_locked() before arriving here to resolve an
    1778             :      * index to a segment_map.
    1779             :      *
    1780             :      * Either way we can assert that we aren't returning a freed segment.
    1781             :      */
    1782             :     Assert(!area->segment_maps[index].header->freed);
    1783             : 
    1784        8450 :     return &area->segment_maps[index];
    1785             : }
    1786             : 
    1787             : /*
    1788             :  * Return a superblock to the free page manager.  If the underlying segment
    1789             :  * has become entirely free, then return it to the operating system.
    1790             :  *
    1791             :  * The appropriate pool lock must be held.
    1792             :  */
    1793             : static void
    1794           0 : destroy_superblock(dsa_area *area, dsa_pointer span_pointer)
    1795             : {
    1796           0 :     dsa_area_span *span = dsa_get_address(area, span_pointer);
    1797           0 :     int         size_class = span->size_class;
    1798             :     dsa_segment_map *segment_map;
    1799             : 
    1800             : 
    1801             :     /* Remove it from its fullness class list. */
    1802           0 :     unlink_span(area, span);
    1803             : 
    1804             :     /*
    1805             :      * Note: Here we acquire the area lock while we already hold a per-pool
    1806             :      * lock.  We never hold the area lock and then take a pool lock, or we
    1807             :      * could deadlock.
    1808             :      */
    1809           0 :     LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    1810           0 :     check_for_freed_segments_locked(area);
    1811           0 :     segment_map =
    1812           0 :         get_segment_by_index(area, DSA_EXTRACT_SEGMENT_NUMBER(span->start));
    1813           0 :     FreePageManagerPut(segment_map->fpm,
    1814           0 :                        DSA_EXTRACT_OFFSET(span->start) / FPM_PAGE_SIZE,
    1815             :                        span->npages);
    1816             :     /* Check if the segment is now entirely free. */
    1817           0 :     if (fpm_largest(segment_map->fpm) == segment_map->header->usable_pages)
    1818             :     {
    1819           0 :         dsa_segment_index index = get_segment_index(area, segment_map);
    1820             : 
    1821             :         /* If it's not the segment with extra control data, free it. */
    1822           0 :         if (index != 0)
    1823             :         {
    1824             :             /*
    1825             :              * Give it back to the OS, and allow other backends to detect that
    1826             :              * they need to detach.
    1827             :              */
    1828           0 :             unlink_segment(area, segment_map);
    1829           0 :             segment_map->header->freed = true;
    1830             :             Assert(area->control->total_segment_size >=
    1831             :                    segment_map->header->size);
    1832           0 :             area->control->total_segment_size -=
    1833           0 :                 segment_map->header->size;
    1834           0 :             dsm_unpin_segment(dsm_segment_handle(segment_map->segment));
    1835           0 :             dsm_detach(segment_map->segment);
    1836           0 :             area->control->segment_handles[index] = DSM_HANDLE_INVALID;
    1837           0 :             ++area->control->freed_segment_counter;
    1838           0 :             segment_map->segment = NULL;
    1839           0 :             segment_map->header = NULL;
    1840           0 :             segment_map->mapped_address = NULL;
    1841             :         }
    1842             :     }
    1843           0 :     LWLockRelease(DSA_AREA_LOCK(area));
    1844             : 
    1845             :     /*
    1846             :      * Span-of-spans blocks store the span which describes them within the
    1847             :      * block itself, so freeing the storage implicitly frees the descriptor
    1848             :      * also.  If this is a block of any other type, we need to separately free
    1849             :      * the span object also.  This recursive call to dsa_free will acquire the
    1850             :      * span pool's lock.  We can't deadlock because the acquisition order is
    1851             :      * always some other pool and then the span pool.
    1852             :      */
    1853           0 :     if (size_class != DSA_SCLASS_BLOCK_OF_SPANS)
    1854           0 :         dsa_free(area, span_pointer);
    1855           0 : }
    1856             : 
    1857             : static void
    1858        2486 : unlink_span(dsa_area *area, dsa_area_span *span)
    1859             : {
    1860        2486 :     if (DsaPointerIsValid(span->nextspan))
    1861             :     {
    1862        2018 :         dsa_area_span *next = dsa_get_address(area, span->nextspan);
    1863             : 
    1864        2018 :         next->prevspan = span->prevspan;
    1865             :     }
    1866        2486 :     if (DsaPointerIsValid(span->prevspan))
    1867             :     {
    1868        1074 :         dsa_area_span *prev = dsa_get_address(area, span->prevspan);
    1869             : 
    1870        1074 :         prev->nextspan = span->nextspan;
    1871             :     }
    1872             :     else
    1873             :     {
    1874        1412 :         dsa_area_pool *pool = dsa_get_address(area, span->pool);
    1875             : 
    1876        1412 :         pool->spans[span->fclass] = span->nextspan;
    1877             :     }
    1878        2486 : }
    1879             : 
    1880             : static void
    1881           0 : add_span_to_fullness_class(dsa_area *area, dsa_area_span *span,
    1882             :                            dsa_pointer span_pointer,
    1883             :                            int fclass)
    1884             : {
    1885           0 :     dsa_area_pool *pool = dsa_get_address(area, span->pool);
    1886             : 
    1887           0 :     if (DsaPointerIsValid(pool->spans[fclass]))
    1888             :     {
    1889           0 :         dsa_area_span *head = dsa_get_address(area,
    1890             :                                               pool->spans[fclass]);
    1891             : 
    1892           0 :         head->prevspan = span_pointer;
    1893             :     }
    1894           0 :     span->prevspan = InvalidDsaPointer;
    1895           0 :     span->nextspan = pool->spans[fclass];
    1896           0 :     pool->spans[fclass] = span_pointer;
    1897           0 :     span->fclass = fclass;
    1898           0 : }
    1899             : 
    1900             : /*
    1901             :  * Detach from an area that was either created or attached to by this process.
    1902             :  */
    1903             : void
    1904        3444 : dsa_detach(dsa_area *area)
    1905             : {
    1906             :     int         i;
    1907             : 
    1908             :     /* Detach from all segments. */
    1909        7358 :     for (i = 0; i <= area->high_segment_index; ++i)
    1910        3914 :         if (area->segment_maps[i].segment != NULL)
    1911         470 :             dsm_detach(area->segment_maps[i].segment);
    1912             : 
    1913             :     /*
    1914             :      * Note that 'detaching' (= detaching from DSM segments) doesn't include
    1915             :      * 'releasing' (= adjusting the reference count).  It would be nice to
    1916             :      * combine these operations, but client code might never get around to
    1917             :      * calling dsa_detach because of an error path, and a detach hook on any
    1918             :      * particular segment is too late to detach other segments in the area
    1919             :      * without risking a 'leak' warning in the non-error path.
    1920             :      */
    1921             : 
    1922             :     /* Free the backend-local area object. */
    1923        3444 :     pfree(area);
    1924        3444 : }
    1925             : 
    1926             : /*
    1927             :  * Unlink a segment from the bin that contains it.
    1928             :  */
    1929             : static void
    1930         168 : unlink_segment(dsa_area *area, dsa_segment_map *segment_map)
    1931             : {
    1932         168 :     if (segment_map->header->prev != DSA_SEGMENT_INDEX_NONE)
    1933             :     {
    1934             :         dsa_segment_map *prev;
    1935             : 
    1936           0 :         prev = get_segment_by_index(area, segment_map->header->prev);
    1937           0 :         prev->header->next = segment_map->header->next;
    1938             :     }
    1939             :     else
    1940             :     {
    1941             :         Assert(area->control->segment_bins[segment_map->header->bin] ==
    1942             :                get_segment_index(area, segment_map));
    1943         336 :         area->control->segment_bins[segment_map->header->bin] =
    1944         168 :             segment_map->header->next;
    1945             :     }
    1946         168 :     if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
    1947             :     {
    1948             :         dsa_segment_map *next;
    1949             : 
    1950           0 :         next = get_segment_by_index(area, segment_map->header->next);
    1951           0 :         next->header->prev = segment_map->header->prev;
    1952             :     }
    1953         168 : }
    1954             : 
    1955             : /*
    1956             :  * Find a segment that could satisfy a request for 'npages' of contiguous
    1957             :  * memory, or return NULL if none can be found.  This may involve attaching to
    1958             :  * segments that weren't previously attached so that we can query their free
    1959             :  * pages map.
    1960             :  */
    1961             : static dsa_segment_map *
    1962        2862 : get_best_segment(dsa_area *area, size_t npages)
    1963             : {
    1964             :     size_t      bin;
    1965             : 
    1966             :     Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
    1967        2862 :     check_for_freed_segments_locked(area);
    1968             : 
    1969             :     /*
    1970             :      * Start searching from the first bin that *might* have enough contiguous
    1971             :      * pages.
    1972             :      */
    1973       16936 :     for (bin = contiguous_pages_to_segment_bin(npages);
    1974             :          bin < DSA_NUM_SEGMENT_BINS;
    1975       11212 :          ++bin)
    1976             :     {
    1977             :         /*
    1978             :          * The minimum contiguous size that any segment in this bin should
    1979             :          * have.  We'll re-bin if we see segments with fewer.
    1980             :          */
    1981       13966 :         size_t      threshold = (size_t) 1 << (bin - 1);
    1982             :         dsa_segment_index segment_index;
    1983             : 
    1984             :         /* Search this bin for a segment with enough contiguous space. */
    1985       13966 :         segment_index = area->control->segment_bins[bin];
    1986       27960 :         while (segment_index != DSA_SEGMENT_INDEX_NONE)
    1987             :         {
    1988             :             dsa_segment_map *segment_map;
    1989             :             dsa_segment_index next_segment_index;
    1990             :             size_t      contiguous_pages;
    1991             : 
    1992        2782 :             segment_map = get_segment_by_index(area, segment_index);
    1993        2782 :             next_segment_index = segment_map->header->next;
    1994        2782 :             contiguous_pages = fpm_largest(segment_map->fpm);
    1995             : 
    1996             :             /* Not enough for the request, still enough for this bin. */
    1997        2782 :             if (contiguous_pages >= threshold && contiguous_pages < npages)
    1998             :             {
    1999           0 :                 segment_index = next_segment_index;
    2000           0 :                 continue;
    2001             :             }
    2002             : 
    2003             :             /* Re-bin it if it's no longer in the appropriate bin. */
    2004        2782 :             if (contiguous_pages < threshold)
    2005             :             {
    2006             :                 size_t      new_bin;
    2007             : 
    2008         168 :                 new_bin = contiguous_pages_to_segment_bin(contiguous_pages);
    2009             : 
    2010             :                 /* Remove it from its current bin. */
    2011         168 :                 unlink_segment(area, segment_map);
    2012             : 
    2013             :                 /* Push it onto the front of its new bin. */
    2014         168 :                 segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
    2015         336 :                 segment_map->header->next =
    2016         168 :                     area->control->segment_bins[new_bin];
    2017         168 :                 segment_map->header->bin = new_bin;
    2018         168 :                 area->control->segment_bins[new_bin] = segment_index;
    2019         168 :                 if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
    2020             :                 {
    2021             :                     dsa_segment_map *next;
    2022             : 
    2023           0 :                     next = get_segment_by_index(area,
    2024           0 :                                                 segment_map->header->next);
    2025             :                     Assert(next->header->bin == new_bin);
    2026           0 :                     next->header->prev = segment_index;
    2027             :                 }
    2028             : 
    2029             :                 /*
    2030             :                  * But fall through to see if it's enough to satisfy this
    2031             :                  * request anyway....
    2032             :                  */
    2033             :             }
    2034             : 
    2035             :             /* Check if we are done. */
    2036        2782 :             if (contiguous_pages >= npages)
    2037        2754 :                 return segment_map;
    2038             : 
    2039             :             /* Continue searching the same bin. */
    2040          28 :             segment_index = next_segment_index;
    2041             :         }
    2042             :     }
    2043             : 
    2044             :     /* Not found. */
    2045         108 :     return NULL;
    2046             : }
    2047             : 
    2048             : /*
    2049             :  * Create a new segment that can handle at least requested_pages.  Returns
    2050             :  * NULL if the requested total size limit or maximum allowed number of
    2051             :  * segments would be exceeded.
    2052             :  */
    2053             : static dsa_segment_map *
    2054         108 : make_new_segment(dsa_area *area, size_t requested_pages)
    2055             : {
    2056             :     dsa_segment_index new_index;
    2057             :     size_t      metadata_bytes;
    2058             :     size_t      total_size;
    2059             :     size_t      total_pages;
    2060             :     size_t      usable_pages;
    2061             :     dsa_segment_map *segment_map;
    2062             :     dsm_segment *segment;
    2063             : 
    2064             :     Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
    2065             : 
    2066             :     /* Find a segment slot that is not in use (linearly for now). */
    2067         128 :     for (new_index = 1; new_index < DSA_MAX_SEGMENTS; ++new_index)
    2068             :     {
    2069         128 :         if (area->control->segment_handles[new_index] == DSM_HANDLE_INVALID)
    2070         108 :             break;
    2071             :     }
    2072         108 :     if (new_index == DSA_MAX_SEGMENTS)
    2073           0 :         return NULL;
    2074             : 
    2075             :     /*
    2076             :      * If the total size limit is already exceeded, then we exit early and
    2077             :      * avoid arithmetic wraparound in the unsigned expressions below.
    2078             :      */
    2079         216 :     if (area->control->total_segment_size >=
    2080         108 :         area->control->max_total_segment_size)
    2081           0 :         return NULL;
    2082             : 
    2083             :     /*
    2084             :      * The size should be at least as big as requested, and at least big
    2085             :      * enough to follow a geometric series that approximately doubles the
    2086             :      * total storage each time we create a new segment.  We use geometric
    2087             :      * growth because the underlying DSM system isn't designed for large
    2088             :      * numbers of segments (otherwise we might even consider just using one
    2089             :      * DSM segment for each large allocation and for each superblock, and then
    2090             :      * we wouldn't need to use FreePageManager).
    2091             :      *
    2092             :      * We decide on a total segment size first, so that we produce tidy
    2093             :      * power-of-two sized segments.  This is a good property to have if we
    2094             :      * move to huge pages in the future.  Then we work back to the number of
    2095             :      * pages we can fit.
    2096             :      */
    2097         108 :     total_size = DSA_INITIAL_SEGMENT_SIZE *
    2098         108 :         ((size_t) 1 << (new_index / DSA_NUM_SEGMENTS_AT_EACH_SIZE));
    2099         108 :     total_size = Min(total_size, DSA_MAX_SEGMENT_SIZE);
    2100         108 :     total_size = Min(total_size,
    2101             :                      area->control->max_total_segment_size -
    2102             :                      area->control->total_segment_size);
    2103             : 
    2104         108 :     total_pages = total_size / FPM_PAGE_SIZE;
    2105         108 :     metadata_bytes =
    2106             :         MAXALIGN(sizeof(dsa_segment_header)) +
    2107         108 :         MAXALIGN(sizeof(FreePageManager)) +
    2108             :         sizeof(dsa_pointer) * total_pages;
    2109             : 
    2110             :     /* Add padding up to next page boundary. */
    2111         108 :     if (metadata_bytes % FPM_PAGE_SIZE != 0)
    2112         108 :         metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
    2113         108 :     if (total_size <= metadata_bytes)
    2114           0 :         return NULL;
    2115         108 :     usable_pages = (total_size - metadata_bytes) / FPM_PAGE_SIZE;
    2116             :     Assert(metadata_bytes + usable_pages * FPM_PAGE_SIZE <= total_size);
    2117             : 
    2118             :     /* See if that is enough... */
    2119         108 :     if (requested_pages > usable_pages)
    2120             :     {
    2121             :         /*
    2122             :          * We'll make an odd-sized segment, working forward from the requested
    2123             :          * number of pages.
    2124             :          */
    2125           0 :         usable_pages = requested_pages;
    2126           0 :         metadata_bytes =
    2127             :             MAXALIGN(sizeof(dsa_segment_header)) +
    2128           0 :             MAXALIGN(sizeof(FreePageManager)) +
    2129             :             usable_pages * sizeof(dsa_pointer);
    2130             : 
    2131             :         /* Add padding up to next page boundary. */
    2132           0 :         if (metadata_bytes % FPM_PAGE_SIZE != 0)
    2133           0 :             metadata_bytes += FPM_PAGE_SIZE - (metadata_bytes % FPM_PAGE_SIZE);
    2134           0 :         total_size = metadata_bytes + usable_pages * FPM_PAGE_SIZE;
    2135             : 
    2136             :         /* Is that too large for dsa_pointer's addressing scheme? */
    2137           0 :         if (total_size > DSA_MAX_SEGMENT_SIZE)
    2138           0 :             return NULL;
    2139             : 
    2140             :         /* Would that exceed the limit? */
    2141           0 :         if (total_size > area->control->max_total_segment_size -
    2142           0 :             area->control->total_segment_size)
    2143           0 :             return NULL;
    2144             :     }
    2145             : 
    2146             :     /* Create the segment. */
    2147         108 :     segment = dsm_create(total_size, 0);
    2148         108 :     if (segment == NULL)
    2149           0 :         return NULL;
    2150         108 :     dsm_pin_segment(segment);
    2151         108 :     if (area->mapping_pinned)
    2152           4 :         dsm_pin_mapping(segment);
    2153             : 
    2154             :     /* Store the handle in shared memory to be found by index. */
    2155         216 :     area->control->segment_handles[new_index] =
    2156         108 :         dsm_segment_handle(segment);
    2157             :     /* Track the highest segment index in the history of the area. */
    2158         108 :     if (area->control->high_segment_index < new_index)
    2159         108 :         area->control->high_segment_index = new_index;
    2160             :     /* Track the highest segment index this backend has ever mapped. */
    2161         108 :     if (area->high_segment_index < new_index)
    2162         108 :         area->high_segment_index = new_index;
    2163             :     /* Track total size of all segments. */
    2164         108 :     area->control->total_segment_size += total_size;
    2165             :     Assert(area->control->total_segment_size <=
    2166             :            area->control->max_total_segment_size);
    2167             : 
    2168             :     /* Build a segment map for this segment in this backend. */
    2169         108 :     segment_map = &area->segment_maps[new_index];
    2170         108 :     segment_map->segment = segment;
    2171         108 :     segment_map->mapped_address = dsm_segment_address(segment);
    2172         108 :     segment_map->header = (dsa_segment_header *) segment_map->mapped_address;
    2173         108 :     segment_map->fpm = (FreePageManager *)
    2174         108 :         (segment_map->mapped_address +
    2175             :          MAXALIGN(sizeof(dsa_segment_header)));
    2176         108 :     segment_map->pagemap = (dsa_pointer *)
    2177         108 :         (segment_map->mapped_address +
    2178         108 :          MAXALIGN(sizeof(dsa_segment_header)) +
    2179             :          MAXALIGN(sizeof(FreePageManager)));
    2180             : 
    2181             :     /* Set up the free page map. */
    2182         108 :     FreePageManagerInitialize(segment_map->fpm, segment_map->mapped_address);
    2183         108 :     FreePageManagerPut(segment_map->fpm, metadata_bytes / FPM_PAGE_SIZE,
    2184             :                        usable_pages);
    2185             : 
    2186             :     /* Set up the segment header and put it in the appropriate bin. */
    2187         216 :     segment_map->header->magic =
    2188         108 :         DSA_SEGMENT_HEADER_MAGIC ^ area->control->handle ^ new_index;
    2189         108 :     segment_map->header->usable_pages = usable_pages;
    2190         108 :     segment_map->header->size = total_size;
    2191         108 :     segment_map->header->bin = contiguous_pages_to_segment_bin(usable_pages);
    2192         108 :     segment_map->header->prev = DSA_SEGMENT_INDEX_NONE;
    2193         216 :     segment_map->header->next =
    2194         108 :         area->control->segment_bins[segment_map->header->bin];
    2195         108 :     segment_map->header->freed = false;
    2196         108 :     area->control->segment_bins[segment_map->header->bin] = new_index;
    2197         108 :     if (segment_map->header->next != DSA_SEGMENT_INDEX_NONE)
    2198             :     {
    2199           0 :         dsa_segment_map *next =
    2200           0 :         get_segment_by_index(area, segment_map->header->next);
    2201             : 
    2202             :         Assert(next->header->bin == segment_map->header->bin);
    2203           0 :         next->header->prev = new_index;
    2204             :     }
    2205             : 
    2206         108 :     return segment_map;
    2207             : }
    2208             : 
    2209             : /*
    2210             :  * Check if any segments have been freed by destroy_superblock, so we can
    2211             :  * detach from them in this backend.  This function is called by
    2212             :  * dsa_get_address and dsa_free to make sure that a dsa_pointer they have
    2213             :  * received can be resolved to the correct segment.
    2214             :  *
    2215             :  * The danger we want to defend against is that there could be an old segment
    2216             :  * mapped into a given slot in this backend, and the dsa_pointer they have
    2217             :  * might refer to some new segment in the same slot.  So those functions must
    2218             :  * be sure to process all instructions to detach from a freed segment that had
    2219             :  * been generated by the time this process received the dsa_pointer, before
    2220             :  * they call get_segment_by_index.
    2221             :  */
    2222             : static void
    2223     2041382 : check_for_freed_segments(dsa_area *area)
    2224             : {
    2225             :     size_t      freed_segment_counter;
    2226             : 
    2227             :     /*
    2228             :      * Any other process that has freed a segment has incremented
    2229             :      * free_segment_counter while holding an LWLock, and that must precede any
    2230             :      * backend creating a new segment in the same slot while holding an
    2231             :      * LWLock, and that must precede the creation of any dsa_pointer pointing
    2232             :      * into the new segment which might reach us here, and the caller must
    2233             :      * have sent the dsa_pointer to this process using appropriate memory
    2234             :      * synchronization (some kind of locking or atomic primitive or system
    2235             :      * call).  So all we need to do on the reading side is ask for the load of
    2236             :      * freed_segment_counter to follow the caller's load of the dsa_pointer it
    2237             :      * has, and we can be sure to detect any segments that had been freed as
    2238             :      * of the time that the dsa_pointer reached this process.
    2239             :      */
    2240     2041382 :     pg_read_barrier();
    2241     2041382 :     freed_segment_counter = area->control->freed_segment_counter;
    2242     2041382 :     if (unlikely(area->freed_segment_counter != freed_segment_counter))
    2243             :     {
    2244             :         /* Check all currently mapped segments to find what's been freed. */
    2245           0 :         LWLockAcquire(DSA_AREA_LOCK(area), LW_EXCLUSIVE);
    2246           0 :         check_for_freed_segments_locked(area);
    2247           0 :         LWLockRelease(DSA_AREA_LOCK(area));
    2248             :     }
    2249     2041382 : }
    2250             : 
    2251             : /*
    2252             :  * Workhorse for check_for_free_segments(), and also used directly in path
    2253             :  * where the area lock is already held.  This should be called after acquiring
    2254             :  * the lock but before looking up any segment by index number, to make sure we
    2255             :  * unmap any stale segments that might have previously had the same index as a
    2256             :  * current segment.
    2257             :  */
    2258             : static void
    2259        2862 : check_for_freed_segments_locked(dsa_area *area)
    2260             : {
    2261             :     size_t      freed_segment_counter;
    2262             :     int     i;
    2263             : 
    2264             :     Assert(LWLockHeldByMe(DSA_AREA_LOCK(area)));
    2265        2862 :     freed_segment_counter = area->control->freed_segment_counter;
    2266        2862 :     if (unlikely(area->freed_segment_counter != freed_segment_counter))
    2267             :     {
    2268           0 :         for (i = 0; i <= area->high_segment_index; ++i)
    2269             :         {
    2270           0 :             if (area->segment_maps[i].header != NULL &&
    2271           0 :                 area->segment_maps[i].header->freed)
    2272             :             {
    2273           0 :                 dsm_detach(area->segment_maps[i].segment);
    2274           0 :                 area->segment_maps[i].segment = NULL;
    2275           0 :                 area->segment_maps[i].header = NULL;
    2276           0 :                 area->segment_maps[i].mapped_address = NULL;
    2277             :             }
    2278             :         }
    2279           0 :         area->freed_segment_counter = freed_segment_counter;
    2280             :     }
    2281        2862 : }

Generated by: LCOV version 1.13