LCOV - code coverage report
Current view: top level - src/backend/storage/buffer - bufmgr.c (source / functions) Hit Total Coverage
Test: PostgreSQL 18devel Lines: 1343 1530 87.8 %
Date: 2024-12-02 19:14:57 Functions: 86 94 91.5 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * bufmgr.c
       4             :  *    buffer manager interface routines
       5             :  *
       6             :  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
       7             :  * Portions Copyright (c) 1994, Regents of the University of California
       8             :  *
       9             :  *
      10             :  * IDENTIFICATION
      11             :  *    src/backend/storage/buffer/bufmgr.c
      12             :  *
      13             :  *-------------------------------------------------------------------------
      14             :  */
      15             : /*
      16             :  * Principal entry points:
      17             :  *
      18             :  * ReadBuffer() -- find or create a buffer holding the requested page,
      19             :  *      and pin it so that no one can destroy it while this process
      20             :  *      is using it.
      21             :  *
      22             :  * StartReadBuffer() -- as above, with separate wait step
      23             :  * StartReadBuffers() -- multiple block version
      24             :  * WaitReadBuffers() -- second step of above
      25             :  *
      26             :  * ReleaseBuffer() -- unpin a buffer
      27             :  *
      28             :  * MarkBufferDirty() -- mark a pinned buffer's contents as "dirty".
      29             :  *      The disk write is delayed until buffer replacement or checkpoint.
      30             :  *
      31             :  * See also these files:
      32             :  *      freelist.c -- chooses victim for buffer replacement
      33             :  *      buf_table.c -- manages the buffer lookup table
      34             :  */
      35             : #include "postgres.h"
      36             : 
      37             : #include <sys/file.h>
      38             : #include <unistd.h>
      39             : 
      40             : #include "access/tableam.h"
      41             : #include "access/xloginsert.h"
      42             : #include "access/xlogutils.h"
      43             : #include "catalog/storage.h"
      44             : #include "catalog/storage_xlog.h"
      45             : #include "executor/instrument.h"
      46             : #include "lib/binaryheap.h"
      47             : #include "miscadmin.h"
      48             : #include "pg_trace.h"
      49             : #include "pgstat.h"
      50             : #include "postmaster/bgwriter.h"
      51             : #include "storage/buf_internals.h"
      52             : #include "storage/bufmgr.h"
      53             : #include "storage/fd.h"
      54             : #include "storage/ipc.h"
      55             : #include "storage/lmgr.h"
      56             : #include "storage/proc.h"
      57             : #include "storage/read_stream.h"
      58             : #include "storage/smgr.h"
      59             : #include "storage/standby.h"
      60             : #include "utils/memdebug.h"
      61             : #include "utils/ps_status.h"
      62             : #include "utils/rel.h"
      63             : #include "utils/resowner.h"
      64             : #include "utils/timestamp.h"
      65             : 
      66             : 
      67             : /* Note: these two macros only work on shared buffers, not local ones! */
      68             : #define BufHdrGetBlock(bufHdr)  ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ))
      69             : #define BufferGetLSN(bufHdr)    (PageGetLSN(BufHdrGetBlock(bufHdr)))
      70             : 
      71             : /* Note: this macro only works on local buffers, not shared ones! */
      72             : #define LocalBufHdrGetBlock(bufHdr) \
      73             :     LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)]
      74             : 
      75             : /* Bits in SyncOneBuffer's return value */
      76             : #define BUF_WRITTEN             0x01
      77             : #define BUF_REUSABLE            0x02
      78             : 
      79             : #define RELS_BSEARCH_THRESHOLD      20
      80             : 
      81             : /*
      82             :  * This is the size (in the number of blocks) above which we scan the
      83             :  * entire buffer pool to remove the buffers for all the pages of relation
      84             :  * being dropped. For the relations with size below this threshold, we find
      85             :  * the buffers by doing lookups in BufMapping table.
      86             :  */
      87             : #define BUF_DROP_FULL_SCAN_THRESHOLD        (uint64) (NBuffers / 32)
      88             : 
      89             : typedef struct PrivateRefCountEntry
      90             : {
      91             :     Buffer      buffer;
      92             :     int32       refcount;
      93             : } PrivateRefCountEntry;
      94             : 
      95             : /* 64 bytes, about the size of a cache line on common systems */
      96             : #define REFCOUNT_ARRAY_ENTRIES 8
      97             : 
      98             : /*
      99             :  * Status of buffers to checkpoint for a particular tablespace, used
     100             :  * internally in BufferSync.
     101             :  */
     102             : typedef struct CkptTsStatus
     103             : {
     104             :     /* oid of the tablespace */
     105             :     Oid         tsId;
     106             : 
     107             :     /*
     108             :      * Checkpoint progress for this tablespace. To make progress comparable
     109             :      * between tablespaces the progress is, for each tablespace, measured as a
     110             :      * number between 0 and the total number of to-be-checkpointed pages. Each
     111             :      * page checkpointed in this tablespace increments this space's progress
     112             :      * by progress_slice.
     113             :      */
     114             :     float8      progress;
     115             :     float8      progress_slice;
     116             : 
     117             :     /* number of to-be checkpointed pages in this tablespace */
     118             :     int         num_to_scan;
     119             :     /* already processed pages in this tablespace */
     120             :     int         num_scanned;
     121             : 
     122             :     /* current offset in CkptBufferIds for this tablespace */
     123             :     int         index;
     124             : } CkptTsStatus;
     125             : 
     126             : /*
     127             :  * Type for array used to sort SMgrRelations
     128             :  *
     129             :  * FlushRelationsAllBuffers shares the same comparator function with
     130             :  * DropRelationsAllBuffers. Pointer to this struct and RelFileLocator must be
     131             :  * compatible.
     132             :  */
     133             : typedef struct SMgrSortArray
     134             : {
     135             :     RelFileLocator rlocator;    /* This must be the first member */
     136             :     SMgrRelation srel;
     137             : } SMgrSortArray;
     138             : 
     139             : /* GUC variables */
     140             : bool        zero_damaged_pages = false;
     141             : int         bgwriter_lru_maxpages = 100;
     142             : double      bgwriter_lru_multiplier = 2.0;
     143             : bool        track_io_timing = false;
     144             : 
     145             : /*
     146             :  * How many buffers PrefetchBuffer callers should try to stay ahead of their
     147             :  * ReadBuffer calls by.  Zero means "never prefetch".  This value is only used
     148             :  * for buffers not belonging to tablespaces that have their
     149             :  * effective_io_concurrency parameter set.
     150             :  */
     151             : int         effective_io_concurrency = DEFAULT_EFFECTIVE_IO_CONCURRENCY;
     152             : 
     153             : /*
     154             :  * Like effective_io_concurrency, but used by maintenance code paths that might
     155             :  * benefit from a higher setting because they work on behalf of many sessions.
     156             :  * Overridden by the tablespace setting of the same name.
     157             :  */
     158             : int         maintenance_io_concurrency = DEFAULT_MAINTENANCE_IO_CONCURRENCY;
     159             : 
     160             : /*
     161             :  * Limit on how many blocks should be handled in single I/O operations.
     162             :  * StartReadBuffers() callers should respect it, as should other operations
     163             :  * that call smgr APIs directly.
     164             :  */
     165             : int         io_combine_limit = DEFAULT_IO_COMBINE_LIMIT;
     166             : 
     167             : /*
     168             :  * GUC variables about triggering kernel writeback for buffers written; OS
     169             :  * dependent defaults are set via the GUC mechanism.
     170             :  */
     171             : int         checkpoint_flush_after = DEFAULT_CHECKPOINT_FLUSH_AFTER;
     172             : int         bgwriter_flush_after = DEFAULT_BGWRITER_FLUSH_AFTER;
     173             : int         backend_flush_after = DEFAULT_BACKEND_FLUSH_AFTER;
     174             : 
     175             : /* local state for LockBufferForCleanup */
     176             : static BufferDesc *PinCountWaitBuf = NULL;
     177             : 
     178             : /*
     179             :  * Backend-Private refcount management:
     180             :  *
     181             :  * Each buffer also has a private refcount that keeps track of the number of
     182             :  * times the buffer is pinned in the current process.  This is so that the
     183             :  * shared refcount needs to be modified only once if a buffer is pinned more
     184             :  * than once by an individual backend.  It's also used to check that no buffers
     185             :  * are still pinned at the end of transactions and when exiting.
     186             :  *
     187             :  *
     188             :  * To avoid - as we used to - requiring an array with NBuffers entries to keep
     189             :  * track of local buffers, we use a small sequentially searched array
     190             :  * (PrivateRefCountArray) and an overflow hash table (PrivateRefCountHash) to
     191             :  * keep track of backend local pins.
     192             :  *
     193             :  * Until no more than REFCOUNT_ARRAY_ENTRIES buffers are pinned at once, all
     194             :  * refcounts are kept track of in the array; after that, new array entries
     195             :  * displace old ones into the hash table. That way a frequently used entry
     196             :  * can't get "stuck" in the hashtable while infrequent ones clog the array.
     197             :  *
     198             :  * Note that in most scenarios the number of pinned buffers will not exceed
     199             :  * REFCOUNT_ARRAY_ENTRIES.
     200             :  *
     201             :  *
     202             :  * To enter a buffer into the refcount tracking mechanism first reserve a free
     203             :  * entry using ReservePrivateRefCountEntry() and then later, if necessary,
     204             :  * fill it with NewPrivateRefCountEntry(). That split lets us avoid doing
     205             :  * memory allocations in NewPrivateRefCountEntry() which can be important
     206             :  * because in some scenarios it's called with a spinlock held...
     207             :  */
     208             : static struct PrivateRefCountEntry PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES];
     209             : static HTAB *PrivateRefCountHash = NULL;
     210             : static int32 PrivateRefCountOverflowed = 0;
     211             : static uint32 PrivateRefCountClock = 0;
     212             : static PrivateRefCountEntry *ReservedRefCountEntry = NULL;
     213             : 
     214             : static void ReservePrivateRefCountEntry(void);
     215             : static PrivateRefCountEntry *NewPrivateRefCountEntry(Buffer buffer);
     216             : static PrivateRefCountEntry *GetPrivateRefCountEntry(Buffer buffer, bool do_move);
     217             : static inline int32 GetPrivateRefCount(Buffer buffer);
     218             : static void ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref);
     219             : 
     220             : /* ResourceOwner callbacks to hold in-progress I/Os and buffer pins */
     221             : static void ResOwnerReleaseBufferIO(Datum res);
     222             : static char *ResOwnerPrintBufferIO(Datum res);
     223             : static void ResOwnerReleaseBufferPin(Datum res);
     224             : static char *ResOwnerPrintBufferPin(Datum res);
     225             : 
     226             : const ResourceOwnerDesc buffer_io_resowner_desc =
     227             : {
     228             :     .name = "buffer io",
     229             :     .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
     230             :     .release_priority = RELEASE_PRIO_BUFFER_IOS,
     231             :     .ReleaseResource = ResOwnerReleaseBufferIO,
     232             :     .DebugPrint = ResOwnerPrintBufferIO
     233             : };
     234             : 
     235             : const ResourceOwnerDesc buffer_pin_resowner_desc =
     236             : {
     237             :     .name = "buffer pin",
     238             :     .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
     239             :     .release_priority = RELEASE_PRIO_BUFFER_PINS,
     240             :     .ReleaseResource = ResOwnerReleaseBufferPin,
     241             :     .DebugPrint = ResOwnerPrintBufferPin
     242             : };
     243             : 
     244             : /*
     245             :  * Ensure that the PrivateRefCountArray has sufficient space to store one more
     246             :  * entry. This has to be called before using NewPrivateRefCountEntry() to fill
     247             :  * a new entry - but it's perfectly fine to not use a reserved entry.
     248             :  */
     249             : static void
     250   114030714 : ReservePrivateRefCountEntry(void)
     251             : {
     252             :     /* Already reserved (or freed), nothing to do */
     253   114030714 :     if (ReservedRefCountEntry != NULL)
     254   106697662 :         return;
     255             : 
     256             :     /*
     257             :      * First search for a free entry the array, that'll be sufficient in the
     258             :      * majority of cases.
     259             :      */
     260             :     {
     261             :         int         i;
     262             : 
     263    18324092 :         for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
     264             :         {
     265             :             PrivateRefCountEntry *res;
     266             : 
     267    18093146 :             res = &PrivateRefCountArray[i];
     268             : 
     269    18093146 :             if (res->buffer == InvalidBuffer)
     270             :             {
     271     7102106 :                 ReservedRefCountEntry = res;
     272     7102106 :                 return;
     273             :             }
     274             :         }
     275             :     }
     276             : 
     277             :     /*
     278             :      * No luck. All array entries are full. Move one array entry into the hash
     279             :      * table.
     280             :      */
     281             :     {
     282             :         /*
     283             :          * Move entry from the current clock position in the array into the
     284             :          * hashtable. Use that slot.
     285             :          */
     286             :         PrivateRefCountEntry *hashent;
     287             :         bool        found;
     288             : 
     289             :         /* select victim slot */
     290      230946 :         ReservedRefCountEntry =
     291      230946 :             &PrivateRefCountArray[PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES];
     292             : 
     293             :         /* Better be used, otherwise we shouldn't get here. */
     294             :         Assert(ReservedRefCountEntry->buffer != InvalidBuffer);
     295             : 
     296             :         /* enter victim array entry into hashtable */
     297      230946 :         hashent = hash_search(PrivateRefCountHash,
     298      230946 :                               &(ReservedRefCountEntry->buffer),
     299             :                               HASH_ENTER,
     300             :                               &found);
     301             :         Assert(!found);
     302      230946 :         hashent->refcount = ReservedRefCountEntry->refcount;
     303             : 
     304             :         /* clear the now free array slot */
     305      230946 :         ReservedRefCountEntry->buffer = InvalidBuffer;
     306      230946 :         ReservedRefCountEntry->refcount = 0;
     307             : 
     308      230946 :         PrivateRefCountOverflowed++;
     309             :     }
     310             : }
     311             : 
     312             : /*
     313             :  * Fill a previously reserved refcount entry.
     314             :  */
     315             : static PrivateRefCountEntry *
     316   103792392 : NewPrivateRefCountEntry(Buffer buffer)
     317             : {
     318             :     PrivateRefCountEntry *res;
     319             : 
     320             :     /* only allowed to be called when a reservation has been made */
     321             :     Assert(ReservedRefCountEntry != NULL);
     322             : 
     323             :     /* use up the reserved entry */
     324   103792392 :     res = ReservedRefCountEntry;
     325   103792392 :     ReservedRefCountEntry = NULL;
     326             : 
     327             :     /* and fill it */
     328   103792392 :     res->buffer = buffer;
     329   103792392 :     res->refcount = 0;
     330             : 
     331   103792392 :     return res;
     332             : }
     333             : 
     334             : /*
     335             :  * Return the PrivateRefCount entry for the passed buffer.
     336             :  *
     337             :  * Returns NULL if a buffer doesn't have a refcount entry. Otherwise, if
     338             :  * do_move is true, and the entry resides in the hashtable the entry is
     339             :  * optimized for frequent access by moving it to the array.
     340             :  */
     341             : static PrivateRefCountEntry *
     342   253864474 : GetPrivateRefCountEntry(Buffer buffer, bool do_move)
     343             : {
     344             :     PrivateRefCountEntry *res;
     345             :     int         i;
     346             : 
     347             :     Assert(BufferIsValid(buffer));
     348             :     Assert(!BufferIsLocal(buffer));
     349             : 
     350             :     /*
     351             :      * First search for references in the array, that'll be sufficient in the
     352             :      * majority of cases.
     353             :      */
     354  1207661970 :     for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
     355             :     {
     356  1107557510 :         res = &PrivateRefCountArray[i];
     357             : 
     358  1107557510 :         if (res->buffer == buffer)
     359   153760014 :             return res;
     360             :     }
     361             : 
     362             :     /*
     363             :      * By here we know that the buffer, if already pinned, isn't residing in
     364             :      * the array.
     365             :      *
     366             :      * Only look up the buffer in the hashtable if we've previously overflowed
     367             :      * into it.
     368             :      */
     369   100104460 :     if (PrivateRefCountOverflowed == 0)
     370    99456028 :         return NULL;
     371             : 
     372      648432 :     res = hash_search(PrivateRefCountHash, &buffer, HASH_FIND, NULL);
     373             : 
     374      648432 :     if (res == NULL)
     375      416788 :         return NULL;
     376      231644 :     else if (!do_move)
     377             :     {
     378             :         /* caller doesn't want us to move the hash entry into the array */
     379      216518 :         return res;
     380             :     }
     381             :     else
     382             :     {
     383             :         /* move buffer from hashtable into the free array slot */
     384             :         bool        found;
     385             :         PrivateRefCountEntry *free;
     386             : 
     387             :         /* Ensure there's a free array slot */
     388       15126 :         ReservePrivateRefCountEntry();
     389             : 
     390             :         /* Use up the reserved slot */
     391             :         Assert(ReservedRefCountEntry != NULL);
     392       15126 :         free = ReservedRefCountEntry;
     393       15126 :         ReservedRefCountEntry = NULL;
     394             :         Assert(free->buffer == InvalidBuffer);
     395             : 
     396             :         /* and fill it */
     397       15126 :         free->buffer = buffer;
     398       15126 :         free->refcount = res->refcount;
     399             : 
     400             :         /* delete from hashtable */
     401       15126 :         hash_search(PrivateRefCountHash, &buffer, HASH_REMOVE, &found);
     402             :         Assert(found);
     403             :         Assert(PrivateRefCountOverflowed > 0);
     404       15126 :         PrivateRefCountOverflowed--;
     405             : 
     406       15126 :         return free;
     407             :     }
     408             : }
     409             : 
     410             : /*
     411             :  * Returns how many times the passed buffer is pinned by this backend.
     412             :  *
     413             :  * Only works for shared memory buffers!
     414             :  */
     415             : static inline int32
     416     3954314 : GetPrivateRefCount(Buffer buffer)
     417             : {
     418             :     PrivateRefCountEntry *ref;
     419             : 
     420             :     Assert(BufferIsValid(buffer));
     421             :     Assert(!BufferIsLocal(buffer));
     422             : 
     423             :     /*
     424             :      * Not moving the entry - that's ok for the current users, but we might
     425             :      * want to change this one day.
     426             :      */
     427     3954314 :     ref = GetPrivateRefCountEntry(buffer, false);
     428             : 
     429     3954314 :     if (ref == NULL)
     430        9084 :         return 0;
     431     3945230 :     return ref->refcount;
     432             : }
     433             : 
     434             : /*
     435             :  * Release resources used to track the reference count of a buffer which we no
     436             :  * longer have pinned and don't want to pin again immediately.
     437             :  */
     438             : static void
     439   103792392 : ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref)
     440             : {
     441             :     Assert(ref->refcount == 0);
     442             : 
     443   103792392 :     if (ref >= &PrivateRefCountArray[0] &&
     444             :         ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES])
     445             :     {
     446   103576572 :         ref->buffer = InvalidBuffer;
     447             : 
     448             :         /*
     449             :          * Mark the just used entry as reserved - in many scenarios that
     450             :          * allows us to avoid ever having to search the array/hash for free
     451             :          * entries.
     452             :          */
     453   103576572 :         ReservedRefCountEntry = ref;
     454             :     }
     455             :     else
     456             :     {
     457             :         bool        found;
     458      215820 :         Buffer      buffer = ref->buffer;
     459             : 
     460      215820 :         hash_search(PrivateRefCountHash, &buffer, HASH_REMOVE, &found);
     461             :         Assert(found);
     462             :         Assert(PrivateRefCountOverflowed > 0);
     463      215820 :         PrivateRefCountOverflowed--;
     464             :     }
     465   103792392 : }
     466             : 
     467             : /*
     468             :  * BufferIsPinned
     469             :  *      True iff the buffer is pinned (also checks for valid buffer number).
     470             :  *
     471             :  *      NOTE: what we check here is that *this* backend holds a pin on
     472             :  *      the buffer.  We do not care whether some other backend does.
     473             :  */
     474             : #define BufferIsPinned(bufnum) \
     475             : ( \
     476             :     !BufferIsValid(bufnum) ? \
     477             :         false \
     478             :     : \
     479             :         BufferIsLocal(bufnum) ? \
     480             :             (LocalRefCount[-(bufnum) - 1] > 0) \
     481             :         : \
     482             :     (GetPrivateRefCount(bufnum) > 0) \
     483             : )
     484             : 
     485             : 
     486             : static Buffer ReadBuffer_common(Relation rel,
     487             :                                 SMgrRelation smgr, char smgr_persistence,
     488             :                                 ForkNumber forkNum, BlockNumber blockNum,
     489             :                                 ReadBufferMode mode, BufferAccessStrategy strategy);
     490             : static BlockNumber ExtendBufferedRelCommon(BufferManagerRelation bmr,
     491             :                                            ForkNumber fork,
     492             :                                            BufferAccessStrategy strategy,
     493             :                                            uint32 flags,
     494             :                                            uint32 extend_by,
     495             :                                            BlockNumber extend_upto,
     496             :                                            Buffer *buffers,
     497             :                                            uint32 *extended_by);
     498             : static BlockNumber ExtendBufferedRelShared(BufferManagerRelation bmr,
     499             :                                            ForkNumber fork,
     500             :                                            BufferAccessStrategy strategy,
     501             :                                            uint32 flags,
     502             :                                            uint32 extend_by,
     503             :                                            BlockNumber extend_upto,
     504             :                                            Buffer *buffers,
     505             :                                            uint32 *extended_by);
     506             : static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy);
     507             : static void PinBuffer_Locked(BufferDesc *buf);
     508             : static void UnpinBuffer(BufferDesc *buf);
     509             : static void UnpinBufferNoOwner(BufferDesc *buf);
     510             : static void BufferSync(int flags);
     511             : static uint32 WaitBufHdrUnlocked(BufferDesc *buf);
     512             : static int  SyncOneBuffer(int buf_id, bool skip_recently_used,
     513             :                           WritebackContext *wb_context);
     514             : static void WaitIO(BufferDesc *buf);
     515             : static bool StartBufferIO(BufferDesc *buf, bool forInput, bool nowait);
     516             : static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty,
     517             :                               uint32 set_flag_bits, bool forget_owner);
     518             : static void AbortBufferIO(Buffer buffer);
     519             : static void shared_buffer_write_error_callback(void *arg);
     520             : static void local_buffer_write_error_callback(void *arg);
     521             : static inline BufferDesc *BufferAlloc(SMgrRelation smgr,
     522             :                                       char relpersistence,
     523             :                                       ForkNumber forkNum,
     524             :                                       BlockNumber blockNum,
     525             :                                       BufferAccessStrategy strategy,
     526             :                                       bool *foundPtr, IOContext io_context);
     527             : static Buffer GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context);
     528             : static void FlushBuffer(BufferDesc *buf, SMgrRelation reln,
     529             :                         IOObject io_object, IOContext io_context);
     530             : static void FindAndDropRelationBuffers(RelFileLocator rlocator,
     531             :                                        ForkNumber forkNum,
     532             :                                        BlockNumber nForkBlock,
     533             :                                        BlockNumber firstDelBlock);
     534             : static void RelationCopyStorageUsingBuffer(RelFileLocator srclocator,
     535             :                                            RelFileLocator dstlocator,
     536             :                                            ForkNumber forkNum, bool permanent);
     537             : static void AtProcExit_Buffers(int code, Datum arg);
     538             : static void CheckForBufferLeaks(void);
     539             : static int  rlocator_comparator(const void *p1, const void *p2);
     540             : static inline int buffertag_comparator(const BufferTag *ba, const BufferTag *bb);
     541             : static inline int ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b);
     542             : static int  ts_ckpt_progress_comparator(Datum a, Datum b, void *arg);
     543             : 
     544             : 
     545             : /*
     546             :  * Implementation of PrefetchBuffer() for shared buffers.
     547             :  */
     548             : PrefetchBufferResult
     549      403454 : PrefetchSharedBuffer(SMgrRelation smgr_reln,
     550             :                      ForkNumber forkNum,
     551             :                      BlockNumber blockNum)
     552             : {
     553      403454 :     PrefetchBufferResult result = {InvalidBuffer, false};
     554             :     BufferTag   newTag;         /* identity of requested block */
     555             :     uint32      newHash;        /* hash value for newTag */
     556             :     LWLock     *newPartitionLock;   /* buffer partition lock for it */
     557             :     int         buf_id;
     558             : 
     559             :     Assert(BlockNumberIsValid(blockNum));
     560             : 
     561             :     /* create a tag so we can lookup the buffer */
     562      403454 :     InitBufferTag(&newTag, &smgr_reln->smgr_rlocator.locator,
     563             :                   forkNum, blockNum);
     564             : 
     565             :     /* determine its hash code and partition lock ID */
     566      403454 :     newHash = BufTableHashCode(&newTag);
     567      403454 :     newPartitionLock = BufMappingPartitionLock(newHash);
     568             : 
     569             :     /* see if the block is in the buffer pool already */
     570      403454 :     LWLockAcquire(newPartitionLock, LW_SHARED);
     571      403454 :     buf_id = BufTableLookup(&newTag, newHash);
     572      403454 :     LWLockRelease(newPartitionLock);
     573             : 
     574             :     /* If not in buffers, initiate prefetch */
     575      403454 :     if (buf_id < 0)
     576             :     {
     577             : #ifdef USE_PREFETCH
     578             :         /*
     579             :          * Try to initiate an asynchronous read.  This returns false in
     580             :          * recovery if the relation file doesn't exist.
     581             :          */
     582      346946 :         if ((io_direct_flags & IO_DIRECT_DATA) == 0 &&
     583      173252 :             smgrprefetch(smgr_reln, forkNum, blockNum, 1))
     584             :         {
     585      173252 :             result.initiated_io = true;
     586             :         }
     587             : #endif                          /* USE_PREFETCH */
     588             :     }
     589             :     else
     590             :     {
     591             :         /*
     592             :          * Report the buffer it was in at that time.  The caller may be able
     593             :          * to avoid a buffer table lookup, but it's not pinned and it must be
     594             :          * rechecked!
     595             :          */
     596      229760 :         result.recent_buffer = buf_id + 1;
     597             :     }
     598             : 
     599             :     /*
     600             :      * If the block *is* in buffers, we do nothing.  This is not really ideal:
     601             :      * the block might be just about to be evicted, which would be stupid
     602             :      * since we know we are going to need it soon.  But the only easy answer
     603             :      * is to bump the usage_count, which does not seem like a great solution:
     604             :      * when the caller does ultimately touch the block, usage_count would get
     605             :      * bumped again, resulting in too much favoritism for blocks that are
     606             :      * involved in a prefetch sequence. A real fix would involve some
     607             :      * additional per-buffer state, and it's not clear that there's enough of
     608             :      * a problem to justify that.
     609             :      */
     610             : 
     611      403454 :     return result;
     612             : }
     613             : 
     614             : /*
     615             :  * PrefetchBuffer -- initiate asynchronous read of a block of a relation
     616             :  *
     617             :  * This is named by analogy to ReadBuffer but doesn't actually allocate a
     618             :  * buffer.  Instead it tries to ensure that a future ReadBuffer for the given
     619             :  * block will not be delayed by the I/O.  Prefetching is optional.
     620             :  *
     621             :  * There are three possible outcomes:
     622             :  *
     623             :  * 1.  If the block is already cached, the result includes a valid buffer that
     624             :  * could be used by the caller to avoid the need for a later buffer lookup, but
     625             :  * it's not pinned, so the caller must recheck it.
     626             :  *
     627             :  * 2.  If the kernel has been asked to initiate I/O, the initiated_io member is
     628             :  * true.  Currently there is no way to know if the data was already cached by
     629             :  * the kernel and therefore didn't really initiate I/O, and no way to know when
     630             :  * the I/O completes other than using synchronous ReadBuffer().
     631             :  *
     632             :  * 3.  Otherwise, the buffer wasn't already cached by PostgreSQL, and
     633             :  * USE_PREFETCH is not defined (this build doesn't support prefetching due to
     634             :  * lack of a kernel facility), direct I/O is enabled, or the underlying
     635             :  * relation file wasn't found and we are in recovery.  (If the relation file
     636             :  * wasn't found and we are not in recovery, an error is raised).
     637             :  */
     638             : PrefetchBufferResult
     639      387860 : PrefetchBuffer(Relation reln, ForkNumber forkNum, BlockNumber blockNum)
     640             : {
     641             :     Assert(RelationIsValid(reln));
     642             :     Assert(BlockNumberIsValid(blockNum));
     643             : 
     644      387860 :     if (RelationUsesLocalBuffers(reln))
     645             :     {
     646             :         /* see comments in ReadBufferExtended */
     647        6224 :         if (RELATION_IS_OTHER_TEMP(reln))
     648           0 :             ereport(ERROR,
     649             :                     (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
     650             :                      errmsg("cannot access temporary tables of other sessions")));
     651             : 
     652             :         /* pass it off to localbuf.c */
     653        6224 :         return PrefetchLocalBuffer(RelationGetSmgr(reln), forkNum, blockNum);
     654             :     }
     655             :     else
     656             :     {
     657             :         /* pass it to the shared buffer version */
     658      381636 :         return PrefetchSharedBuffer(RelationGetSmgr(reln), forkNum, blockNum);
     659             :     }
     660             : }
     661             : 
     662             : /*
     663             :  * ReadRecentBuffer -- try to pin a block in a recently observed buffer
     664             :  *
     665             :  * Compared to ReadBuffer(), this avoids a buffer mapping lookup when it's
     666             :  * successful.  Return true if the buffer is valid and still has the expected
     667             :  * tag.  In that case, the buffer is pinned and the usage count is bumped.
     668             :  */
     669             : bool
     670        9078 : ReadRecentBuffer(RelFileLocator rlocator, ForkNumber forkNum, BlockNumber blockNum,
     671             :                  Buffer recent_buffer)
     672             : {
     673             :     BufferDesc *bufHdr;
     674             :     BufferTag   tag;
     675             :     uint32      buf_state;
     676             :     bool        have_private_ref;
     677             : 
     678             :     Assert(BufferIsValid(recent_buffer));
     679             : 
     680        9078 :     ResourceOwnerEnlarge(CurrentResourceOwner);
     681        9078 :     ReservePrivateRefCountEntry();
     682        9078 :     InitBufferTag(&tag, &rlocator, forkNum, blockNum);
     683             : 
     684        9078 :     if (BufferIsLocal(recent_buffer))
     685             :     {
     686           0 :         int         b = -recent_buffer - 1;
     687             : 
     688           0 :         bufHdr = GetLocalBufferDescriptor(b);
     689           0 :         buf_state = pg_atomic_read_u32(&bufHdr->state);
     690             : 
     691             :         /* Is it still valid and holding the right tag? */
     692           0 :         if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag))
     693             :         {
     694           0 :             PinLocalBuffer(bufHdr, true);
     695             : 
     696           0 :             pgBufferUsage.local_blks_hit++;
     697             : 
     698           0 :             return true;
     699             :         }
     700             :     }
     701             :     else
     702             :     {
     703        9078 :         bufHdr = GetBufferDescriptor(recent_buffer - 1);
     704        9078 :         have_private_ref = GetPrivateRefCount(recent_buffer) > 0;
     705             : 
     706             :         /*
     707             :          * Do we already have this buffer pinned with a private reference?  If
     708             :          * so, it must be valid and it is safe to check the tag without
     709             :          * locking.  If not, we have to lock the header first and then check.
     710             :          */
     711        9078 :         if (have_private_ref)
     712           0 :             buf_state = pg_atomic_read_u32(&bufHdr->state);
     713             :         else
     714        9078 :             buf_state = LockBufHdr(bufHdr);
     715             : 
     716        9078 :         if ((buf_state & BM_VALID) && BufferTagsEqual(&tag, &bufHdr->tag))
     717             :         {
     718             :             /*
     719             :              * It's now safe to pin the buffer.  We can't pin first and ask
     720             :              * questions later, because it might confuse code paths like
     721             :              * InvalidateBuffer() if we pinned a random non-matching buffer.
     722             :              */
     723        9004 :             if (have_private_ref)
     724           0 :                 PinBuffer(bufHdr, NULL);    /* bump pin count */
     725             :             else
     726        9004 :                 PinBuffer_Locked(bufHdr);   /* pin for first time */
     727             : 
     728        9004 :             pgBufferUsage.shared_blks_hit++;
     729             : 
     730        9004 :             return true;
     731             :         }
     732             : 
     733             :         /* If we locked the header above, now unlock. */
     734          74 :         if (!have_private_ref)
     735          74 :             UnlockBufHdr(bufHdr, buf_state);
     736             :     }
     737             : 
     738          74 :     return false;
     739             : }
     740             : 
     741             : /*
     742             :  * ReadBuffer -- a shorthand for ReadBufferExtended, for reading from main
     743             :  *      fork with RBM_NORMAL mode and default strategy.
     744             :  */
     745             : Buffer
     746    76346668 : ReadBuffer(Relation reln, BlockNumber blockNum)
     747             : {
     748    76346668 :     return ReadBufferExtended(reln, MAIN_FORKNUM, blockNum, RBM_NORMAL, NULL);
     749             : }
     750             : 
     751             : /*
     752             :  * ReadBufferExtended -- returns a buffer containing the requested
     753             :  *      block of the requested relation.  If the blknum
     754             :  *      requested is P_NEW, extend the relation file and
     755             :  *      allocate a new block.  (Caller is responsible for
     756             :  *      ensuring that only one backend tries to extend a
     757             :  *      relation at the same time!)
     758             :  *
     759             :  * Returns: the buffer number for the buffer containing
     760             :  *      the block read.  The returned buffer has been pinned.
     761             :  *      Does not return on error --- elog's instead.
     762             :  *
     763             :  * Assume when this function is called, that reln has been opened already.
     764             :  *
     765             :  * In RBM_NORMAL mode, the page is read from disk, and the page header is
     766             :  * validated.  An error is thrown if the page header is not valid.  (But
     767             :  * note that an all-zero page is considered "valid"; see
     768             :  * PageIsVerifiedExtended().)
     769             :  *
     770             :  * RBM_ZERO_ON_ERROR is like the normal mode, but if the page header is not
     771             :  * valid, the page is zeroed instead of throwing an error. This is intended
     772             :  * for non-critical data, where the caller is prepared to repair errors.
     773             :  *
     774             :  * In RBM_ZERO_AND_LOCK mode, if the page isn't in buffer cache already, it's
     775             :  * filled with zeros instead of reading it from disk.  Useful when the caller
     776             :  * is going to fill the page from scratch, since this saves I/O and avoids
     777             :  * unnecessary failure if the page-on-disk has corrupt page headers.
     778             :  * The page is returned locked to ensure that the caller has a chance to
     779             :  * initialize the page before it's made visible to others.
     780             :  * Caution: do not use this mode to read a page that is beyond the relation's
     781             :  * current physical EOF; that is likely to cause problems in md.c when
     782             :  * the page is modified and written out. P_NEW is OK, though.
     783             :  *
     784             :  * RBM_ZERO_AND_CLEANUP_LOCK is the same as RBM_ZERO_AND_LOCK, but acquires
     785             :  * a cleanup-strength lock on the page.
     786             :  *
     787             :  * RBM_NORMAL_NO_LOG mode is treated the same as RBM_NORMAL here.
     788             :  *
     789             :  * If strategy is not NULL, a nondefault buffer access strategy is used.
     790             :  * See buffer/README for details.
     791             :  */
     792             : inline Buffer
     793    92482318 : ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum,
     794             :                    ReadBufferMode mode, BufferAccessStrategy strategy)
     795             : {
     796             :     Buffer      buf;
     797             : 
     798             :     /*
     799             :      * Reject attempts to read non-local temporary relations; we would be
     800             :      * likely to get wrong data since we have no visibility into the owning
     801             :      * session's local buffers.
     802             :      */
     803    92482318 :     if (RELATION_IS_OTHER_TEMP(reln))
     804           0 :         ereport(ERROR,
     805             :                 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
     806             :                  errmsg("cannot access temporary tables of other sessions")));
     807             : 
     808             :     /*
     809             :      * Read the buffer, and update pgstat counters to reflect a cache hit or
     810             :      * miss.
     811             :      */
     812    92482318 :     buf = ReadBuffer_common(reln, RelationGetSmgr(reln), 0,
     813             :                             forkNum, blockNum, mode, strategy);
     814             : 
     815    92482288 :     return buf;
     816             : }
     817             : 
     818             : 
     819             : /*
     820             :  * ReadBufferWithoutRelcache -- like ReadBufferExtended, but doesn't require
     821             :  *      a relcache entry for the relation.
     822             :  *
     823             :  * Pass permanent = true for a RELPERSISTENCE_PERMANENT relation, and
     824             :  * permanent = false for a RELPERSISTENCE_UNLOGGED relation. This function
     825             :  * cannot be used for temporary relations (and making that work might be
     826             :  * difficult, unless we only want to read temporary relations for our own
     827             :  * ProcNumber).
     828             :  */
     829             : Buffer
     830    10946006 : ReadBufferWithoutRelcache(RelFileLocator rlocator, ForkNumber forkNum,
     831             :                           BlockNumber blockNum, ReadBufferMode mode,
     832             :                           BufferAccessStrategy strategy, bool permanent)
     833             : {
     834    10946006 :     SMgrRelation smgr = smgropen(rlocator, INVALID_PROC_NUMBER);
     835             : 
     836    10946006 :     return ReadBuffer_common(NULL, smgr,
     837             :                              permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED,
     838             :                              forkNum, blockNum,
     839             :                              mode, strategy);
     840             : }
     841             : 
     842             : /*
     843             :  * Convenience wrapper around ExtendBufferedRelBy() extending by one block.
     844             :  */
     845             : Buffer
     846       89334 : ExtendBufferedRel(BufferManagerRelation bmr,
     847             :                   ForkNumber forkNum,
     848             :                   BufferAccessStrategy strategy,
     849             :                   uint32 flags)
     850             : {
     851             :     Buffer      buf;
     852       89334 :     uint32      extend_by = 1;
     853             : 
     854       89334 :     ExtendBufferedRelBy(bmr, forkNum, strategy, flags, extend_by,
     855             :                         &buf, &extend_by);
     856             : 
     857       89334 :     return buf;
     858             : }
     859             : 
     860             : /*
     861             :  * Extend relation by multiple blocks.
     862             :  *
     863             :  * Tries to extend the relation by extend_by blocks. Depending on the
     864             :  * availability of resources the relation may end up being extended by a
     865             :  * smaller number of pages (unless an error is thrown, always by at least one
     866             :  * page). *extended_by is updated to the number of pages the relation has been
     867             :  * extended to.
     868             :  *
     869             :  * buffers needs to be an array that is at least extend_by long. Upon
     870             :  * completion, the first extend_by array elements will point to a pinned
     871             :  * buffer.
     872             :  *
     873             :  * If EB_LOCK_FIRST is part of flags, the first returned buffer is
     874             :  * locked. This is useful for callers that want a buffer that is guaranteed to
     875             :  * be empty.
     876             :  */
     877             : BlockNumber
     878      289644 : ExtendBufferedRelBy(BufferManagerRelation bmr,
     879             :                     ForkNumber fork,
     880             :                     BufferAccessStrategy strategy,
     881             :                     uint32 flags,
     882             :                     uint32 extend_by,
     883             :                     Buffer *buffers,
     884             :                     uint32 *extended_by)
     885             : {
     886             :     Assert((bmr.rel != NULL) != (bmr.smgr != NULL));
     887             :     Assert(bmr.smgr == NULL || bmr.relpersistence != 0);
     888             :     Assert(extend_by > 0);
     889             : 
     890      289644 :     if (bmr.smgr == NULL)
     891             :     {
     892      289644 :         bmr.smgr = RelationGetSmgr(bmr.rel);
     893      289644 :         bmr.relpersistence = bmr.rel->rd_rel->relpersistence;
     894             :     }
     895             : 
     896      289644 :     return ExtendBufferedRelCommon(bmr, fork, strategy, flags,
     897             :                                    extend_by, InvalidBlockNumber,
     898             :                                    buffers, extended_by);
     899             : }
     900             : 
     901             : /*
     902             :  * Extend the relation so it is at least extend_to blocks large, return buffer
     903             :  * (extend_to - 1).
     904             :  *
     905             :  * This is useful for callers that want to write a specific page, regardless
     906             :  * of the current size of the relation (e.g. useful for visibilitymap and for
     907             :  * crash recovery).
     908             :  */
     909             : Buffer
     910       97182 : ExtendBufferedRelTo(BufferManagerRelation bmr,
     911             :                     ForkNumber fork,
     912             :                     BufferAccessStrategy strategy,
     913             :                     uint32 flags,
     914             :                     BlockNumber extend_to,
     915             :                     ReadBufferMode mode)
     916             : {
     917             :     BlockNumber current_size;
     918       97182 :     uint32      extended_by = 0;
     919       97182 :     Buffer      buffer = InvalidBuffer;
     920             :     Buffer      buffers[64];
     921             : 
     922             :     Assert((bmr.rel != NULL) != (bmr.smgr != NULL));
     923             :     Assert(bmr.smgr == NULL || bmr.relpersistence != 0);
     924             :     Assert(extend_to != InvalidBlockNumber && extend_to > 0);
     925             : 
     926       97182 :     if (bmr.smgr == NULL)
     927             :     {
     928       12786 :         bmr.smgr = RelationGetSmgr(bmr.rel);
     929       12786 :         bmr.relpersistence = bmr.rel->rd_rel->relpersistence;
     930             :     }
     931             : 
     932             :     /*
     933             :      * If desired, create the file if it doesn't exist.  If
     934             :      * smgr_cached_nblocks[fork] is positive then it must exist, no need for
     935             :      * an smgrexists call.
     936             :      */
     937       97182 :     if ((flags & EB_CREATE_FORK_IF_NEEDED) &&
     938       12786 :         (bmr.smgr->smgr_cached_nblocks[fork] == 0 ||
     939          32 :          bmr.smgr->smgr_cached_nblocks[fork] == InvalidBlockNumber) &&
     940       12754 :         !smgrexists(bmr.smgr, fork))
     941             :     {
     942       12742 :         LockRelationForExtension(bmr.rel, ExclusiveLock);
     943             : 
     944             :         /* recheck, fork might have been created concurrently */
     945       12742 :         if (!smgrexists(bmr.smgr, fork))
     946       12740 :             smgrcreate(bmr.smgr, fork, flags & EB_PERFORMING_RECOVERY);
     947             : 
     948       12742 :         UnlockRelationForExtension(bmr.rel, ExclusiveLock);
     949             :     }
     950             : 
     951             :     /*
     952             :      * If requested, invalidate size cache, so that smgrnblocks asks the
     953             :      * kernel.
     954             :      */
     955       97182 :     if (flags & EB_CLEAR_SIZE_CACHE)
     956       12786 :         bmr.smgr->smgr_cached_nblocks[fork] = InvalidBlockNumber;
     957             : 
     958             :     /*
     959             :      * Estimate how many pages we'll need to extend by. This avoids acquiring
     960             :      * unnecessarily many victim buffers.
     961             :      */
     962       97182 :     current_size = smgrnblocks(bmr.smgr, fork);
     963             : 
     964             :     /*
     965             :      * Since no-one else can be looking at the page contents yet, there is no
     966             :      * difference between an exclusive lock and a cleanup-strength lock. Note
     967             :      * that we pass the original mode to ReadBuffer_common() below, when
     968             :      * falling back to reading the buffer to a concurrent relation extension.
     969             :      */
     970       97182 :     if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
     971       83698 :         flags |= EB_LOCK_TARGET;
     972             : 
     973      198330 :     while (current_size < extend_to)
     974             :     {
     975      101148 :         uint32      num_pages = lengthof(buffers);
     976             :         BlockNumber first_block;
     977             : 
     978      101148 :         if ((uint64) current_size + num_pages > extend_to)
     979      101016 :             num_pages = extend_to - current_size;
     980             : 
     981      101148 :         first_block = ExtendBufferedRelCommon(bmr, fork, strategy, flags,
     982             :                                               num_pages, extend_to,
     983             :                                               buffers, &extended_by);
     984             : 
     985      101148 :         current_size = first_block + extended_by;
     986             :         Assert(num_pages != 0 || current_size >= extend_to);
     987             : 
     988      215370 :         for (uint32 i = 0; i < extended_by; i++)
     989             :         {
     990      114222 :             if (first_block + i != extend_to - 1)
     991       17042 :                 ReleaseBuffer(buffers[i]);
     992             :             else
     993       97180 :                 buffer = buffers[i];
     994             :         }
     995             :     }
     996             : 
     997             :     /*
     998             :      * It's possible that another backend concurrently extended the relation.
     999             :      * In that case read the buffer.
    1000             :      *
    1001             :      * XXX: Should we control this via a flag?
    1002             :      */
    1003       97182 :     if (buffer == InvalidBuffer)
    1004             :     {
    1005             :         Assert(extended_by == 0);
    1006           2 :         buffer = ReadBuffer_common(bmr.rel, bmr.smgr, bmr.relpersistence,
    1007             :                                    fork, extend_to - 1, mode, strategy);
    1008             :     }
    1009             : 
    1010       97182 :     return buffer;
    1011             : }
    1012             : 
    1013             : /*
    1014             :  * Lock and optionally zero a buffer, as part of the implementation of
    1015             :  * RBM_ZERO_AND_LOCK or RBM_ZERO_AND_CLEANUP_LOCK.  The buffer must be already
    1016             :  * pinned.  If the buffer is not already valid, it is zeroed and made valid.
    1017             :  */
    1018             : static void
    1019      550686 : ZeroAndLockBuffer(Buffer buffer, ReadBufferMode mode, bool already_valid)
    1020             : {
    1021             :     BufferDesc *bufHdr;
    1022             :     bool        need_to_zero;
    1023      550686 :     bool        isLocalBuf = BufferIsLocal(buffer);
    1024             : 
    1025             :     Assert(mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK);
    1026             : 
    1027      550686 :     if (already_valid)
    1028             :     {
    1029             :         /*
    1030             :          * If the caller already knew the buffer was valid, we can skip some
    1031             :          * header interaction.  The caller just wants to lock the buffer.
    1032             :          */
    1033       73202 :         need_to_zero = false;
    1034             :     }
    1035      477484 :     else if (isLocalBuf)
    1036             :     {
    1037             :         /* Simple case for non-shared buffers. */
    1038           0 :         bufHdr = GetLocalBufferDescriptor(-buffer - 1);
    1039           0 :         need_to_zero = (pg_atomic_read_u32(&bufHdr->state) & BM_VALID) == 0;
    1040             :     }
    1041             :     else
    1042             :     {
    1043             :         /*
    1044             :          * Take BM_IO_IN_PROGRESS, or discover that BM_VALID has been set
    1045             :          * concurrently.  Even though we aren't doing I/O, that ensures that
    1046             :          * we don't zero a page that someone else has pinned.  An exclusive
    1047             :          * content lock wouldn't be enough, because readers are allowed to
    1048             :          * drop the content lock after determining that a tuple is visible
    1049             :          * (see buffer access rules in README).
    1050             :          */
    1051      477484 :         bufHdr = GetBufferDescriptor(buffer - 1);
    1052      477484 :         need_to_zero = StartBufferIO(bufHdr, true, false);
    1053             :     }
    1054             : 
    1055      550686 :     if (need_to_zero)
    1056             :     {
    1057      477484 :         memset(BufferGetPage(buffer), 0, BLCKSZ);
    1058             : 
    1059             :         /*
    1060             :          * Grab the buffer content lock before marking the page as valid, to
    1061             :          * make sure that no other backend sees the zeroed page before the
    1062             :          * caller has had a chance to initialize it.
    1063             :          *
    1064             :          * Since no-one else can be looking at the page contents yet, there is
    1065             :          * no difference between an exclusive lock and a cleanup-strength
    1066             :          * lock. (Note that we cannot use LockBuffer() or
    1067             :          * LockBufferForCleanup() here, because they assert that the buffer is
    1068             :          * already valid.)
    1069             :          */
    1070      477484 :         if (!isLocalBuf)
    1071      477484 :             LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE);
    1072             : 
    1073      477484 :         if (isLocalBuf)
    1074             :         {
    1075             :             /* Only need to adjust flags */
    1076           0 :             uint32      buf_state = pg_atomic_read_u32(&bufHdr->state);
    1077             : 
    1078           0 :             buf_state |= BM_VALID;
    1079           0 :             pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
    1080             :         }
    1081             :         else
    1082             :         {
    1083             :             /* Set BM_VALID, terminate IO, and wake up any waiters */
    1084      477484 :             TerminateBufferIO(bufHdr, false, BM_VALID, true);
    1085             :         }
    1086             :     }
    1087       73202 :     else if (!isLocalBuf)
    1088             :     {
    1089             :         /*
    1090             :          * The buffer is valid, so we can't zero it.  The caller still expects
    1091             :          * the page to be locked on return.
    1092             :          */
    1093       73202 :         if (mode == RBM_ZERO_AND_LOCK)
    1094       73114 :             LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    1095             :         else
    1096          88 :             LockBufferForCleanup(buffer);
    1097             :     }
    1098      550686 : }
    1099             : 
    1100             : /*
    1101             :  * Pin a buffer for a given block.  *foundPtr is set to true if the block was
    1102             :  * already present, or false if more work is required to either read it in or
    1103             :  * zero it.
    1104             :  */
    1105             : static pg_attribute_always_inline Buffer
    1106   109115826 : PinBufferForBlock(Relation rel,
    1107             :                   SMgrRelation smgr,
    1108             :                   char persistence,
    1109             :                   ForkNumber forkNum,
    1110             :                   BlockNumber blockNum,
    1111             :                   BufferAccessStrategy strategy,
    1112             :                   bool *foundPtr)
    1113             : {
    1114             :     BufferDesc *bufHdr;
    1115             :     IOContext   io_context;
    1116             :     IOObject    io_object;
    1117             : 
    1118             :     Assert(blockNum != P_NEW);
    1119             : 
    1120             :     /* Persistence should be set before */
    1121             :     Assert((persistence == RELPERSISTENCE_TEMP ||
    1122             :             persistence == RELPERSISTENCE_PERMANENT ||
    1123             :             persistence == RELPERSISTENCE_UNLOGGED));
    1124             : 
    1125   109115826 :     if (persistence == RELPERSISTENCE_TEMP)
    1126             :     {
    1127     2124410 :         io_context = IOCONTEXT_NORMAL;
    1128     2124410 :         io_object = IOOBJECT_TEMP_RELATION;
    1129             :     }
    1130             :     else
    1131             :     {
    1132   106991416 :         io_context = IOContextForStrategy(strategy);
    1133   106991416 :         io_object = IOOBJECT_RELATION;
    1134             :     }
    1135             : 
    1136             :     TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum,
    1137             :                                        smgr->smgr_rlocator.locator.spcOid,
    1138             :                                        smgr->smgr_rlocator.locator.dbOid,
    1139             :                                        smgr->smgr_rlocator.locator.relNumber,
    1140             :                                        smgr->smgr_rlocator.backend);
    1141             : 
    1142   109115826 :     if (persistence == RELPERSISTENCE_TEMP)
    1143             :     {
    1144     2124410 :         bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, foundPtr);
    1145     2124410 :         if (*foundPtr)
    1146     2116754 :             pgBufferUsage.local_blks_hit++;
    1147             :     }
    1148             :     else
    1149             :     {
    1150   106991416 :         bufHdr = BufferAlloc(smgr, persistence, forkNum, blockNum,
    1151             :                              strategy, foundPtr, io_context);
    1152   106991416 :         if (*foundPtr)
    1153   104002166 :             pgBufferUsage.shared_blks_hit++;
    1154             :     }
    1155   109115826 :     if (rel)
    1156             :     {
    1157             :         /*
    1158             :          * While pgBufferUsage's "read" counter isn't bumped unless we reach
    1159             :          * WaitReadBuffers() (so, not for hits, and not for buffers that are
    1160             :          * zeroed instead), the per-relation stats always count them.
    1161             :          */
    1162    97768286 :         pgstat_count_buffer_read(rel);
    1163    97768286 :         if (*foundPtr)
    1164    95564132 :             pgstat_count_buffer_hit(rel);
    1165             :     }
    1166   109115826 :     if (*foundPtr)
    1167             :     {
    1168   106118920 :         pgstat_count_io_op(io_object, io_context, IOOP_HIT);
    1169   106118920 :         if (VacuumCostActive)
    1170     2323562 :             VacuumCostBalance += VacuumCostPageHit;
    1171             : 
    1172             :         TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum,
    1173             :                                           smgr->smgr_rlocator.locator.spcOid,
    1174             :                                           smgr->smgr_rlocator.locator.dbOid,
    1175             :                                           smgr->smgr_rlocator.locator.relNumber,
    1176             :                                           smgr->smgr_rlocator.backend,
    1177             :                                           true);
    1178             :     }
    1179             : 
    1180   109115826 :     return BufferDescriptorGetBuffer(bufHdr);
    1181             : }
    1182             : 
    1183             : /*
    1184             :  * ReadBuffer_common -- common logic for all ReadBuffer variants
    1185             :  *
    1186             :  * smgr is required, rel is optional unless using P_NEW.
    1187             :  */
    1188             : static pg_attribute_always_inline Buffer
    1189   103428326 : ReadBuffer_common(Relation rel, SMgrRelation smgr, char smgr_persistence,
    1190             :                   ForkNumber forkNum,
    1191             :                   BlockNumber blockNum, ReadBufferMode mode,
    1192             :                   BufferAccessStrategy strategy)
    1193             : {
    1194             :     ReadBuffersOperation operation;
    1195             :     Buffer      buffer;
    1196             :     int         flags;
    1197             :     char        persistence;
    1198             : 
    1199             :     /*
    1200             :      * Backward compatibility path, most code should use ExtendBufferedRel()
    1201             :      * instead, as acquiring the extension lock inside ExtendBufferedRel()
    1202             :      * scales a lot better.
    1203             :      */
    1204   103428326 :     if (unlikely(blockNum == P_NEW))
    1205             :     {
    1206         498 :         uint32      flags = EB_SKIP_EXTENSION_LOCK;
    1207             : 
    1208             :         /*
    1209             :          * Since no-one else can be looking at the page contents yet, there is
    1210             :          * no difference between an exclusive lock and a cleanup-strength
    1211             :          * lock.
    1212             :          */
    1213         498 :         if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
    1214           0 :             flags |= EB_LOCK_FIRST;
    1215             : 
    1216         498 :         return ExtendBufferedRel(BMR_REL(rel), forkNum, strategy, flags);
    1217             :     }
    1218             : 
    1219   103427828 :     if (rel)
    1220    92481822 :         persistence = rel->rd_rel->relpersistence;
    1221             :     else
    1222    10946006 :         persistence = smgr_persistence;
    1223             : 
    1224   103427828 :     if (unlikely(mode == RBM_ZERO_AND_CLEANUP_LOCK ||
    1225             :                  mode == RBM_ZERO_AND_LOCK))
    1226             :     {
    1227             :         bool        found;
    1228             : 
    1229      550686 :         buffer = PinBufferForBlock(rel, smgr, persistence,
    1230             :                                    forkNum, blockNum, strategy, &found);
    1231      550686 :         ZeroAndLockBuffer(buffer, mode, found);
    1232      550686 :         return buffer;
    1233             :     }
    1234             : 
    1235   102877142 :     if (mode == RBM_ZERO_ON_ERROR)
    1236     1836468 :         flags = READ_BUFFERS_ZERO_ON_ERROR;
    1237             :     else
    1238   101040674 :         flags = 0;
    1239   102877142 :     operation.smgr = smgr;
    1240   102877142 :     operation.rel = rel;
    1241   102877142 :     operation.persistence = persistence;
    1242   102877142 :     operation.forknum = forkNum;
    1243   102877142 :     operation.strategy = strategy;
    1244   102877142 :     if (StartReadBuffer(&operation,
    1245             :                         &buffer,
    1246             :                         blockNum,
    1247             :                         flags))
    1248     1362650 :         WaitReadBuffers(&operation);
    1249             : 
    1250   102877112 :     return buffer;
    1251             : }
    1252             : 
    1253             : static pg_attribute_always_inline bool
    1254   108324700 : StartReadBuffersImpl(ReadBuffersOperation *operation,
    1255             :                      Buffer *buffers,
    1256             :                      BlockNumber blockNum,
    1257             :                      int *nblocks,
    1258             :                      int flags)
    1259             : {
    1260   108324700 :     int         actual_nblocks = *nblocks;
    1261   108324700 :     int         io_buffers_len = 0;
    1262   108324700 :     int         maxcombine = 0;
    1263             : 
    1264             :     Assert(*nblocks > 0);
    1265             :     Assert(*nblocks <= MAX_IO_COMBINE_LIMIT);
    1266             : 
    1267   110844122 :     for (int i = 0; i < actual_nblocks; ++i)
    1268             :     {
    1269             :         bool        found;
    1270             : 
    1271   217130280 :         buffers[i] = PinBufferForBlock(operation->rel,
    1272   108565140 :                                        operation->smgr,
    1273   108565140 :                                        operation->persistence,
    1274             :                                        operation->forknum,
    1275             :                                        blockNum + i,
    1276             :                                        operation->strategy,
    1277             :                                        &found);
    1278             : 
    1279   108565140 :         if (found)
    1280             :         {
    1281             :             /*
    1282             :              * Terminate the read as soon as we get a hit.  It could be a
    1283             :              * single buffer hit, or it could be a hit that follows a readable
    1284             :              * range.  We don't want to create more than one readable range,
    1285             :              * so we stop here.
    1286             :              */
    1287   106045718 :             actual_nblocks = i + 1;
    1288   106045718 :             break;
    1289             :         }
    1290             :         else
    1291             :         {
    1292             :             /* Extend the readable range to cover this block. */
    1293     2519422 :             io_buffers_len++;
    1294             : 
    1295             :             /*
    1296             :              * Check how many blocks we can cover with the same IO. The smgr
    1297             :              * implementation might e.g. be limited due to a segment boundary.
    1298             :              */
    1299     2519422 :             if (i == 0 && actual_nblocks > 1)
    1300             :             {
    1301       51546 :                 maxcombine = smgrmaxcombine(operation->smgr,
    1302             :                                             operation->forknum,
    1303             :                                             blockNum);
    1304       51546 :                 if (unlikely(maxcombine < actual_nblocks))
    1305             :                 {
    1306           0 :                     elog(DEBUG2, "limiting nblocks at %u from %u to %u",
    1307             :                          blockNum, actual_nblocks, maxcombine);
    1308           0 :                     actual_nblocks = maxcombine;
    1309             :                 }
    1310             :             }
    1311             :         }
    1312             :     }
    1313   108324700 :     *nblocks = actual_nblocks;
    1314             : 
    1315   108324700 :     if (likely(io_buffers_len == 0))
    1316   106043570 :         return false;
    1317             : 
    1318             :     /* Populate information needed for I/O. */
    1319     2281130 :     operation->buffers = buffers;
    1320     2281130 :     operation->blocknum = blockNum;
    1321     2281130 :     operation->flags = flags;
    1322     2281130 :     operation->nblocks = actual_nblocks;
    1323     2281130 :     operation->io_buffers_len = io_buffers_len;
    1324             : 
    1325     2281130 :     if (flags & READ_BUFFERS_ISSUE_ADVICE)
    1326             :     {
    1327             :         /*
    1328             :          * In theory we should only do this if PinBufferForBlock() had to
    1329             :          * allocate new buffers above.  That way, if two calls to
    1330             :          * StartReadBuffers() were made for the same blocks before
    1331             :          * WaitReadBuffers(), only the first would issue the advice. That'd be
    1332             :          * a better simulation of true asynchronous I/O, which would only
    1333             :          * start the I/O once, but isn't done here for simplicity.  Note also
    1334             :          * that the following call might actually issue two advice calls if we
    1335             :          * cross a segment boundary; in a true asynchronous version we might
    1336             :          * choose to process only one real I/O at a time in that case.
    1337             :          */
    1338         532 :         smgrprefetch(operation->smgr,
    1339             :                      operation->forknum,
    1340             :                      blockNum,
    1341         532 :                      operation->io_buffers_len);
    1342             :     }
    1343             : 
    1344             :     /* Indicate that WaitReadBuffers() should be called. */
    1345     2281130 :     return true;
    1346             : }
    1347             : 
    1348             : /*
    1349             :  * Begin reading a range of blocks beginning at blockNum and extending for
    1350             :  * *nblocks.  On return, up to *nblocks pinned buffers holding those blocks
    1351             :  * are written into the buffers array, and *nblocks is updated to contain the
    1352             :  * actual number, which may be fewer than requested.  Caller sets some of the
    1353             :  * members of operation; see struct definition.
    1354             :  *
    1355             :  * If false is returned, no I/O is necessary.  If true is returned, one I/O
    1356             :  * has been started, and WaitReadBuffers() must be called with the same
    1357             :  * operation object before the buffers are accessed.  Along with the operation
    1358             :  * object, the caller-supplied array of buffers must remain valid until
    1359             :  * WaitReadBuffers() is called.
    1360             :  *
    1361             :  * Currently the I/O is only started with optional operating system advice if
    1362             :  * requested by the caller with READ_BUFFERS_ISSUE_ADVICE, and the real I/O
    1363             :  * happens synchronously in WaitReadBuffers().  In future work, true I/O could
    1364             :  * be initiated here.
    1365             :  */
    1366             : bool
    1367     2149060 : StartReadBuffers(ReadBuffersOperation *operation,
    1368             :                  Buffer *buffers,
    1369             :                  BlockNumber blockNum,
    1370             :                  int *nblocks,
    1371             :                  int flags)
    1372             : {
    1373     2149060 :     return StartReadBuffersImpl(operation, buffers, blockNum, nblocks, flags);
    1374             : }
    1375             : 
    1376             : /*
    1377             :  * Single block version of the StartReadBuffers().  This might save a few
    1378             :  * instructions when called from another translation unit, because it is
    1379             :  * specialized for nblocks == 1.
    1380             :  */
    1381             : bool
    1382   106175640 : StartReadBuffer(ReadBuffersOperation *operation,
    1383             :                 Buffer *buffer,
    1384             :                 BlockNumber blocknum,
    1385             :                 int flags)
    1386             : {
    1387   106175640 :     int         nblocks = 1;
    1388             :     bool        result;
    1389             : 
    1390   106175640 :     result = StartReadBuffersImpl(operation, buffer, blocknum, &nblocks, flags);
    1391             :     Assert(nblocks == 1);       /* single block can't be short */
    1392             : 
    1393   106175640 :     return result;
    1394             : }
    1395             : 
    1396             : static inline bool
    1397     2519416 : WaitReadBuffersCanStartIO(Buffer buffer, bool nowait)
    1398             : {
    1399     2519416 :     if (BufferIsLocal(buffer))
    1400             :     {
    1401        7656 :         BufferDesc *bufHdr = GetLocalBufferDescriptor(-buffer - 1);
    1402             : 
    1403        7656 :         return (pg_atomic_read_u32(&bufHdr->state) & BM_VALID) == 0;
    1404             :     }
    1405             :     else
    1406     2511760 :         return StartBufferIO(GetBufferDescriptor(buffer - 1), true, nowait);
    1407             : }
    1408             : 
    1409             : void
    1410     2281124 : WaitReadBuffers(ReadBuffersOperation *operation)
    1411             : {
    1412             :     Buffer     *buffers;
    1413             :     int         nblocks;
    1414             :     BlockNumber blocknum;
    1415             :     ForkNumber  forknum;
    1416             :     IOContext   io_context;
    1417             :     IOObject    io_object;
    1418             :     char        persistence;
    1419             : 
    1420             :     /*
    1421             :      * Currently operations are only allowed to include a read of some range,
    1422             :      * with an optional extra buffer that is already pinned at the end.  So
    1423             :      * nblocks can be at most one more than io_buffers_len.
    1424             :      */
    1425             :     Assert((operation->nblocks == operation->io_buffers_len) ||
    1426             :            (operation->nblocks == operation->io_buffers_len + 1));
    1427             : 
    1428             :     /* Find the range of the physical read we need to perform. */
    1429     2281124 :     nblocks = operation->io_buffers_len;
    1430     2281124 :     if (nblocks == 0)
    1431           0 :         return;                 /* nothing to do */
    1432             : 
    1433     2281124 :     buffers = &operation->buffers[0];
    1434     2281124 :     blocknum = operation->blocknum;
    1435     2281124 :     forknum = operation->forknum;
    1436     2281124 :     persistence = operation->persistence;
    1437             : 
    1438     2281124 :     if (persistence == RELPERSISTENCE_TEMP)
    1439             :     {
    1440        1632 :         io_context = IOCONTEXT_NORMAL;
    1441        1632 :         io_object = IOOBJECT_TEMP_RELATION;
    1442             :     }
    1443             :     else
    1444             :     {
    1445     2279492 :         io_context = IOContextForStrategy(operation->strategy);
    1446     2279492 :         io_object = IOOBJECT_RELATION;
    1447             :     }
    1448             : 
    1449             :     /*
    1450             :      * We count all these blocks as read by this backend.  This is traditional
    1451             :      * behavior, but might turn out to be not true if we find that someone
    1452             :      * else has beaten us and completed the read of some of these blocks.  In
    1453             :      * that case the system globally double-counts, but we traditionally don't
    1454             :      * count this as a "hit", and we don't have a separate counter for "miss,
    1455             :      * but another backend completed the read".
    1456             :      */
    1457     2281124 :     if (persistence == RELPERSISTENCE_TEMP)
    1458        1632 :         pgBufferUsage.local_blks_read += nblocks;
    1459             :     else
    1460     2279492 :         pgBufferUsage.shared_blks_read += nblocks;
    1461             : 
    1462     4562220 :     for (int i = 0; i < nblocks; ++i)
    1463             :     {
    1464             :         int         io_buffers_len;
    1465             :         Buffer      io_buffers[MAX_IO_COMBINE_LIMIT];
    1466             :         void       *io_pages[MAX_IO_COMBINE_LIMIT];
    1467             :         instr_time  io_start;
    1468             :         BlockNumber io_first_block;
    1469             : 
    1470             :         /*
    1471             :          * Skip this block if someone else has already completed it.  If an
    1472             :          * I/O is already in progress in another backend, this will wait for
    1473             :          * the outcome: either done, or something went wrong and we will
    1474             :          * retry.
    1475             :          */
    1476     2281126 :         if (!WaitReadBuffersCanStartIO(buffers[i], false))
    1477             :         {
    1478             :             /*
    1479             :              * Report this as a 'hit' for this backend, even though it must
    1480             :              * have started out as a miss in PinBufferForBlock().
    1481             :              */
    1482             :             TRACE_POSTGRESQL_BUFFER_READ_DONE(forknum, blocknum + i,
    1483             :                                               operation->smgr->smgr_rlocator.locator.spcOid,
    1484             :                                               operation->smgr->smgr_rlocator.locator.dbOid,
    1485             :                                               operation->smgr->smgr_rlocator.locator.relNumber,
    1486             :                                               operation->smgr->smgr_rlocator.backend,
    1487             :                                               true);
    1488        3756 :             continue;
    1489             :         }
    1490             : 
    1491             :         /* We found a buffer that we need to read in. */
    1492     2277370 :         io_buffers[0] = buffers[i];
    1493     2277370 :         io_pages[0] = BufferGetBlock(buffers[i]);
    1494     2277370 :         io_first_block = blocknum + i;
    1495     2277370 :         io_buffers_len = 1;
    1496             : 
    1497             :         /*
    1498             :          * How many neighboring-on-disk blocks can we scatter-read into other
    1499             :          * buffers at the same time?  In this case we don't wait if we see an
    1500             :          * I/O already in progress.  We already hold BM_IO_IN_PROGRESS for the
    1501             :          * head block, so we should get on with that I/O as soon as possible.
    1502             :          * We'll come back to this block again, above.
    1503             :          */
    1504     2753950 :         while ((i + 1) < nblocks &&
    1505      238290 :                WaitReadBuffersCanStartIO(buffers[i + 1], true))
    1506             :         {
    1507             :             /* Must be consecutive block numbers. */
    1508             :             Assert(BufferGetBlockNumber(buffers[i + 1]) ==
    1509             :                    BufferGetBlockNumber(buffers[i]) + 1);
    1510             : 
    1511      238290 :             io_buffers[io_buffers_len] = buffers[++i];
    1512      238290 :             io_pages[io_buffers_len++] = BufferGetBlock(buffers[i]);
    1513             :         }
    1514             : 
    1515     2277370 :         io_start = pgstat_prepare_io_time(track_io_timing);
    1516     2277370 :         smgrreadv(operation->smgr, forknum, io_first_block, io_pages, io_buffers_len);
    1517     2277340 :         pgstat_count_io_op_time(io_object, io_context, IOOP_READ, io_start,
    1518             :                                 io_buffers_len);
    1519             : 
    1520             :         /* Verify each block we read, and terminate the I/O. */
    1521     4792970 :         for (int j = 0; j < io_buffers_len; ++j)
    1522             :         {
    1523             :             BufferDesc *bufHdr;
    1524             :             Block       bufBlock;
    1525             : 
    1526     2515630 :             if (persistence == RELPERSISTENCE_TEMP)
    1527             :             {
    1528        7656 :                 bufHdr = GetLocalBufferDescriptor(-io_buffers[j] - 1);
    1529        7656 :                 bufBlock = LocalBufHdrGetBlock(bufHdr);
    1530             :             }
    1531             :             else
    1532             :             {
    1533     2507974 :                 bufHdr = GetBufferDescriptor(io_buffers[j] - 1);
    1534     2507974 :                 bufBlock = BufHdrGetBlock(bufHdr);
    1535             :             }
    1536             : 
    1537             :             /* check for garbage data */
    1538     2515630 :             if (!PageIsVerifiedExtended((Page) bufBlock, io_first_block + j,
    1539             :                                         PIV_LOG_WARNING | PIV_REPORT_STAT))
    1540             :             {
    1541           0 :                 if ((operation->flags & READ_BUFFERS_ZERO_ON_ERROR) || zero_damaged_pages)
    1542             :                 {
    1543           0 :                     ereport(WARNING,
    1544             :                             (errcode(ERRCODE_DATA_CORRUPTED),
    1545             :                              errmsg("invalid page in block %u of relation %s; zeroing out page",
    1546             :                                     io_first_block + j,
    1547             :                                     relpath(operation->smgr->smgr_rlocator, forknum))));
    1548           0 :                     memset(bufBlock, 0, BLCKSZ);
    1549             :                 }
    1550             :                 else
    1551           0 :                     ereport(ERROR,
    1552             :                             (errcode(ERRCODE_DATA_CORRUPTED),
    1553             :                              errmsg("invalid page in block %u of relation %s",
    1554             :                                     io_first_block + j,
    1555             :                                     relpath(operation->smgr->smgr_rlocator, forknum))));
    1556             :             }
    1557             : 
    1558             :             /* Terminate I/O and set BM_VALID. */
    1559     2515630 :             if (persistence == RELPERSISTENCE_TEMP)
    1560             :             {
    1561        7656 :                 uint32      buf_state = pg_atomic_read_u32(&bufHdr->state);
    1562             : 
    1563        7656 :                 buf_state |= BM_VALID;
    1564        7656 :                 pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
    1565             :             }
    1566             :             else
    1567             :             {
    1568             :                 /* Set BM_VALID, terminate IO, and wake up any waiters */
    1569     2507974 :                 TerminateBufferIO(bufHdr, false, BM_VALID, true);
    1570             :             }
    1571             : 
    1572             :             /* Report I/Os as completing individually. */
    1573             :             TRACE_POSTGRESQL_BUFFER_READ_DONE(forknum, io_first_block + j,
    1574             :                                               operation->smgr->smgr_rlocator.locator.spcOid,
    1575             :                                               operation->smgr->smgr_rlocator.locator.dbOid,
    1576             :                                               operation->smgr->smgr_rlocator.locator.relNumber,
    1577             :                                               operation->smgr->smgr_rlocator.backend,
    1578             :                                               false);
    1579             :         }
    1580             : 
    1581     2277340 :         if (VacuumCostActive)
    1582       10272 :             VacuumCostBalance += VacuumCostPageMiss * io_buffers_len;
    1583             :     }
    1584             : }
    1585             : 
    1586             : /*
    1587             :  * BufferAlloc -- subroutine for PinBufferForBlock.  Handles lookup of a shared
    1588             :  *      buffer.  If no buffer exists already, selects a replacement victim and
    1589             :  *      evicts the old page, but does NOT read in new page.
    1590             :  *
    1591             :  * "strategy" can be a buffer replacement strategy object, or NULL for
    1592             :  * the default strategy.  The selected buffer's usage_count is advanced when
    1593             :  * using the default strategy, but otherwise possibly not (see PinBuffer).
    1594             :  *
    1595             :  * The returned buffer is pinned and is already marked as holding the
    1596             :  * desired page.  If it already did have the desired page, *foundPtr is
    1597             :  * set true.  Otherwise, *foundPtr is set false.
    1598             :  *
    1599             :  * io_context is passed as an output parameter to avoid calling
    1600             :  * IOContextForStrategy() when there is a shared buffers hit and no IO
    1601             :  * statistics need be captured.
    1602             :  *
    1603             :  * No locks are held either at entry or exit.
    1604             :  */
    1605             : static pg_attribute_always_inline BufferDesc *
    1606   106991416 : BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
    1607             :             BlockNumber blockNum,
    1608             :             BufferAccessStrategy strategy,
    1609             :             bool *foundPtr, IOContext io_context)
    1610             : {
    1611             :     BufferTag   newTag;         /* identity of requested block */
    1612             :     uint32      newHash;        /* hash value for newTag */
    1613             :     LWLock     *newPartitionLock;   /* buffer partition lock for it */
    1614             :     int         existing_buf_id;
    1615             :     Buffer      victim_buffer;
    1616             :     BufferDesc *victim_buf_hdr;
    1617             :     uint32      victim_buf_state;
    1618             : 
    1619             :     /* Make sure we will have room to remember the buffer pin */
    1620   106991416 :     ResourceOwnerEnlarge(CurrentResourceOwner);
    1621   106991416 :     ReservePrivateRefCountEntry();
    1622             : 
    1623             :     /* create a tag so we can lookup the buffer */
    1624   106991416 :     InitBufferTag(&newTag, &smgr->smgr_rlocator.locator, forkNum, blockNum);
    1625             : 
    1626             :     /* determine its hash code and partition lock ID */
    1627   106991416 :     newHash = BufTableHashCode(&newTag);
    1628   106991416 :     newPartitionLock = BufMappingPartitionLock(newHash);
    1629             : 
    1630             :     /* see if the block is in the buffer pool already */
    1631   106991416 :     LWLockAcquire(newPartitionLock, LW_SHARED);
    1632   106991416 :     existing_buf_id = BufTableLookup(&newTag, newHash);
    1633   106991416 :     if (existing_buf_id >= 0)
    1634             :     {
    1635             :         BufferDesc *buf;
    1636             :         bool        valid;
    1637             : 
    1638             :         /*
    1639             :          * Found it.  Now, pin the buffer so no one can steal it from the
    1640             :          * buffer pool, and check to see if the correct data has been loaded
    1641             :          * into the buffer.
    1642             :          */
    1643   104005336 :         buf = GetBufferDescriptor(existing_buf_id);
    1644             : 
    1645   104005336 :         valid = PinBuffer(buf, strategy);
    1646             : 
    1647             :         /* Can release the mapping lock as soon as we've pinned it */
    1648   104005336 :         LWLockRelease(newPartitionLock);
    1649             : 
    1650   104005336 :         *foundPtr = true;
    1651             : 
    1652   104005336 :         if (!valid)
    1653             :         {
    1654             :             /*
    1655             :              * We can only get here if (a) someone else is still reading in
    1656             :              * the page, (b) a previous read attempt failed, or (c) someone
    1657             :              * called StartReadBuffers() but not yet WaitReadBuffers().
    1658             :              */
    1659        3466 :             *foundPtr = false;
    1660             :         }
    1661             : 
    1662   104005336 :         return buf;
    1663             :     }
    1664             : 
    1665             :     /*
    1666             :      * Didn't find it in the buffer pool.  We'll have to initialize a new
    1667             :      * buffer.  Remember to unlock the mapping lock while doing the work.
    1668             :      */
    1669     2986080 :     LWLockRelease(newPartitionLock);
    1670             : 
    1671             :     /*
    1672             :      * Acquire a victim buffer. Somebody else might try to do the same, we
    1673             :      * don't hold any conflicting locks. If so we'll have to undo our work
    1674             :      * later.
    1675             :      */
    1676     2986080 :     victim_buffer = GetVictimBuffer(strategy, io_context);
    1677     2986080 :     victim_buf_hdr = GetBufferDescriptor(victim_buffer - 1);
    1678             : 
    1679             :     /*
    1680             :      * Try to make a hashtable entry for the buffer under its new tag. If
    1681             :      * somebody else inserted another buffer for the tag, we'll release the
    1682             :      * victim buffer we acquired and use the already inserted one.
    1683             :      */
    1684     2986080 :     LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
    1685     2986080 :     existing_buf_id = BufTableInsert(&newTag, newHash, victim_buf_hdr->buf_id);
    1686     2986080 :     if (existing_buf_id >= 0)
    1687             :     {
    1688             :         BufferDesc *existing_buf_hdr;
    1689             :         bool        valid;
    1690             : 
    1691             :         /*
    1692             :          * Got a collision. Someone has already done what we were about to do.
    1693             :          * We'll just handle this as if it were found in the buffer pool in
    1694             :          * the first place.  First, give up the buffer we were planning to
    1695             :          * use.
    1696             :          *
    1697             :          * We could do this after releasing the partition lock, but then we'd
    1698             :          * have to call ResourceOwnerEnlarge() & ReservePrivateRefCountEntry()
    1699             :          * before acquiring the lock, for the rare case of such a collision.
    1700             :          */
    1701         614 :         UnpinBuffer(victim_buf_hdr);
    1702             : 
    1703             :         /*
    1704             :          * The victim buffer we acquired previously is clean and unused, let
    1705             :          * it be found again quickly
    1706             :          */
    1707         614 :         StrategyFreeBuffer(victim_buf_hdr);
    1708             : 
    1709             :         /* remaining code should match code at top of routine */
    1710             : 
    1711         614 :         existing_buf_hdr = GetBufferDescriptor(existing_buf_id);
    1712             : 
    1713         614 :         valid = PinBuffer(existing_buf_hdr, strategy);
    1714             : 
    1715             :         /* Can release the mapping lock as soon as we've pinned it */
    1716         614 :         LWLockRelease(newPartitionLock);
    1717             : 
    1718         614 :         *foundPtr = true;
    1719             : 
    1720         614 :         if (!valid)
    1721             :         {
    1722             :             /*
    1723             :              * We can only get here if (a) someone else is still reading in
    1724             :              * the page, (b) a previous read attempt failed, or (c) someone
    1725             :              * called StartReadBuffers() but not yet WaitReadBuffers().
    1726             :              */
    1727         318 :             *foundPtr = false;
    1728             :         }
    1729             : 
    1730         614 :         return existing_buf_hdr;
    1731             :     }
    1732             : 
    1733             :     /*
    1734             :      * Need to lock the buffer header too in order to change its tag.
    1735             :      */
    1736     2985466 :     victim_buf_state = LockBufHdr(victim_buf_hdr);
    1737             : 
    1738             :     /* some sanity checks while we hold the buffer header lock */
    1739             :     Assert(BUF_STATE_GET_REFCOUNT(victim_buf_state) == 1);
    1740             :     Assert(!(victim_buf_state & (BM_TAG_VALID | BM_VALID | BM_DIRTY | BM_IO_IN_PROGRESS)));
    1741             : 
    1742     2985466 :     victim_buf_hdr->tag = newTag;
    1743             : 
    1744             :     /*
    1745             :      * Make sure BM_PERMANENT is set for buffers that must be written at every
    1746             :      * checkpoint.  Unlogged buffers only need to be written at shutdown
    1747             :      * checkpoints, except for their "init" forks, which need to be treated
    1748             :      * just like permanent relations.
    1749             :      */
    1750     2985466 :     victim_buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
    1751     2985466 :     if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM)
    1752     2984796 :         victim_buf_state |= BM_PERMANENT;
    1753             : 
    1754     2985466 :     UnlockBufHdr(victim_buf_hdr, victim_buf_state);
    1755             : 
    1756     2985466 :     LWLockRelease(newPartitionLock);
    1757             : 
    1758             :     /*
    1759             :      * Buffer contents are currently invalid.
    1760             :      */
    1761     2985466 :     *foundPtr = false;
    1762             : 
    1763     2985466 :     return victim_buf_hdr;
    1764             : }
    1765             : 
    1766             : /*
    1767             :  * InvalidateBuffer -- mark a shared buffer invalid and return it to the
    1768             :  * freelist.
    1769             :  *
    1770             :  * The buffer header spinlock must be held at entry.  We drop it before
    1771             :  * returning.  (This is sane because the caller must have locked the
    1772             :  * buffer in order to be sure it should be dropped.)
    1773             :  *
    1774             :  * This is used only in contexts such as dropping a relation.  We assume
    1775             :  * that no other backend could possibly be interested in using the page,
    1776             :  * so the only reason the buffer might be pinned is if someone else is
    1777             :  * trying to write it out.  We have to let them finish before we can
    1778             :  * reclaim the buffer.
    1779             :  *
    1780             :  * The buffer could get reclaimed by someone else while we are waiting
    1781             :  * to acquire the necessary locks; if so, don't mess it up.
    1782             :  */
    1783             : static void
    1784      196860 : InvalidateBuffer(BufferDesc *buf)
    1785             : {
    1786             :     BufferTag   oldTag;
    1787             :     uint32      oldHash;        /* hash value for oldTag */
    1788             :     LWLock     *oldPartitionLock;   /* buffer partition lock for it */
    1789             :     uint32      oldFlags;
    1790             :     uint32      buf_state;
    1791             : 
    1792             :     /* Save the original buffer tag before dropping the spinlock */
    1793      196860 :     oldTag = buf->tag;
    1794             : 
    1795      196860 :     buf_state = pg_atomic_read_u32(&buf->state);
    1796             :     Assert(buf_state & BM_LOCKED);
    1797      196860 :     UnlockBufHdr(buf, buf_state);
    1798             : 
    1799             :     /*
    1800             :      * Need to compute the old tag's hashcode and partition lock ID. XXX is it
    1801             :      * worth storing the hashcode in BufferDesc so we need not recompute it
    1802             :      * here?  Probably not.
    1803             :      */
    1804      196860 :     oldHash = BufTableHashCode(&oldTag);
    1805      196860 :     oldPartitionLock = BufMappingPartitionLock(oldHash);
    1806             : 
    1807      196866 : retry:
    1808             : 
    1809             :     /*
    1810             :      * Acquire exclusive mapping lock in preparation for changing the buffer's
    1811             :      * association.
    1812             :      */
    1813      196866 :     LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
    1814             : 
    1815             :     /* Re-lock the buffer header */
    1816      196866 :     buf_state = LockBufHdr(buf);
    1817             : 
    1818             :     /* If it's changed while we were waiting for lock, do nothing */
    1819      196866 :     if (!BufferTagsEqual(&buf->tag, &oldTag))
    1820             :     {
    1821           6 :         UnlockBufHdr(buf, buf_state);
    1822           6 :         LWLockRelease(oldPartitionLock);
    1823           6 :         return;
    1824             :     }
    1825             : 
    1826             :     /*
    1827             :      * We assume the only reason for it to be pinned is that someone else is
    1828             :      * flushing the page out.  Wait for them to finish.  (This could be an
    1829             :      * infinite loop if the refcount is messed up... it would be nice to time
    1830             :      * out after awhile, but there seems no way to be sure how many loops may
    1831             :      * be needed.  Note that if the other guy has pinned the buffer but not
    1832             :      * yet done StartBufferIO, WaitIO will fall through and we'll effectively
    1833             :      * be busy-looping here.)
    1834             :      */
    1835      196860 :     if (BUF_STATE_GET_REFCOUNT(buf_state) != 0)
    1836             :     {
    1837           6 :         UnlockBufHdr(buf, buf_state);
    1838           6 :         LWLockRelease(oldPartitionLock);
    1839             :         /* safety check: should definitely not be our *own* pin */
    1840           6 :         if (GetPrivateRefCount(BufferDescriptorGetBuffer(buf)) > 0)
    1841           0 :             elog(ERROR, "buffer is pinned in InvalidateBuffer");
    1842           6 :         WaitIO(buf);
    1843           6 :         goto retry;
    1844             :     }
    1845             : 
    1846             :     /*
    1847             :      * Clear out the buffer's tag and flags.  We must do this to ensure that
    1848             :      * linear scans of the buffer array don't think the buffer is valid.
    1849             :      */
    1850      196854 :     oldFlags = buf_state & BUF_FLAG_MASK;
    1851      196854 :     ClearBufferTag(&buf->tag);
    1852      196854 :     buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK);
    1853      196854 :     UnlockBufHdr(buf, buf_state);
    1854             : 
    1855             :     /*
    1856             :      * Remove the buffer from the lookup hashtable, if it was in there.
    1857             :      */
    1858      196854 :     if (oldFlags & BM_TAG_VALID)
    1859      196854 :         BufTableDelete(&oldTag, oldHash);
    1860             : 
    1861             :     /*
    1862             :      * Done with mapping lock.
    1863             :      */
    1864      196854 :     LWLockRelease(oldPartitionLock);
    1865             : 
    1866             :     /*
    1867             :      * Insert the buffer at the head of the list of free buffers.
    1868             :      */
    1869      196854 :     StrategyFreeBuffer(buf);
    1870             : }
    1871             : 
    1872             : /*
    1873             :  * Helper routine for GetVictimBuffer()
    1874             :  *
    1875             :  * Needs to be called on a buffer with a valid tag, pinned, but without the
    1876             :  * buffer header spinlock held.
    1877             :  *
    1878             :  * Returns true if the buffer can be reused, in which case the buffer is only
    1879             :  * pinned by this backend and marked as invalid, false otherwise.
    1880             :  */
    1881             : static bool
    1882     2226986 : InvalidateVictimBuffer(BufferDesc *buf_hdr)
    1883             : {
    1884             :     uint32      buf_state;
    1885             :     uint32      hash;
    1886             :     LWLock     *partition_lock;
    1887             :     BufferTag   tag;
    1888             : 
    1889             :     Assert(GetPrivateRefCount(BufferDescriptorGetBuffer(buf_hdr)) == 1);
    1890             : 
    1891             :     /* have buffer pinned, so it's safe to read tag without lock */
    1892     2226986 :     tag = buf_hdr->tag;
    1893             : 
    1894     2226986 :     hash = BufTableHashCode(&tag);
    1895     2226986 :     partition_lock = BufMappingPartitionLock(hash);
    1896             : 
    1897     2226986 :     LWLockAcquire(partition_lock, LW_EXCLUSIVE);
    1898             : 
    1899             :     /* lock the buffer header */
    1900     2226986 :     buf_state = LockBufHdr(buf_hdr);
    1901             : 
    1902             :     /*
    1903             :      * We have the buffer pinned nobody else should have been able to unset
    1904             :      * this concurrently.
    1905             :      */
    1906             :     Assert(buf_state & BM_TAG_VALID);
    1907             :     Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    1908             :     Assert(BufferTagsEqual(&buf_hdr->tag, &tag));
    1909             : 
    1910             :     /*
    1911             :      * If somebody else pinned the buffer since, or even worse, dirtied it,
    1912             :      * give up on this buffer: It's clearly in use.
    1913             :      */
    1914     2226986 :     if (BUF_STATE_GET_REFCOUNT(buf_state) != 1 || (buf_state & BM_DIRTY))
    1915             :     {
    1916             :         Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    1917             : 
    1918         598 :         UnlockBufHdr(buf_hdr, buf_state);
    1919         598 :         LWLockRelease(partition_lock);
    1920             : 
    1921         598 :         return false;
    1922             :     }
    1923             : 
    1924             :     /*
    1925             :      * Clear out the buffer's tag and flags and usagecount.  This is not
    1926             :      * strictly required, as BM_TAG_VALID/BM_VALID needs to be checked before
    1927             :      * doing anything with the buffer. But currently it's beneficial, as the
    1928             :      * cheaper pre-check for several linear scans of shared buffers use the
    1929             :      * tag (see e.g. FlushDatabaseBuffers()).
    1930             :      */
    1931     2226388 :     ClearBufferTag(&buf_hdr->tag);
    1932     2226388 :     buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK);
    1933     2226388 :     UnlockBufHdr(buf_hdr, buf_state);
    1934             : 
    1935             :     Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    1936             : 
    1937             :     /* finally delete buffer from the buffer mapping table */
    1938     2226388 :     BufTableDelete(&tag, hash);
    1939             : 
    1940     2226388 :     LWLockRelease(partition_lock);
    1941             : 
    1942             :     Assert(!(buf_state & (BM_DIRTY | BM_VALID | BM_TAG_VALID)));
    1943             :     Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    1944             :     Assert(BUF_STATE_GET_REFCOUNT(pg_atomic_read_u32(&buf_hdr->state)) > 0);
    1945             : 
    1946     2226388 :     return true;
    1947             : }
    1948             : 
    1949             : static Buffer
    1950     3399594 : GetVictimBuffer(BufferAccessStrategy strategy, IOContext io_context)
    1951             : {
    1952             :     BufferDesc *buf_hdr;
    1953             :     Buffer      buf;
    1954             :     uint32      buf_state;
    1955             :     bool        from_ring;
    1956             : 
    1957             :     /*
    1958             :      * Ensure, while the spinlock's not yet held, that there's a free refcount
    1959             :      * entry, and a resource owner slot for the pin.
    1960             :      */
    1961     3399594 :     ReservePrivateRefCountEntry();
    1962     3399594 :     ResourceOwnerEnlarge(CurrentResourceOwner);
    1963             : 
    1964             :     /* we return here if a prospective victim buffer gets used concurrently */
    1965     3411750 : again:
    1966             : 
    1967             :     /*
    1968             :      * Select a victim buffer.  The buffer is returned with its header
    1969             :      * spinlock still held!
    1970             :      */
    1971     3411750 :     buf_hdr = StrategyGetBuffer(strategy, &buf_state, &from_ring);
    1972     3411750 :     buf = BufferDescriptorGetBuffer(buf_hdr);
    1973             : 
    1974             :     Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 0);
    1975             : 
    1976             :     /* Pin the buffer and then release the buffer spinlock */
    1977     3411750 :     PinBuffer_Locked(buf_hdr);
    1978             : 
    1979             :     /*
    1980             :      * We shouldn't have any other pins for this buffer.
    1981             :      */
    1982     3411750 :     CheckBufferIsPinnedOnce(buf);
    1983             : 
    1984             :     /*
    1985             :      * If the buffer was dirty, try to write it out.  There is a race
    1986             :      * condition here, in that someone might dirty it after we released the
    1987             :      * buffer header lock above, or even while we are writing it out (since
    1988             :      * our share-lock won't prevent hint-bit updates).  We will recheck the
    1989             :      * dirty bit after re-locking the buffer header.
    1990             :      */
    1991     3411750 :     if (buf_state & BM_DIRTY)
    1992             :     {
    1993             :         LWLock     *content_lock;
    1994             : 
    1995             :         Assert(buf_state & BM_TAG_VALID);
    1996             :         Assert(buf_state & BM_VALID);
    1997             : 
    1998             :         /*
    1999             :          * We need a share-lock on the buffer contents to write it out (else
    2000             :          * we might write invalid data, eg because someone else is compacting
    2001             :          * the page contents while we write).  We must use a conditional lock
    2002             :          * acquisition here to avoid deadlock.  Even though the buffer was not
    2003             :          * pinned (and therefore surely not locked) when StrategyGetBuffer
    2004             :          * returned it, someone else could have pinned and exclusive-locked it
    2005             :          * by the time we get here. If we try to get the lock unconditionally,
    2006             :          * we'd block waiting for them; if they later block waiting for us,
    2007             :          * deadlock ensues. (This has been observed to happen when two
    2008             :          * backends are both trying to split btree index pages, and the second
    2009             :          * one just happens to be trying to split the page the first one got
    2010             :          * from StrategyGetBuffer.)
    2011             :          */
    2012      456198 :         content_lock = BufferDescriptorGetContentLock(buf_hdr);
    2013      456198 :         if (!LWLockConditionalAcquire(content_lock, LW_SHARED))
    2014             :         {
    2015             :             /*
    2016             :              * Someone else has locked the buffer, so give it up and loop back
    2017             :              * to get another one.
    2018             :              */
    2019           0 :             UnpinBuffer(buf_hdr);
    2020           0 :             goto again;
    2021             :         }
    2022             : 
    2023             :         /*
    2024             :          * If using a nondefault strategy, and writing the buffer would
    2025             :          * require a WAL flush, let the strategy decide whether to go ahead
    2026             :          * and write/reuse the buffer or to choose another victim.  We need a
    2027             :          * lock to inspect the page LSN, so this can't be done inside
    2028             :          * StrategyGetBuffer.
    2029             :          */
    2030      456198 :         if (strategy != NULL)
    2031             :         {
    2032             :             XLogRecPtr  lsn;
    2033             : 
    2034             :             /* Read the LSN while holding buffer header lock */
    2035      129334 :             buf_state = LockBufHdr(buf_hdr);
    2036      129334 :             lsn = BufferGetLSN(buf_hdr);
    2037      129334 :             UnlockBufHdr(buf_hdr, buf_state);
    2038             : 
    2039      129334 :             if (XLogNeedsFlush(lsn)
    2040       15970 :                 && StrategyRejectBuffer(strategy, buf_hdr, from_ring))
    2041             :             {
    2042       11558 :                 LWLockRelease(content_lock);
    2043       11558 :                 UnpinBuffer(buf_hdr);
    2044       11558 :                 goto again;
    2045             :             }
    2046             :         }
    2047             : 
    2048             :         /* OK, do the I/O */
    2049      444640 :         FlushBuffer(buf_hdr, NULL, IOOBJECT_RELATION, io_context);
    2050      444640 :         LWLockRelease(content_lock);
    2051             : 
    2052      444640 :         ScheduleBufferTagForWriteback(&BackendWritebackContext, io_context,
    2053             :                                       &buf_hdr->tag);
    2054             :     }
    2055             : 
    2056             : 
    2057     3400192 :     if (buf_state & BM_VALID)
    2058             :     {
    2059             :         /*
    2060             :          * When a BufferAccessStrategy is in use, blocks evicted from shared
    2061             :          * buffers are counted as IOOP_EVICT in the corresponding context
    2062             :          * (e.g. IOCONTEXT_BULKWRITE). Shared buffers are evicted by a
    2063             :          * strategy in two cases: 1) while initially claiming buffers for the
    2064             :          * strategy ring 2) to replace an existing strategy ring buffer
    2065             :          * because it is pinned or in use and cannot be reused.
    2066             :          *
    2067             :          * Blocks evicted from buffers already in the strategy ring are
    2068             :          * counted as IOOP_REUSE in the corresponding strategy context.
    2069             :          *
    2070             :          * At this point, we can accurately count evictions and reuses,
    2071             :          * because we have successfully claimed the valid buffer. Previously,
    2072             :          * we may have been forced to release the buffer due to concurrent
    2073             :          * pinners or erroring out.
    2074             :          */
    2075     2226984 :         pgstat_count_io_op(IOOBJECT_RELATION, io_context,
    2076     2226984 :                            from_ring ? IOOP_REUSE : IOOP_EVICT);
    2077             :     }
    2078             : 
    2079             :     /*
    2080             :      * If the buffer has an entry in the buffer mapping table, delete it. This
    2081             :      * can fail because another backend could have pinned or dirtied the
    2082             :      * buffer.
    2083             :      */
    2084     3400192 :     if ((buf_state & BM_TAG_VALID) && !InvalidateVictimBuffer(buf_hdr))
    2085             :     {
    2086         598 :         UnpinBuffer(buf_hdr);
    2087         598 :         goto again;
    2088             :     }
    2089             : 
    2090             :     /* a final set of sanity checks */
    2091             : #ifdef USE_ASSERT_CHECKING
    2092             :     buf_state = pg_atomic_read_u32(&buf_hdr->state);
    2093             : 
    2094             :     Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 1);
    2095             :     Assert(!(buf_state & (BM_TAG_VALID | BM_VALID | BM_DIRTY)));
    2096             : 
    2097             :     CheckBufferIsPinnedOnce(buf);
    2098             : #endif
    2099             : 
    2100     3399594 :     return buf;
    2101             : }
    2102             : 
    2103             : /*
    2104             :  * Limit the number of pins a batch operation may additionally acquire, to
    2105             :  * avoid running out of pinnable buffers.
    2106             :  *
    2107             :  * One additional pin is always allowed, as otherwise the operation likely
    2108             :  * cannot be performed at all.
    2109             :  *
    2110             :  * The number of allowed pins for a backend is computed based on
    2111             :  * shared_buffers and the maximum number of connections possible. That's very
    2112             :  * pessimistic, but outside of toy-sized shared_buffers it should allow
    2113             :  * sufficient pins.
    2114             :  */
    2115             : void
    2116     1124396 : LimitAdditionalPins(uint32 *additional_pins)
    2117             : {
    2118             :     uint32      max_backends;
    2119             :     int         max_proportional_pins;
    2120             : 
    2121     1124396 :     if (*additional_pins <= 1)
    2122      354612 :         return;
    2123             : 
    2124      769784 :     max_backends = MaxBackends + NUM_AUXILIARY_PROCS;
    2125      769784 :     max_proportional_pins = NBuffers / max_backends;
    2126             : 
    2127             :     /*
    2128             :      * Subtract the approximate number of buffers already pinned by this
    2129             :      * backend. We get the number of "overflowed" pins for free, but don't
    2130             :      * know the number of pins in PrivateRefCountArray. The cost of
    2131             :      * calculating that exactly doesn't seem worth it, so just assume the max.
    2132             :      */
    2133      769784 :     max_proportional_pins -= PrivateRefCountOverflowed + REFCOUNT_ARRAY_ENTRIES;
    2134             : 
    2135      769784 :     if (max_proportional_pins <= 0)
    2136      165530 :         max_proportional_pins = 1;
    2137             : 
    2138      769784 :     if (*additional_pins > max_proportional_pins)
    2139      166192 :         *additional_pins = max_proportional_pins;
    2140             : }
    2141             : 
    2142             : /*
    2143             :  * Logic shared between ExtendBufferedRelBy(), ExtendBufferedRelTo(). Just to
    2144             :  * avoid duplicating the tracing and relpersistence related logic.
    2145             :  */
    2146             : static BlockNumber
    2147      390792 : ExtendBufferedRelCommon(BufferManagerRelation bmr,
    2148             :                         ForkNumber fork,
    2149             :                         BufferAccessStrategy strategy,
    2150             :                         uint32 flags,
    2151             :                         uint32 extend_by,
    2152             :                         BlockNumber extend_upto,
    2153             :                         Buffer *buffers,
    2154             :                         uint32 *extended_by)
    2155             : {
    2156             :     BlockNumber first_block;
    2157             : 
    2158             :     TRACE_POSTGRESQL_BUFFER_EXTEND_START(fork,
    2159             :                                          bmr.smgr->smgr_rlocator.locator.spcOid,
    2160             :                                          bmr.smgr->smgr_rlocator.locator.dbOid,
    2161             :                                          bmr.smgr->smgr_rlocator.locator.relNumber,
    2162             :                                          bmr.smgr->smgr_rlocator.backend,
    2163             :                                          extend_by);
    2164             : 
    2165      390792 :     if (bmr.relpersistence == RELPERSISTENCE_TEMP)
    2166       17658 :         first_block = ExtendBufferedRelLocal(bmr, fork, flags,
    2167             :                                              extend_by, extend_upto,
    2168             :                                              buffers, &extend_by);
    2169             :     else
    2170      373134 :         first_block = ExtendBufferedRelShared(bmr, fork, strategy, flags,
    2171             :                                               extend_by, extend_upto,
    2172             :                                               buffers, &extend_by);
    2173      390792 :     *extended_by = extend_by;
    2174             : 
    2175             :     TRACE_POSTGRESQL_BUFFER_EXTEND_DONE(fork,
    2176             :                                         bmr.smgr->smgr_rlocator.locator.spcOid,
    2177             :                                         bmr.smgr->smgr_rlocator.locator.dbOid,
    2178             :                                         bmr.smgr->smgr_rlocator.locator.relNumber,
    2179             :                                         bmr.smgr->smgr_rlocator.backend,
    2180             :                                         *extended_by,
    2181             :                                         first_block);
    2182             : 
    2183      390792 :     return first_block;
    2184             : }
    2185             : 
    2186             : /*
    2187             :  * Implementation of ExtendBufferedRelBy() and ExtendBufferedRelTo() for
    2188             :  * shared buffers.
    2189             :  */
    2190             : static BlockNumber
    2191      373134 : ExtendBufferedRelShared(BufferManagerRelation bmr,
    2192             :                         ForkNumber fork,
    2193             :                         BufferAccessStrategy strategy,
    2194             :                         uint32 flags,
    2195             :                         uint32 extend_by,
    2196             :                         BlockNumber extend_upto,
    2197             :                         Buffer *buffers,
    2198             :                         uint32 *extended_by)
    2199             : {
    2200             :     BlockNumber first_block;
    2201      373134 :     IOContext   io_context = IOContextForStrategy(strategy);
    2202             :     instr_time  io_start;
    2203             : 
    2204      373134 :     LimitAdditionalPins(&extend_by);
    2205             : 
    2206             :     /*
    2207             :      * Acquire victim buffers for extension without holding extension lock.
    2208             :      * Writing out victim buffers is the most expensive part of extending the
    2209             :      * relation, particularly when doing so requires WAL flushes. Zeroing out
    2210             :      * the buffers is also quite expensive, so do that before holding the
    2211             :      * extension lock as well.
    2212             :      *
    2213             :      * These pages are pinned by us and not valid. While we hold the pin they
    2214             :      * can't be acquired as victim buffers by another backend.
    2215             :      */
    2216      786648 :     for (uint32 i = 0; i < extend_by; i++)
    2217             :     {
    2218             :         Block       buf_block;
    2219             : 
    2220      413514 :         buffers[i] = GetVictimBuffer(strategy, io_context);
    2221      413514 :         buf_block = BufHdrGetBlock(GetBufferDescriptor(buffers[i] - 1));
    2222             : 
    2223             :         /* new buffers are zero-filled */
    2224      413514 :         MemSet((char *) buf_block, 0, BLCKSZ);
    2225             :     }
    2226             : 
    2227             :     /*
    2228             :      * Lock relation against concurrent extensions, unless requested not to.
    2229             :      *
    2230             :      * We use the same extension lock for all forks. That's unnecessarily
    2231             :      * restrictive, but currently extensions for forks don't happen often
    2232             :      * enough to make it worth locking more granularly.
    2233             :      *
    2234             :      * Note that another backend might have extended the relation by the time
    2235             :      * we get the lock.
    2236             :      */
    2237      373134 :     if (!(flags & EB_SKIP_EXTENSION_LOCK))
    2238      274676 :         LockRelationForExtension(bmr.rel, ExclusiveLock);
    2239             : 
    2240             :     /*
    2241             :      * If requested, invalidate size cache, so that smgrnblocks asks the
    2242             :      * kernel.
    2243             :      */
    2244      373134 :     if (flags & EB_CLEAR_SIZE_CACHE)
    2245       14036 :         bmr.smgr->smgr_cached_nblocks[fork] = InvalidBlockNumber;
    2246             : 
    2247      373134 :     first_block = smgrnblocks(bmr.smgr, fork);
    2248             : 
    2249             :     /*
    2250             :      * Now that we have the accurate relation size, check if the caller wants
    2251             :      * us to extend to only up to a specific size. If there were concurrent
    2252             :      * extensions, we might have acquired too many buffers and need to release
    2253             :      * them.
    2254             :      */
    2255      373134 :     if (extend_upto != InvalidBlockNumber)
    2256             :     {
    2257      100866 :         uint32      orig_extend_by = extend_by;
    2258             : 
    2259      100866 :         if (first_block > extend_upto)
    2260           0 :             extend_by = 0;
    2261      100866 :         else if ((uint64) first_block + extend_by > extend_upto)
    2262           2 :             extend_by = extend_upto - first_block;
    2263             : 
    2264      100868 :         for (uint32 i = extend_by; i < orig_extend_by; i++)
    2265             :         {
    2266           2 :             BufferDesc *buf_hdr = GetBufferDescriptor(buffers[i] - 1);
    2267             : 
    2268             :             /*
    2269             :              * The victim buffer we acquired previously is clean and unused,
    2270             :              * let it be found again quickly
    2271             :              */
    2272           2 :             StrategyFreeBuffer(buf_hdr);
    2273           2 :             UnpinBuffer(buf_hdr);
    2274             :         }
    2275             : 
    2276      100866 :         if (extend_by == 0)
    2277             :         {
    2278           2 :             if (!(flags & EB_SKIP_EXTENSION_LOCK))
    2279           2 :                 UnlockRelationForExtension(bmr.rel, ExclusiveLock);
    2280           2 :             *extended_by = extend_by;
    2281           2 :             return first_block;
    2282             :         }
    2283             :     }
    2284             : 
    2285             :     /* Fail if relation is already at maximum possible length */
    2286      373132 :     if ((uint64) first_block + extend_by >= MaxBlockNumber)
    2287           0 :         ereport(ERROR,
    2288             :                 (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
    2289             :                  errmsg("cannot extend relation %s beyond %u blocks",
    2290             :                         relpath(bmr.smgr->smgr_rlocator, fork),
    2291             :                         MaxBlockNumber)));
    2292             : 
    2293             :     /*
    2294             :      * Insert buffers into buffer table, mark as IO_IN_PROGRESS.
    2295             :      *
    2296             :      * This needs to happen before we extend the relation, because as soon as
    2297             :      * we do, other backends can start to read in those pages.
    2298             :      */
    2299      786644 :     for (uint32 i = 0; i < extend_by; i++)
    2300             :     {
    2301      413512 :         Buffer      victim_buf = buffers[i];
    2302      413512 :         BufferDesc *victim_buf_hdr = GetBufferDescriptor(victim_buf - 1);
    2303             :         BufferTag   tag;
    2304             :         uint32      hash;
    2305             :         LWLock     *partition_lock;
    2306             :         int         existing_id;
    2307             : 
    2308             :         /* in case we need to pin an existing buffer below */
    2309      413512 :         ResourceOwnerEnlarge(CurrentResourceOwner);
    2310      413512 :         ReservePrivateRefCountEntry();
    2311             : 
    2312      413512 :         InitBufferTag(&tag, &bmr.smgr->smgr_rlocator.locator, fork, first_block + i);
    2313      413512 :         hash = BufTableHashCode(&tag);
    2314      413512 :         partition_lock = BufMappingPartitionLock(hash);
    2315             : 
    2316      413512 :         LWLockAcquire(partition_lock, LW_EXCLUSIVE);
    2317             : 
    2318      413512 :         existing_id = BufTableInsert(&tag, hash, victim_buf_hdr->buf_id);
    2319             : 
    2320             :         /*
    2321             :          * We get here only in the corner case where we are trying to extend
    2322             :          * the relation but we found a pre-existing buffer. This can happen
    2323             :          * because a prior attempt at extending the relation failed, and
    2324             :          * because mdread doesn't complain about reads beyond EOF (when
    2325             :          * zero_damaged_pages is ON) and so a previous attempt to read a block
    2326             :          * beyond EOF could have left a "valid" zero-filled buffer.
    2327             :          * Unfortunately, we have also seen this case occurring because of
    2328             :          * buggy Linux kernels that sometimes return an lseek(SEEK_END) result
    2329             :          * that doesn't account for a recent write. In that situation, the
    2330             :          * pre-existing buffer would contain valid data that we don't want to
    2331             :          * overwrite.  Since the legitimate cases should always have left a
    2332             :          * zero-filled buffer, complain if not PageIsNew.
    2333             :          */
    2334      413512 :         if (existing_id >= 0)
    2335             :         {
    2336           0 :             BufferDesc *existing_hdr = GetBufferDescriptor(existing_id);
    2337             :             Block       buf_block;
    2338             :             bool        valid;
    2339             : 
    2340             :             /*
    2341             :              * Pin the existing buffer before releasing the partition lock,
    2342             :              * preventing it from being evicted.
    2343             :              */
    2344           0 :             valid = PinBuffer(existing_hdr, strategy);
    2345             : 
    2346           0 :             LWLockRelease(partition_lock);
    2347             : 
    2348             :             /*
    2349             :              * The victim buffer we acquired previously is clean and unused,
    2350             :              * let it be found again quickly
    2351             :              */
    2352           0 :             StrategyFreeBuffer(victim_buf_hdr);
    2353           0 :             UnpinBuffer(victim_buf_hdr);
    2354             : 
    2355           0 :             buffers[i] = BufferDescriptorGetBuffer(existing_hdr);
    2356           0 :             buf_block = BufHdrGetBlock(existing_hdr);
    2357             : 
    2358           0 :             if (valid && !PageIsNew((Page) buf_block))
    2359           0 :                 ereport(ERROR,
    2360             :                         (errmsg("unexpected data beyond EOF in block %u of relation %s",
    2361             :                                 existing_hdr->tag.blockNum, relpath(bmr.smgr->smgr_rlocator, fork)),
    2362             :                          errhint("This has been seen to occur with buggy kernels; consider updating your system.")));
    2363             : 
    2364             :             /*
    2365             :              * We *must* do smgr[zero]extend before succeeding, else the page
    2366             :              * will not be reserved by the kernel, and the next P_NEW call
    2367             :              * will decide to return the same page.  Clear the BM_VALID bit,
    2368             :              * do StartBufferIO() and proceed.
    2369             :              *
    2370             :              * Loop to handle the very small possibility that someone re-sets
    2371             :              * BM_VALID between our clearing it and StartBufferIO inspecting
    2372             :              * it.
    2373             :              */
    2374             :             do
    2375             :             {
    2376           0 :                 uint32      buf_state = LockBufHdr(existing_hdr);
    2377             : 
    2378           0 :                 buf_state &= ~BM_VALID;
    2379           0 :                 UnlockBufHdr(existing_hdr, buf_state);
    2380           0 :             } while (!StartBufferIO(existing_hdr, true, false));
    2381             :         }
    2382             :         else
    2383             :         {
    2384             :             uint32      buf_state;
    2385             : 
    2386      413512 :             buf_state = LockBufHdr(victim_buf_hdr);
    2387             : 
    2388             :             /* some sanity checks while we hold the buffer header lock */
    2389             :             Assert(!(buf_state & (BM_VALID | BM_TAG_VALID | BM_DIRTY | BM_JUST_DIRTIED)));
    2390             :             Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 1);
    2391             : 
    2392      413512 :             victim_buf_hdr->tag = tag;
    2393             : 
    2394      413512 :             buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
    2395      413512 :             if (bmr.relpersistence == RELPERSISTENCE_PERMANENT || fork == INIT_FORKNUM)
    2396      405344 :                 buf_state |= BM_PERMANENT;
    2397             : 
    2398      413512 :             UnlockBufHdr(victim_buf_hdr, buf_state);
    2399             : 
    2400      413512 :             LWLockRelease(partition_lock);
    2401             : 
    2402             :             /* XXX: could combine the locked operations in it with the above */
    2403      413512 :             StartBufferIO(victim_buf_hdr, true, false);
    2404             :         }
    2405             :     }
    2406             : 
    2407      373132 :     io_start = pgstat_prepare_io_time(track_io_timing);
    2408             : 
    2409             :     /*
    2410             :      * Note: if smgrzeroextend fails, we will end up with buffers that are
    2411             :      * allocated but not marked BM_VALID.  The next relation extension will
    2412             :      * still select the same block number (because the relation didn't get any
    2413             :      * longer on disk) and so future attempts to extend the relation will find
    2414             :      * the same buffers (if they have not been recycled) but come right back
    2415             :      * here to try smgrzeroextend again.
    2416             :      *
    2417             :      * We don't need to set checksum for all-zero pages.
    2418             :      */
    2419      373132 :     smgrzeroextend(bmr.smgr, fork, first_block, extend_by, false);
    2420             : 
    2421             :     /*
    2422             :      * Release the file-extension lock; it's now OK for someone else to extend
    2423             :      * the relation some more.
    2424             :      *
    2425             :      * We remove IO_IN_PROGRESS after this, as waking up waiting backends can
    2426             :      * take noticeable time.
    2427             :      */
    2428      373132 :     if (!(flags & EB_SKIP_EXTENSION_LOCK))
    2429      274674 :         UnlockRelationForExtension(bmr.rel, ExclusiveLock);
    2430             : 
    2431      373132 :     pgstat_count_io_op_time(IOOBJECT_RELATION, io_context, IOOP_EXTEND,
    2432             :                             io_start, extend_by);
    2433             : 
    2434             :     /* Set BM_VALID, terminate IO, and wake up any waiters */
    2435      786644 :     for (uint32 i = 0; i < extend_by; i++)
    2436             :     {
    2437      413512 :         Buffer      buf = buffers[i];
    2438      413512 :         BufferDesc *buf_hdr = GetBufferDescriptor(buf - 1);
    2439      413512 :         bool        lock = false;
    2440             : 
    2441      413512 :         if (flags & EB_LOCK_FIRST && i == 0)
    2442      271770 :             lock = true;
    2443      141742 :         else if (flags & EB_LOCK_TARGET)
    2444             :         {
    2445             :             Assert(extend_upto != InvalidBlockNumber);
    2446       84804 :             if (first_block + i + 1 == extend_upto)
    2447       83698 :                 lock = true;
    2448             :         }
    2449             : 
    2450      413512 :         if (lock)
    2451      355468 :             LWLockAcquire(BufferDescriptorGetContentLock(buf_hdr), LW_EXCLUSIVE);
    2452             : 
    2453      413512 :         TerminateBufferIO(buf_hdr, false, BM_VALID, true);
    2454             :     }
    2455             : 
    2456      373132 :     pgBufferUsage.shared_blks_written += extend_by;
    2457             : 
    2458      373132 :     *extended_by = extend_by;
    2459             : 
    2460      373132 :     return first_block;
    2461             : }
    2462             : 
    2463             : /*
    2464             :  * BufferIsExclusiveLocked
    2465             :  *
    2466             :  *      Checks if buffer is exclusive-locked.
    2467             :  *
    2468             :  * Buffer must be pinned.
    2469             :  */
    2470             : bool
    2471           0 : BufferIsExclusiveLocked(Buffer buffer)
    2472             : {
    2473             :     BufferDesc *bufHdr;
    2474             : 
    2475           0 :     if (BufferIsLocal(buffer))
    2476             :     {
    2477           0 :         int         bufid = -buffer - 1;
    2478             : 
    2479           0 :         bufHdr = GetLocalBufferDescriptor(bufid);
    2480             :     }
    2481             :     else
    2482             :     {
    2483           0 :         bufHdr = GetBufferDescriptor(buffer - 1);
    2484             :     }
    2485             : 
    2486             :     Assert(BufferIsPinned(buffer));
    2487           0 :     return LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
    2488             :                                 LW_EXCLUSIVE);
    2489             : }
    2490             : 
    2491             : /*
    2492             :  * BufferIsDirty
    2493             :  *
    2494             :  *      Checks if buffer is already dirty.
    2495             :  *
    2496             :  * Buffer must be pinned and exclusive-locked.  (Without an exclusive lock,
    2497             :  * the result may be stale before it's returned.)
    2498             :  */
    2499             : bool
    2500           0 : BufferIsDirty(Buffer buffer)
    2501             : {
    2502             :     BufferDesc *bufHdr;
    2503             : 
    2504           0 :     if (BufferIsLocal(buffer))
    2505             :     {
    2506           0 :         int         bufid = -buffer - 1;
    2507             : 
    2508           0 :         bufHdr = GetLocalBufferDescriptor(bufid);
    2509             :     }
    2510             :     else
    2511             :     {
    2512           0 :         bufHdr = GetBufferDescriptor(buffer - 1);
    2513             :     }
    2514             : 
    2515             :     Assert(BufferIsPinned(buffer));
    2516             :     Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
    2517             :                                 LW_EXCLUSIVE));
    2518             : 
    2519           0 :     return pg_atomic_read_u32(&bufHdr->state) & BM_DIRTY;
    2520             : }
    2521             : 
    2522             : /*
    2523             :  * MarkBufferDirty
    2524             :  *
    2525             :  *      Marks buffer contents as dirty (actual write happens later).
    2526             :  *
    2527             :  * Buffer must be pinned and exclusive-locked.  (If caller does not hold
    2528             :  * exclusive lock, then somebody could be in process of writing the buffer,
    2529             :  * leading to risk of bad data written to disk.)
    2530             :  */
    2531             : void
    2532    40583498 : MarkBufferDirty(Buffer buffer)
    2533             : {
    2534             :     BufferDesc *bufHdr;
    2535             :     uint32      buf_state;
    2536             :     uint32      old_buf_state;
    2537             : 
    2538    40583498 :     if (!BufferIsValid(buffer))
    2539           0 :         elog(ERROR, "bad buffer ID: %d", buffer);
    2540             : 
    2541    40583498 :     if (BufferIsLocal(buffer))
    2542             :     {
    2543     2090856 :         MarkLocalBufferDirty(buffer);
    2544     2090856 :         return;
    2545             :     }
    2546             : 
    2547    38492642 :     bufHdr = GetBufferDescriptor(buffer - 1);
    2548             : 
    2549             :     Assert(BufferIsPinned(buffer));
    2550             :     Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
    2551             :                                 LW_EXCLUSIVE));
    2552             : 
    2553    38492642 :     old_buf_state = pg_atomic_read_u32(&bufHdr->state);
    2554             :     for (;;)
    2555             :     {
    2556    38492928 :         if (old_buf_state & BM_LOCKED)
    2557          62 :             old_buf_state = WaitBufHdrUnlocked(bufHdr);
    2558             : 
    2559    38492928 :         buf_state = old_buf_state;
    2560             : 
    2561             :         Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    2562    38492928 :         buf_state |= BM_DIRTY | BM_JUST_DIRTIED;
    2563             : 
    2564    38492928 :         if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state,
    2565             :                                            buf_state))
    2566    38492642 :             break;
    2567             :     }
    2568             : 
    2569             :     /*
    2570             :      * If the buffer was not dirty already, do vacuum accounting.
    2571             :      */
    2572    38492642 :     if (!(old_buf_state & BM_DIRTY))
    2573             :     {
    2574     1127036 :         pgBufferUsage.shared_blks_dirtied++;
    2575     1127036 :         if (VacuumCostActive)
    2576        4418 :             VacuumCostBalance += VacuumCostPageDirty;
    2577             :     }
    2578             : }
    2579             : 
    2580             : /*
    2581             :  * ReleaseAndReadBuffer -- combine ReleaseBuffer() and ReadBuffer()
    2582             :  *
    2583             :  * Formerly, this saved one cycle of acquiring/releasing the BufMgrLock
    2584             :  * compared to calling the two routines separately.  Now it's mainly just
    2585             :  * a convenience function.  However, if the passed buffer is valid and
    2586             :  * already contains the desired block, we just return it as-is; and that
    2587             :  * does save considerable work compared to a full release and reacquire.
    2588             :  *
    2589             :  * Note: it is OK to pass buffer == InvalidBuffer, indicating that no old
    2590             :  * buffer actually needs to be released.  This case is the same as ReadBuffer,
    2591             :  * but can save some tests in the caller.
    2592             :  */
    2593             : Buffer
    2594    50398702 : ReleaseAndReadBuffer(Buffer buffer,
    2595             :                      Relation relation,
    2596             :                      BlockNumber blockNum)
    2597             : {
    2598    50398702 :     ForkNumber  forkNum = MAIN_FORKNUM;
    2599             :     BufferDesc *bufHdr;
    2600             : 
    2601    50398702 :     if (BufferIsValid(buffer))
    2602             :     {
    2603             :         Assert(BufferIsPinned(buffer));
    2604    29837796 :         if (BufferIsLocal(buffer))
    2605             :         {
    2606       20862 :             bufHdr = GetLocalBufferDescriptor(-buffer - 1);
    2607       27906 :             if (bufHdr->tag.blockNum == blockNum &&
    2608       14088 :                 BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) &&
    2609        7044 :                 BufTagGetForkNum(&bufHdr->tag) == forkNum)
    2610        7044 :                 return buffer;
    2611       13818 :             UnpinLocalBuffer(buffer);
    2612             :         }
    2613             :         else
    2614             :         {
    2615    29816934 :             bufHdr = GetBufferDescriptor(buffer - 1);
    2616             :             /* we have pin, so it's ok to examine tag without spinlock */
    2617    39975784 :             if (bufHdr->tag.blockNum == blockNum &&
    2618    20317700 :                 BufTagMatchesRelFileLocator(&bufHdr->tag, &relation->rd_locator) &&
    2619    10158850 :                 BufTagGetForkNum(&bufHdr->tag) == forkNum)
    2620    10158850 :                 return buffer;
    2621    19658084 :             UnpinBuffer(bufHdr);
    2622             :         }
    2623             :     }
    2624             : 
    2625    40232808 :     return ReadBuffer(relation, blockNum);
    2626             : }
    2627             : 
    2628             : /*
    2629             :  * PinBuffer -- make buffer unavailable for replacement.
    2630             :  *
    2631             :  * For the default access strategy, the buffer's usage_count is incremented
    2632             :  * when we first pin it; for other strategies we just make sure the usage_count
    2633             :  * isn't zero.  (The idea of the latter is that we don't want synchronized
    2634             :  * heap scans to inflate the count, but we need it to not be zero to discourage
    2635             :  * other backends from stealing buffers from our ring.  As long as we cycle
    2636             :  * through the ring faster than the global clock-sweep cycles, buffers in
    2637             :  * our ring won't be chosen as victims for replacement by other backends.)
    2638             :  *
    2639             :  * This should be applied only to shared buffers, never local ones.
    2640             :  *
    2641             :  * Since buffers are pinned/unpinned very frequently, pin buffers without
    2642             :  * taking the buffer header lock; instead update the state variable in loop of
    2643             :  * CAS operations. Hopefully it's just a single CAS.
    2644             :  *
    2645             :  * Note that ResourceOwnerEnlarge() and ReservePrivateRefCountEntry()
    2646             :  * must have been done already.
    2647             :  *
    2648             :  * Returns true if buffer is BM_VALID, else false.  This provision allows
    2649             :  * some callers to avoid an extra spinlock cycle.
    2650             :  */
    2651             : static bool
    2652   104005950 : PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy)
    2653             : {
    2654   104005950 :     Buffer      b = BufferDescriptorGetBuffer(buf);
    2655             :     bool        result;
    2656             :     PrivateRefCountEntry *ref;
    2657             : 
    2658             :     Assert(!BufferIsLocal(b));
    2659             :     Assert(ReservedRefCountEntry != NULL);
    2660             : 
    2661   104005950 :     ref = GetPrivateRefCountEntry(b, true);
    2662             : 
    2663   104005950 :     if (ref == NULL)
    2664             :     {
    2665             :         uint32      buf_state;
    2666             :         uint32      old_buf_state;
    2667             : 
    2668    99863732 :         ref = NewPrivateRefCountEntry(b);
    2669             : 
    2670    99863732 :         old_buf_state = pg_atomic_read_u32(&buf->state);
    2671             :         for (;;)
    2672             :         {
    2673    99887404 :             if (old_buf_state & BM_LOCKED)
    2674        1972 :                 old_buf_state = WaitBufHdrUnlocked(buf);
    2675             : 
    2676    99887404 :             buf_state = old_buf_state;
    2677             : 
    2678             :             /* increase refcount */
    2679    99887404 :             buf_state += BUF_REFCOUNT_ONE;
    2680             : 
    2681    99887404 :             if (strategy == NULL)
    2682             :             {
    2683             :                 /* Default case: increase usagecount unless already max. */
    2684    98846104 :                 if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
    2685     5554656 :                     buf_state += BUF_USAGECOUNT_ONE;
    2686             :             }
    2687             :             else
    2688             :             {
    2689             :                 /*
    2690             :                  * Ring buffers shouldn't evict others from pool.  Thus we
    2691             :                  * don't make usagecount more than 1.
    2692             :                  */
    2693     1041300 :                 if (BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
    2694       71836 :                     buf_state += BUF_USAGECOUNT_ONE;
    2695             :             }
    2696             : 
    2697    99887404 :             if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state,
    2698             :                                                buf_state))
    2699             :             {
    2700    99863732 :                 result = (buf_state & BM_VALID) != 0;
    2701             : 
    2702             :                 /*
    2703             :                  * Assume that we acquired a buffer pin for the purposes of
    2704             :                  * Valgrind buffer client checks (even in !result case) to
    2705             :                  * keep things simple.  Buffers that are unsafe to access are
    2706             :                  * not generally guaranteed to be marked undefined or
    2707             :                  * non-accessible in any case.
    2708             :                  */
    2709             :                 VALGRIND_MAKE_MEM_DEFINED(BufHdrGetBlock(buf), BLCKSZ);
    2710    99863732 :                 break;
    2711             :             }
    2712             :         }
    2713             :     }
    2714             :     else
    2715             :     {
    2716             :         /*
    2717             :          * If we previously pinned the buffer, it is likely to be valid, but
    2718             :          * it may not be if StartReadBuffers() was called and
    2719             :          * WaitReadBuffers() hasn't been called yet.  We'll check by loading
    2720             :          * the flags without locking.  This is racy, but it's OK to return
    2721             :          * false spuriously: when WaitReadBuffers() calls StartBufferIO(),
    2722             :          * it'll see that it's now valid.
    2723             :          *
    2724             :          * Note: We deliberately avoid a Valgrind client request here.
    2725             :          * Individual access methods can optionally superimpose buffer page
    2726             :          * client requests on top of our client requests to enforce that
    2727             :          * buffers are only accessed while locked (and pinned).  It's possible
    2728             :          * that the buffer page is legitimately non-accessible here.  We
    2729             :          * cannot meddle with that.
    2730             :          */
    2731     4142218 :         result = (pg_atomic_read_u32(&buf->state) & BM_VALID) != 0;
    2732             :     }
    2733             : 
    2734   104005950 :     ref->refcount++;
    2735             :     Assert(ref->refcount > 0);
    2736   104005950 :     ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
    2737   104005950 :     return result;
    2738             : }
    2739             : 
    2740             : /*
    2741             :  * PinBuffer_Locked -- as above, but caller already locked the buffer header.
    2742             :  * The spinlock is released before return.
    2743             :  *
    2744             :  * As this function is called with the spinlock held, the caller has to
    2745             :  * previously call ReservePrivateRefCountEntry() and
    2746             :  * ResourceOwnerEnlarge(CurrentResourceOwner);
    2747             :  *
    2748             :  * Currently, no callers of this function want to modify the buffer's
    2749             :  * usage_count at all, so there's no need for a strategy parameter.
    2750             :  * Also we don't bother with a BM_VALID test (the caller could check that for
    2751             :  * itself).
    2752             :  *
    2753             :  * Also all callers only ever use this function when it's known that the
    2754             :  * buffer can't have a preexisting pin by this backend. That allows us to skip
    2755             :  * searching the private refcount array & hash, which is a boon, because the
    2756             :  * spinlock is still held.
    2757             :  *
    2758             :  * Note: use of this routine is frequently mandatory, not just an optimization
    2759             :  * to save a spin lock/unlock cycle, because we need to pin a buffer before
    2760             :  * its state can change under us.
    2761             :  */
    2762             : static void
    2763     3928660 : PinBuffer_Locked(BufferDesc *buf)
    2764             : {
    2765             :     Buffer      b;
    2766             :     PrivateRefCountEntry *ref;
    2767             :     uint32      buf_state;
    2768             : 
    2769             :     /*
    2770             :      * As explained, We don't expect any preexisting pins. That allows us to
    2771             :      * manipulate the PrivateRefCount after releasing the spinlock
    2772             :      */
    2773             :     Assert(GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf), false) == NULL);
    2774             : 
    2775             :     /*
    2776             :      * Buffer can't have a preexisting pin, so mark its page as defined to
    2777             :      * Valgrind (this is similar to the PinBuffer() case where the backend
    2778             :      * doesn't already have a buffer pin)
    2779             :      */
    2780             :     VALGRIND_MAKE_MEM_DEFINED(BufHdrGetBlock(buf), BLCKSZ);
    2781             : 
    2782             :     /*
    2783             :      * Since we hold the buffer spinlock, we can update the buffer state and
    2784             :      * release the lock in one operation.
    2785             :      */
    2786     3928660 :     buf_state = pg_atomic_read_u32(&buf->state);
    2787             :     Assert(buf_state & BM_LOCKED);
    2788     3928660 :     buf_state += BUF_REFCOUNT_ONE;
    2789     3928660 :     UnlockBufHdr(buf, buf_state);
    2790             : 
    2791     3928660 :     b = BufferDescriptorGetBuffer(buf);
    2792             : 
    2793     3928660 :     ref = NewPrivateRefCountEntry(b);
    2794     3928660 :     ref->refcount++;
    2795             : 
    2796     3928660 :     ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
    2797     3928660 : }
    2798             : 
    2799             : /*
    2800             :  * UnpinBuffer -- make buffer available for replacement.
    2801             :  *
    2802             :  * This should be applied only to shared buffers, never local ones.  This
    2803             :  * always adjusts CurrentResourceOwner.
    2804             :  */
    2805             : static void
    2806   126911248 : UnpinBuffer(BufferDesc *buf)
    2807             : {
    2808   126911248 :     Buffer      b = BufferDescriptorGetBuffer(buf);
    2809             : 
    2810   126911248 :     ResourceOwnerForgetBuffer(CurrentResourceOwner, b);
    2811   126911248 :     UnpinBufferNoOwner(buf);
    2812   126911248 : }
    2813             : 
    2814             : static void
    2815   126919410 : UnpinBufferNoOwner(BufferDesc *buf)
    2816             : {
    2817             :     PrivateRefCountEntry *ref;
    2818   126919410 :     Buffer      b = BufferDescriptorGetBuffer(buf);
    2819             : 
    2820             :     Assert(!BufferIsLocal(b));
    2821             : 
    2822             :     /* not moving as we're likely deleting it soon anyway */
    2823   126919410 :     ref = GetPrivateRefCountEntry(b, false);
    2824             :     Assert(ref != NULL);
    2825             :     Assert(ref->refcount > 0);
    2826   126919410 :     ref->refcount--;
    2827   126919410 :     if (ref->refcount == 0)
    2828             :     {
    2829             :         uint32      buf_state;
    2830             :         uint32      old_buf_state;
    2831             : 
    2832             :         /*
    2833             :          * Mark buffer non-accessible to Valgrind.
    2834             :          *
    2835             :          * Note that the buffer may have already been marked non-accessible
    2836             :          * within access method code that enforces that buffers are only
    2837             :          * accessed while a buffer lock is held.
    2838             :          */
    2839             :         VALGRIND_MAKE_MEM_NOACCESS(BufHdrGetBlock(buf), BLCKSZ);
    2840             : 
    2841             :         /* I'd better not still hold the buffer content lock */
    2842             :         Assert(!LWLockHeldByMe(BufferDescriptorGetContentLock(buf)));
    2843             : 
    2844             :         /*
    2845             :          * Decrement the shared reference count.
    2846             :          *
    2847             :          * Since buffer spinlock holder can update status using just write,
    2848             :          * it's not safe to use atomic decrement here; thus use a CAS loop.
    2849             :          */
    2850   103792392 :         old_buf_state = pg_atomic_read_u32(&buf->state);
    2851             :         for (;;)
    2852             :         {
    2853   103813680 :             if (old_buf_state & BM_LOCKED)
    2854        2324 :                 old_buf_state = WaitBufHdrUnlocked(buf);
    2855             : 
    2856   103813680 :             buf_state = old_buf_state;
    2857             : 
    2858   103813680 :             buf_state -= BUF_REFCOUNT_ONE;
    2859             : 
    2860   103813680 :             if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state,
    2861             :                                                buf_state))
    2862   103792392 :                 break;
    2863             :         }
    2864             : 
    2865             :         /* Support LockBufferForCleanup() */
    2866   103792392 :         if (buf_state & BM_PIN_COUNT_WAITER)
    2867             :         {
    2868             :             /*
    2869             :              * Acquire the buffer header lock, re-check that there's a waiter.
    2870             :              * Another backend could have unpinned this buffer, and already
    2871             :              * woken up the waiter.  There's no danger of the buffer being
    2872             :              * replaced after we unpinned it above, as it's pinned by the
    2873             :              * waiter.
    2874             :              */
    2875           4 :             buf_state = LockBufHdr(buf);
    2876             : 
    2877           4 :             if ((buf_state & BM_PIN_COUNT_WAITER) &&
    2878           4 :                 BUF_STATE_GET_REFCOUNT(buf_state) == 1)
    2879           4 :             {
    2880             :                 /* we just released the last pin other than the waiter's */
    2881           4 :                 int         wait_backend_pgprocno = buf->wait_backend_pgprocno;
    2882             : 
    2883           4 :                 buf_state &= ~BM_PIN_COUNT_WAITER;
    2884           4 :                 UnlockBufHdr(buf, buf_state);
    2885           4 :                 ProcSendSignal(wait_backend_pgprocno);
    2886             :             }
    2887             :             else
    2888           0 :                 UnlockBufHdr(buf, buf_state);
    2889             :         }
    2890   103792392 :         ForgetPrivateRefCountEntry(ref);
    2891             :     }
    2892   126919410 : }
    2893             : 
    2894             : #define ST_SORT sort_checkpoint_bufferids
    2895             : #define ST_ELEMENT_TYPE CkptSortItem
    2896             : #define ST_COMPARE(a, b) ckpt_buforder_comparator(a, b)
    2897             : #define ST_SCOPE static
    2898             : #define ST_DEFINE
    2899             : #include <lib/sort_template.h>
    2900             : 
    2901             : /*
    2902             :  * BufferSync -- Write out all dirty buffers in the pool.
    2903             :  *
    2904             :  * This is called at checkpoint time to write out all dirty shared buffers.
    2905             :  * The checkpoint request flags should be passed in.  If CHECKPOINT_IMMEDIATE
    2906             :  * is set, we disable delays between writes; if CHECKPOINT_IS_SHUTDOWN,
    2907             :  * CHECKPOINT_END_OF_RECOVERY or CHECKPOINT_FLUSH_ALL is set, we write even
    2908             :  * unlogged buffers, which are otherwise skipped.  The remaining flags
    2909             :  * currently have no effect here.
    2910             :  */
    2911             : static void
    2912        2468 : BufferSync(int flags)
    2913             : {
    2914             :     uint32      buf_state;
    2915             :     int         buf_id;
    2916             :     int         num_to_scan;
    2917             :     int         num_spaces;
    2918             :     int         num_processed;
    2919             :     int         num_written;
    2920        2468 :     CkptTsStatus *per_ts_stat = NULL;
    2921             :     Oid         last_tsid;
    2922             :     binaryheap *ts_heap;
    2923             :     int         i;
    2924        2468 :     int         mask = BM_DIRTY;
    2925             :     WritebackContext wb_context;
    2926             : 
    2927             :     /*
    2928             :      * Unless this is a shutdown checkpoint or we have been explicitly told,
    2929             :      * we write only permanent, dirty buffers.  But at shutdown or end of
    2930             :      * recovery, we write all dirty buffers.
    2931             :      */
    2932        2468 :     if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY |
    2933             :                     CHECKPOINT_FLUSH_ALL))))
    2934        1206 :         mask |= BM_PERMANENT;
    2935             : 
    2936             :     /*
    2937             :      * Loop over all buffers, and mark the ones that need to be written with
    2938             :      * BM_CHECKPOINT_NEEDED.  Count them as we go (num_to_scan), so that we
    2939             :      * can estimate how much work needs to be done.
    2940             :      *
    2941             :      * This allows us to write only those pages that were dirty when the
    2942             :      * checkpoint began, and not those that get dirtied while it proceeds.
    2943             :      * Whenever a page with BM_CHECKPOINT_NEEDED is written out, either by us
    2944             :      * later in this function, or by normal backends or the bgwriter cleaning
    2945             :      * scan, the flag is cleared.  Any buffer dirtied after this point won't
    2946             :      * have the flag set.
    2947             :      *
    2948             :      * Note that if we fail to write some buffer, we may leave buffers with
    2949             :      * BM_CHECKPOINT_NEEDED still set.  This is OK since any such buffer would
    2950             :      * certainly need to be written for the next checkpoint attempt, too.
    2951             :      */
    2952        2468 :     num_to_scan = 0;
    2953    20410180 :     for (buf_id = 0; buf_id < NBuffers; buf_id++)
    2954             :     {
    2955    20407712 :         BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
    2956             : 
    2957             :         /*
    2958             :          * Header spinlock is enough to examine BM_DIRTY, see comment in
    2959             :          * SyncOneBuffer.
    2960             :          */
    2961    20407712 :         buf_state = LockBufHdr(bufHdr);
    2962             : 
    2963    20407712 :         if ((buf_state & mask) == mask)
    2964             :         {
    2965             :             CkptSortItem *item;
    2966             : 
    2967      505308 :             buf_state |= BM_CHECKPOINT_NEEDED;
    2968             : 
    2969      505308 :             item = &CkptBufferIds[num_to_scan++];
    2970      505308 :             item->buf_id = buf_id;
    2971      505308 :             item->tsId = bufHdr->tag.spcOid;
    2972      505308 :             item->relNumber = BufTagGetRelNumber(&bufHdr->tag);
    2973      505308 :             item->forkNum = BufTagGetForkNum(&bufHdr->tag);
    2974      505308 :             item->blockNum = bufHdr->tag.blockNum;
    2975             :         }
    2976             : 
    2977    20407712 :         UnlockBufHdr(bufHdr, buf_state);
    2978             : 
    2979             :         /* Check for barrier events in case NBuffers is large. */
    2980    20407712 :         if (ProcSignalBarrierPending)
    2981           0 :             ProcessProcSignalBarrier();
    2982             :     }
    2983             : 
    2984        2468 :     if (num_to_scan == 0)
    2985         604 :         return;                 /* nothing to do */
    2986             : 
    2987        1864 :     WritebackContextInit(&wb_context, &checkpoint_flush_after);
    2988             : 
    2989             :     TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
    2990             : 
    2991             :     /*
    2992             :      * Sort buffers that need to be written to reduce the likelihood of random
    2993             :      * IO. The sorting is also important for the implementation of balancing
    2994             :      * writes between tablespaces. Without balancing writes we'd potentially
    2995             :      * end up writing to the tablespaces one-by-one; possibly overloading the
    2996             :      * underlying system.
    2997             :      */
    2998        1864 :     sort_checkpoint_bufferids(CkptBufferIds, num_to_scan);
    2999             : 
    3000        1864 :     num_spaces = 0;
    3001             : 
    3002             :     /*
    3003             :      * Allocate progress status for each tablespace with buffers that need to
    3004             :      * be flushed. This requires the to-be-flushed array to be sorted.
    3005             :      */
    3006        1864 :     last_tsid = InvalidOid;
    3007      507172 :     for (i = 0; i < num_to_scan; i++)
    3008             :     {
    3009             :         CkptTsStatus *s;
    3010             :         Oid         cur_tsid;
    3011             : 
    3012      505308 :         cur_tsid = CkptBufferIds[i].tsId;
    3013             : 
    3014             :         /*
    3015             :          * Grow array of per-tablespace status structs, every time a new
    3016             :          * tablespace is found.
    3017             :          */
    3018      505308 :         if (last_tsid == InvalidOid || last_tsid != cur_tsid)
    3019        2746 :         {
    3020             :             Size        sz;
    3021             : 
    3022        2746 :             num_spaces++;
    3023             : 
    3024             :             /*
    3025             :              * Not worth adding grow-by-power-of-2 logic here - even with a
    3026             :              * few hundred tablespaces this should be fine.
    3027             :              */
    3028        2746 :             sz = sizeof(CkptTsStatus) * num_spaces;
    3029             : 
    3030        2746 :             if (per_ts_stat == NULL)
    3031        1864 :                 per_ts_stat = (CkptTsStatus *) palloc(sz);
    3032             :             else
    3033         882 :                 per_ts_stat = (CkptTsStatus *) repalloc(per_ts_stat, sz);
    3034             : 
    3035        2746 :             s = &per_ts_stat[num_spaces - 1];
    3036        2746 :             memset(s, 0, sizeof(*s));
    3037        2746 :             s->tsId = cur_tsid;
    3038             : 
    3039             :             /*
    3040             :              * The first buffer in this tablespace. As CkptBufferIds is sorted
    3041             :              * by tablespace all (s->num_to_scan) buffers in this tablespace
    3042             :              * will follow afterwards.
    3043             :              */
    3044        2746 :             s->index = i;
    3045             : 
    3046             :             /*
    3047             :              * progress_slice will be determined once we know how many buffers
    3048             :              * are in each tablespace, i.e. after this loop.
    3049             :              */
    3050             : 
    3051        2746 :             last_tsid = cur_tsid;
    3052             :         }
    3053             :         else
    3054             :         {
    3055      502562 :             s = &per_ts_stat[num_spaces - 1];
    3056             :         }
    3057             : 
    3058      505308 :         s->num_to_scan++;
    3059             : 
    3060             :         /* Check for barrier events. */
    3061      505308 :         if (ProcSignalBarrierPending)
    3062           0 :             ProcessProcSignalBarrier();
    3063             :     }
    3064             : 
    3065             :     Assert(num_spaces > 0);
    3066             : 
    3067             :     /*
    3068             :      * Build a min-heap over the write-progress in the individual tablespaces,
    3069             :      * and compute how large a portion of the total progress a single
    3070             :      * processed buffer is.
    3071             :      */
    3072        1864 :     ts_heap = binaryheap_allocate(num_spaces,
    3073             :                                   ts_ckpt_progress_comparator,
    3074             :                                   NULL);
    3075             : 
    3076        4610 :     for (i = 0; i < num_spaces; i++)
    3077             :     {
    3078        2746 :         CkptTsStatus *ts_stat = &per_ts_stat[i];
    3079             : 
    3080        2746 :         ts_stat->progress_slice = (float8) num_to_scan / ts_stat->num_to_scan;
    3081             : 
    3082        2746 :         binaryheap_add_unordered(ts_heap, PointerGetDatum(ts_stat));
    3083             :     }
    3084             : 
    3085        1864 :     binaryheap_build(ts_heap);
    3086             : 
    3087             :     /*
    3088             :      * Iterate through to-be-checkpointed buffers and write the ones (still)
    3089             :      * marked with BM_CHECKPOINT_NEEDED. The writes are balanced between
    3090             :      * tablespaces; otherwise the sorting would lead to only one tablespace
    3091             :      * receiving writes at a time, making inefficient use of the hardware.
    3092             :      */
    3093        1864 :     num_processed = 0;
    3094        1864 :     num_written = 0;
    3095      507172 :     while (!binaryheap_empty(ts_heap))
    3096             :     {
    3097      505308 :         BufferDesc *bufHdr = NULL;
    3098             :         CkptTsStatus *ts_stat = (CkptTsStatus *)
    3099      505308 :             DatumGetPointer(binaryheap_first(ts_heap));
    3100             : 
    3101      505308 :         buf_id = CkptBufferIds[ts_stat->index].buf_id;
    3102             :         Assert(buf_id != -1);
    3103             : 
    3104      505308 :         bufHdr = GetBufferDescriptor(buf_id);
    3105             : 
    3106      505308 :         num_processed++;
    3107             : 
    3108             :         /*
    3109             :          * We don't need to acquire the lock here, because we're only looking
    3110             :          * at a single bit. It's possible that someone else writes the buffer
    3111             :          * and clears the flag right after we check, but that doesn't matter
    3112             :          * since SyncOneBuffer will then do nothing.  However, there is a
    3113             :          * further race condition: it's conceivable that between the time we
    3114             :          * examine the bit here and the time SyncOneBuffer acquires the lock,
    3115             :          * someone else not only wrote the buffer but replaced it with another
    3116             :          * page and dirtied it.  In that improbable case, SyncOneBuffer will
    3117             :          * write the buffer though we didn't need to.  It doesn't seem worth
    3118             :          * guarding against this, though.
    3119             :          */
    3120      505308 :         if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED)
    3121             :         {
    3122      472174 :             if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN)
    3123             :             {
    3124             :                 TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
    3125      472174 :                 PendingCheckpointerStats.buffers_written++;
    3126      472174 :                 num_written++;
    3127             :             }
    3128             :         }
    3129             : 
    3130             :         /*
    3131             :          * Measure progress independent of actually having to flush the buffer
    3132             :          * - otherwise writing become unbalanced.
    3133             :          */
    3134      505308 :         ts_stat->progress += ts_stat->progress_slice;
    3135      505308 :         ts_stat->num_scanned++;
    3136      505308 :         ts_stat->index++;
    3137             : 
    3138             :         /* Have all the buffers from the tablespace been processed? */
    3139      505308 :         if (ts_stat->num_scanned == ts_stat->num_to_scan)
    3140             :         {
    3141        2746 :             binaryheap_remove_first(ts_heap);
    3142             :         }
    3143             :         else
    3144             :         {
    3145             :             /* update heap with the new progress */
    3146      502562 :             binaryheap_replace_first(ts_heap, PointerGetDatum(ts_stat));
    3147             :         }
    3148             : 
    3149             :         /*
    3150             :          * Sleep to throttle our I/O rate.
    3151             :          *
    3152             :          * (This will check for barrier events even if it doesn't sleep.)
    3153             :          */
    3154      505308 :         CheckpointWriteDelay(flags, (double) num_processed / num_to_scan);
    3155             :     }
    3156             : 
    3157             :     /*
    3158             :      * Issue all pending flushes. Only checkpointer calls BufferSync(), so
    3159             :      * IOContext will always be IOCONTEXT_NORMAL.
    3160             :      */
    3161        1864 :     IssuePendingWritebacks(&wb_context, IOCONTEXT_NORMAL);
    3162             : 
    3163        1864 :     pfree(per_ts_stat);
    3164        1864 :     per_ts_stat = NULL;
    3165        1864 :     binaryheap_free(ts_heap);
    3166             : 
    3167             :     /*
    3168             :      * Update checkpoint statistics. As noted above, this doesn't include
    3169             :      * buffers written by other backends or bgwriter scan.
    3170             :      */
    3171        1864 :     CheckpointStats.ckpt_bufs_written += num_written;
    3172             : 
    3173             :     TRACE_POSTGRESQL_BUFFER_SYNC_DONE(NBuffers, num_written, num_to_scan);
    3174             : }
    3175             : 
    3176             : /*
    3177             :  * BgBufferSync -- Write out some dirty buffers in the pool.
    3178             :  *
    3179             :  * This is called periodically by the background writer process.
    3180             :  *
    3181             :  * Returns true if it's appropriate for the bgwriter process to go into
    3182             :  * low-power hibernation mode.  (This happens if the strategy clock sweep
    3183             :  * has been "lapped" and no buffer allocations have occurred recently,
    3184             :  * or if the bgwriter has been effectively disabled by setting
    3185             :  * bgwriter_lru_maxpages to 0.)
    3186             :  */
    3187             : bool
    3188       17212 : BgBufferSync(WritebackContext *wb_context)
    3189             : {
    3190             :     /* info obtained from freelist.c */
    3191             :     int         strategy_buf_id;
    3192             :     uint32      strategy_passes;
    3193             :     uint32      recent_alloc;
    3194             : 
    3195             :     /*
    3196             :      * Information saved between calls so we can determine the strategy
    3197             :      * point's advance rate and avoid scanning already-cleaned buffers.
    3198             :      */
    3199             :     static bool saved_info_valid = false;
    3200             :     static int  prev_strategy_buf_id;
    3201             :     static uint32 prev_strategy_passes;
    3202             :     static int  next_to_clean;
    3203             :     static uint32 next_passes;
    3204             : 
    3205             :     /* Moving averages of allocation rate and clean-buffer density */
    3206             :     static float smoothed_alloc = 0;
    3207             :     static float smoothed_density = 10.0;
    3208             : 
    3209             :     /* Potentially these could be tunables, but for now, not */
    3210       17212 :     float       smoothing_samples = 16;
    3211       17212 :     float       scan_whole_pool_milliseconds = 120000.0;
    3212             : 
    3213             :     /* Used to compute how far we scan ahead */
    3214             :     long        strategy_delta;
    3215             :     int         bufs_to_lap;
    3216             :     int         bufs_ahead;
    3217             :     float       scans_per_alloc;
    3218             :     int         reusable_buffers_est;
    3219             :     int         upcoming_alloc_est;
    3220             :     int         min_scan_buffers;
    3221             : 
    3222             :     /* Variables for the scanning loop proper */
    3223             :     int         num_to_scan;
    3224             :     int         num_written;
    3225             :     int         reusable_buffers;
    3226             : 
    3227             :     /* Variables for final smoothed_density update */
    3228             :     long        new_strategy_delta;
    3229             :     uint32      new_recent_alloc;
    3230             : 
    3231             :     /*
    3232             :      * Find out where the freelist clock sweep currently is, and how many
    3233             :      * buffer allocations have happened since our last call.
    3234             :      */
    3235       17212 :     strategy_buf_id = StrategySyncStart(&strategy_passes, &recent_alloc);
    3236             : 
    3237             :     /* Report buffer alloc counts to pgstat */
    3238       17212 :     PendingBgWriterStats.buf_alloc += recent_alloc;
    3239             : 
    3240             :     /*
    3241             :      * If we're not running the LRU scan, just stop after doing the stats
    3242             :      * stuff.  We mark the saved state invalid so that we can recover sanely
    3243             :      * if LRU scan is turned back on later.
    3244             :      */
    3245       17212 :     if (bgwriter_lru_maxpages <= 0)
    3246             :     {
    3247          36 :         saved_info_valid = false;
    3248          36 :         return true;
    3249             :     }
    3250             : 
    3251             :     /*
    3252             :      * Compute strategy_delta = how many buffers have been scanned by the
    3253             :      * clock sweep since last time.  If first time through, assume none. Then
    3254             :      * see if we are still ahead of the clock sweep, and if so, how many
    3255             :      * buffers we could scan before we'd catch up with it and "lap" it. Note:
    3256             :      * weird-looking coding of xxx_passes comparisons are to avoid bogus
    3257             :      * behavior when the passes counts wrap around.
    3258             :      */
    3259       17176 :     if (saved_info_valid)
    3260             :     {
    3261       16294 :         int32       passes_delta = strategy_passes - prev_strategy_passes;
    3262             : 
    3263       16294 :         strategy_delta = strategy_buf_id - prev_strategy_buf_id;
    3264       16294 :         strategy_delta += (long) passes_delta * NBuffers;
    3265             : 
    3266             :         Assert(strategy_delta >= 0);
    3267             : 
    3268       16294 :         if ((int32) (next_passes - strategy_passes) > 0)
    3269             :         {
    3270             :             /* we're one pass ahead of the strategy point */
    3271        3600 :             bufs_to_lap = strategy_buf_id - next_to_clean;
    3272             : #ifdef BGW_DEBUG
    3273             :             elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
    3274             :                  next_passes, next_to_clean,
    3275             :                  strategy_passes, strategy_buf_id,
    3276             :                  strategy_delta, bufs_to_lap);
    3277             : #endif
    3278             :         }
    3279       12694 :         else if (next_passes == strategy_passes &&
    3280        9780 :                  next_to_clean >= strategy_buf_id)
    3281             :         {
    3282             :             /* on same pass, but ahead or at least not behind */
    3283        9570 :             bufs_to_lap = NBuffers - (next_to_clean - strategy_buf_id);
    3284             : #ifdef BGW_DEBUG
    3285             :             elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
    3286             :                  next_passes, next_to_clean,
    3287             :                  strategy_passes, strategy_buf_id,
    3288             :                  strategy_delta, bufs_to_lap);
    3289             : #endif
    3290             :         }
    3291             :         else
    3292             :         {
    3293             :             /*
    3294             :              * We're behind, so skip forward to the strategy point and start
    3295             :              * cleaning from there.
    3296             :              */
    3297             : #ifdef BGW_DEBUG
    3298             :             elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld",
    3299             :                  next_passes, next_to_clean,
    3300             :                  strategy_passes, strategy_buf_id,
    3301             :                  strategy_delta);
    3302             : #endif
    3303        3124 :             next_to_clean = strategy_buf_id;
    3304        3124 :             next_passes = strategy_passes;
    3305        3124 :             bufs_to_lap = NBuffers;
    3306             :         }
    3307             :     }
    3308             :     else
    3309             :     {
    3310             :         /*
    3311             :          * Initializing at startup or after LRU scanning had been off. Always
    3312             :          * start at the strategy point.
    3313             :          */
    3314             : #ifdef BGW_DEBUG
    3315             :         elog(DEBUG2, "bgwriter initializing: strategy %u-%u",
    3316             :              strategy_passes, strategy_buf_id);
    3317             : #endif
    3318         882 :         strategy_delta = 0;
    3319         882 :         next_to_clean = strategy_buf_id;
    3320         882 :         next_passes = strategy_passes;
    3321         882 :         bufs_to_lap = NBuffers;
    3322             :     }
    3323             : 
    3324             :     /* Update saved info for next time */
    3325       17176 :     prev_strategy_buf_id = strategy_buf_id;
    3326       17176 :     prev_strategy_passes = strategy_passes;
    3327       17176 :     saved_info_valid = true;
    3328             : 
    3329             :     /*
    3330             :      * Compute how many buffers had to be scanned for each new allocation, ie,
    3331             :      * 1/density of reusable buffers, and track a moving average of that.
    3332             :      *
    3333             :      * If the strategy point didn't move, we don't update the density estimate
    3334             :      */
    3335       17176 :     if (strategy_delta > 0 && recent_alloc > 0)
    3336             :     {
    3337        3852 :         scans_per_alloc = (float) strategy_delta / (float) recent_alloc;
    3338        3852 :         smoothed_density += (scans_per_alloc - smoothed_density) /
    3339             :             smoothing_samples;
    3340             :     }
    3341             : 
    3342             :     /*
    3343             :      * Estimate how many reusable buffers there are between the current
    3344             :      * strategy point and where we've scanned ahead to, based on the smoothed
    3345             :      * density estimate.
    3346             :      */
    3347       17176 :     bufs_ahead = NBuffers - bufs_to_lap;
    3348       17176 :     reusable_buffers_est = (float) bufs_ahead / smoothed_density;
    3349             : 
    3350             :     /*
    3351             :      * Track a moving average of recent buffer allocations.  Here, rather than
    3352             :      * a true average we want a fast-attack, slow-decline behavior: we
    3353             :      * immediately follow any increase.
    3354             :      */
    3355       17176 :     if (smoothed_alloc <= (float) recent_alloc)
    3356        4908 :         smoothed_alloc = recent_alloc;
    3357             :     else
    3358       12268 :         smoothed_alloc += ((float) recent_alloc - smoothed_alloc) /
    3359             :             smoothing_samples;
    3360             : 
    3361             :     /* Scale the estimate by a GUC to allow more aggressive tuning. */
    3362       17176 :     upcoming_alloc_est = (int) (smoothed_alloc * bgwriter_lru_multiplier);
    3363             : 
    3364             :     /*
    3365             :      * If recent_alloc remains at zero for many cycles, smoothed_alloc will
    3366             :      * eventually underflow to zero, and the underflows produce annoying
    3367             :      * kernel warnings on some platforms.  Once upcoming_alloc_est has gone to
    3368             :      * zero, there's no point in tracking smaller and smaller values of
    3369             :      * smoothed_alloc, so just reset it to exactly zero to avoid this
    3370             :      * syndrome.  It will pop back up as soon as recent_alloc increases.
    3371             :      */
    3372       17176 :     if (upcoming_alloc_est == 0)
    3373        2934 :         smoothed_alloc = 0;
    3374             : 
    3375             :     /*
    3376             :      * Even in cases where there's been little or no buffer allocation
    3377             :      * activity, we want to make a small amount of progress through the buffer
    3378             :      * cache so that as many reusable buffers as possible are clean after an
    3379             :      * idle period.
    3380             :      *
    3381             :      * (scan_whole_pool_milliseconds / BgWriterDelay) computes how many times
    3382             :      * the BGW will be called during the scan_whole_pool time; slice the
    3383             :      * buffer pool into that many sections.
    3384             :      */
    3385       17176 :     min_scan_buffers = (int) (NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay));
    3386             : 
    3387       17176 :     if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est))
    3388             :     {
    3389             : #ifdef BGW_DEBUG
    3390             :         elog(DEBUG2, "bgwriter: alloc_est=%d too small, using min=%d + reusable_est=%d",
    3391             :              upcoming_alloc_est, min_scan_buffers, reusable_buffers_est);
    3392             : #endif
    3393        9390 :         upcoming_alloc_est = min_scan_buffers + reusable_buffers_est;
    3394             :     }
    3395             : 
    3396             :     /*
    3397             :      * Now write out dirty reusable buffers, working forward from the
    3398             :      * next_to_clean point, until we have lapped the strategy scan, or cleaned
    3399             :      * enough buffers to match our estimate of the next cycle's allocation
    3400             :      * requirements, or hit the bgwriter_lru_maxpages limit.
    3401             :      */
    3402             : 
    3403       17176 :     num_to_scan = bufs_to_lap;
    3404       17176 :     num_written = 0;
    3405       17176 :     reusable_buffers = reusable_buffers_est;
    3406             : 
    3407             :     /* Execute the LRU scan */
    3408     2738522 :     while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est)
    3409             :     {
    3410     2721348 :         int         sync_state = SyncOneBuffer(next_to_clean, true,
    3411             :                                                wb_context);
    3412             : 
    3413     2721348 :         if (++next_to_clean >= NBuffers)
    3414             :         {
    3415        3472 :             next_to_clean = 0;
    3416        3472 :             next_passes++;
    3417             :         }
    3418     2721348 :         num_to_scan--;
    3419             : 
    3420     2721348 :         if (sync_state & BUF_WRITTEN)
    3421             :         {
    3422       28566 :             reusable_buffers++;
    3423       28566 :             if (++num_written >= bgwriter_lru_maxpages)
    3424             :             {
    3425           2 :                 PendingBgWriterStats.maxwritten_clean++;
    3426           2 :                 break;
    3427             :             }
    3428             :         }
    3429     2692782 :         else if (sync_state & BUF_REUSABLE)
    3430     2000054 :             reusable_buffers++;
    3431             :     }
    3432             : 
    3433       17176 :     PendingBgWriterStats.buf_written_clean += num_written;
    3434             : 
    3435             : #ifdef BGW_DEBUG
    3436             :     elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d",
    3437             :          recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead,
    3438             :          smoothed_density, reusable_buffers_est, upcoming_alloc_est,
    3439             :          bufs_to_lap - num_to_scan,
    3440             :          num_written,
    3441             :          reusable_buffers - reusable_buffers_est);
    3442             : #endif
    3443             : 
    3444             :     /*
    3445             :      * Consider the above scan as being like a new allocation scan.
    3446             :      * Characterize its density and update the smoothed one based on it. This
    3447             :      * effectively halves the moving average period in cases where both the
    3448             :      * strategy and the background writer are doing some useful scanning,
    3449             :      * which is helpful because a long memory isn't as desirable on the
    3450             :      * density estimates.
    3451             :      */
    3452       17176 :     new_strategy_delta = bufs_to_lap - num_to_scan;
    3453       17176 :     new_recent_alloc = reusable_buffers - reusable_buffers_est;
    3454       17176 :     if (new_strategy_delta > 0 && new_recent_alloc > 0)
    3455             :     {
    3456       13312 :         scans_per_alloc = (float) new_strategy_delta / (float) new_recent_alloc;
    3457       13312 :         smoothed_density += (scans_per_alloc - smoothed_density) /
    3458             :             smoothing_samples;
    3459             : 
    3460             : #ifdef BGW_DEBUG
    3461             :         elog(DEBUG2, "bgwriter: cleaner density alloc=%u scan=%ld density=%.2f new smoothed=%.2f",
    3462             :              new_recent_alloc, new_strategy_delta,
    3463             :              scans_per_alloc, smoothed_density);
    3464             : #endif
    3465             :     }
    3466             : 
    3467             :     /* Return true if OK to hibernate */
    3468       17176 :     return (bufs_to_lap == 0 && recent_alloc == 0);
    3469             : }
    3470             : 
    3471             : /*
    3472             :  * SyncOneBuffer -- process a single buffer during syncing.
    3473             :  *
    3474             :  * If skip_recently_used is true, we don't write currently-pinned buffers, nor
    3475             :  * buffers marked recently used, as these are not replacement candidates.
    3476             :  *
    3477             :  * Returns a bitmask containing the following flag bits:
    3478             :  *  BUF_WRITTEN: we wrote the buffer.
    3479             :  *  BUF_REUSABLE: buffer is available for replacement, ie, it has
    3480             :  *      pin count 0 and usage count 0.
    3481             :  *
    3482             :  * (BUF_WRITTEN could be set in error if FlushBuffer finds the buffer clean
    3483             :  * after locking it, but we don't care all that much.)
    3484             :  */
    3485             : static int
    3486     3193522 : SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context)
    3487             : {
    3488     3193522 :     BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
    3489     3193522 :     int         result = 0;
    3490             :     uint32      buf_state;
    3491             :     BufferTag   tag;
    3492             : 
    3493             :     /* Make sure we can handle the pin */
    3494     3193522 :     ReservePrivateRefCountEntry();
    3495     3193522 :     ResourceOwnerEnlarge(CurrentResourceOwner);
    3496             : 
    3497             :     /*
    3498             :      * Check whether buffer needs writing.
    3499             :      *
    3500             :      * We can make this check without taking the buffer content lock so long
    3501             :      * as we mark pages dirty in access methods *before* logging changes with
    3502             :      * XLogInsert(): if someone marks the buffer dirty just after our check we
    3503             :      * don't worry because our checkpoint.redo points before log record for
    3504             :      * upcoming changes and so we are not required to write such dirty buffer.
    3505             :      */
    3506     3193522 :     buf_state = LockBufHdr(bufHdr);
    3507             : 
    3508     3193522 :     if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 &&
    3509     3189160 :         BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
    3510             :     {
    3511     2032548 :         result |= BUF_REUSABLE;
    3512             :     }
    3513     1160974 :     else if (skip_recently_used)
    3514             :     {
    3515             :         /* Caller told us not to write recently-used buffers */
    3516      692728 :         UnlockBufHdr(bufHdr, buf_state);
    3517      692728 :         return result;
    3518             :     }
    3519             : 
    3520     2500794 :     if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY))
    3521             :     {
    3522             :         /* It's clean, so nothing to do */
    3523     2000054 :         UnlockBufHdr(bufHdr, buf_state);
    3524     2000054 :         return result;
    3525             :     }
    3526             : 
    3527             :     /*
    3528             :      * Pin it, share-lock it, write it.  (FlushBuffer will do nothing if the
    3529             :      * buffer is clean by the time we've locked it.)
    3530             :      */
    3531      500740 :     PinBuffer_Locked(bufHdr);
    3532      500740 :     LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
    3533             : 
    3534      500740 :     FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
    3535             : 
    3536      500740 :     LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
    3537             : 
    3538      500740 :     tag = bufHdr->tag;
    3539             : 
    3540      500740 :     UnpinBuffer(bufHdr);
    3541             : 
    3542             :     /*
    3543             :      * SyncOneBuffer() is only called by checkpointer and bgwriter, so
    3544             :      * IOContext will always be IOCONTEXT_NORMAL.
    3545             :      */
    3546      500740 :     ScheduleBufferTagForWriteback(wb_context, IOCONTEXT_NORMAL, &tag);
    3547             : 
    3548      500740 :     return result | BUF_WRITTEN;
    3549             : }
    3550             : 
    3551             : /*
    3552             :  *      AtEOXact_Buffers - clean up at end of transaction.
    3553             :  *
    3554             :  *      As of PostgreSQL 8.0, buffer pins should get released by the
    3555             :  *      ResourceOwner mechanism.  This routine is just a debugging
    3556             :  *      cross-check that no pins remain.
    3557             :  */
    3558             : void
    3559      747580 : AtEOXact_Buffers(bool isCommit)
    3560             : {
    3561      747580 :     CheckForBufferLeaks();
    3562             : 
    3563      747580 :     AtEOXact_LocalBuffers(isCommit);
    3564             : 
    3565             :     Assert(PrivateRefCountOverflowed == 0);
    3566      747580 : }
    3567             : 
    3568             : /*
    3569             :  * Initialize access to shared buffer pool
    3570             :  *
    3571             :  * This is called during backend startup (whether standalone or under the
    3572             :  * postmaster).  It sets up for this backend's access to the already-existing
    3573             :  * buffer pool.
    3574             :  */
    3575             : void
    3576       35568 : InitBufferManagerAccess(void)
    3577             : {
    3578             :     HASHCTL     hash_ctl;
    3579             : 
    3580       35568 :     memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray));
    3581             : 
    3582       35568 :     hash_ctl.keysize = sizeof(int32);
    3583       35568 :     hash_ctl.entrysize = sizeof(PrivateRefCountEntry);
    3584             : 
    3585       35568 :     PrivateRefCountHash = hash_create("PrivateRefCount", 100, &hash_ctl,
    3586             :                                       HASH_ELEM | HASH_BLOBS);
    3587             : 
    3588             :     /*
    3589             :      * AtProcExit_Buffers needs LWLock access, and thereby has to be called at
    3590             :      * the corresponding phase of backend shutdown.
    3591             :      */
    3592             :     Assert(MyProc != NULL);
    3593       35568 :     on_shmem_exit(AtProcExit_Buffers, 0);
    3594       35568 : }
    3595             : 
    3596             : /*
    3597             :  * During backend exit, ensure that we released all shared-buffer locks and
    3598             :  * assert that we have no remaining pins.
    3599             :  */
    3600             : static void
    3601       35568 : AtProcExit_Buffers(int code, Datum arg)
    3602             : {
    3603       35568 :     UnlockBuffers();
    3604             : 
    3605       35568 :     CheckForBufferLeaks();
    3606             : 
    3607             :     /* localbuf.c needs a chance too */
    3608       35568 :     AtProcExit_LocalBuffers();
    3609       35568 : }
    3610             : 
    3611             : /*
    3612             :  *      CheckForBufferLeaks - ensure this backend holds no buffer pins
    3613             :  *
    3614             :  *      As of PostgreSQL 8.0, buffer pins should get released by the
    3615             :  *      ResourceOwner mechanism.  This routine is just a debugging
    3616             :  *      cross-check that no pins remain.
    3617             :  */
    3618             : static void
    3619      783148 : CheckForBufferLeaks(void)
    3620             : {
    3621             : #ifdef USE_ASSERT_CHECKING
    3622             :     int         RefCountErrors = 0;
    3623             :     PrivateRefCountEntry *res;
    3624             :     int         i;
    3625             :     char       *s;
    3626             : 
    3627             :     /* check the array */
    3628             :     for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
    3629             :     {
    3630             :         res = &PrivateRefCountArray[i];
    3631             : 
    3632             :         if (res->buffer != InvalidBuffer)
    3633             :         {
    3634             :             s = DebugPrintBufferRefcount(res->buffer);
    3635             :             elog(WARNING, "buffer refcount leak: %s", s);
    3636             :             pfree(s);
    3637             : 
    3638             :             RefCountErrors++;
    3639             :         }
    3640             :     }
    3641             : 
    3642             :     /* if necessary search the hash */
    3643             :     if (PrivateRefCountOverflowed)
    3644             :     {
    3645             :         HASH_SEQ_STATUS hstat;
    3646             : 
    3647             :         hash_seq_init(&hstat, PrivateRefCountHash);
    3648             :         while ((res = (PrivateRefCountEntry *) hash_seq_search(&hstat)) != NULL)
    3649             :         {
    3650             :             s = DebugPrintBufferRefcount(res->buffer);
    3651             :             elog(WARNING, "buffer refcount leak: %s", s);
    3652             :             pfree(s);
    3653             :             RefCountErrors++;
    3654             :         }
    3655             :     }
    3656             : 
    3657             :     Assert(RefCountErrors == 0);
    3658             : #endif
    3659      783148 : }
    3660             : 
    3661             : /*
    3662             :  * Helper routine to issue warnings when a buffer is unexpectedly pinned
    3663             :  */
    3664             : char *
    3665           0 : DebugPrintBufferRefcount(Buffer buffer)
    3666             : {
    3667             :     BufferDesc *buf;
    3668             :     int32       loccount;
    3669             :     char       *path;
    3670             :     char       *result;
    3671             :     ProcNumber  backend;
    3672             :     uint32      buf_state;
    3673             : 
    3674             :     Assert(BufferIsValid(buffer));
    3675           0 :     if (BufferIsLocal(buffer))
    3676             :     {
    3677           0 :         buf = GetLocalBufferDescriptor(-buffer - 1);
    3678           0 :         loccount = LocalRefCount[-buffer - 1];
    3679           0 :         backend = MyProcNumber;
    3680             :     }
    3681             :     else
    3682             :     {
    3683           0 :         buf = GetBufferDescriptor(buffer - 1);
    3684           0 :         loccount = GetPrivateRefCount(buffer);
    3685           0 :         backend = INVALID_PROC_NUMBER;
    3686             :     }
    3687             : 
    3688             :     /* theoretically we should lock the bufhdr here */
    3689           0 :     path = relpathbackend(BufTagGetRelFileLocator(&buf->tag), backend,
    3690             :                           BufTagGetForkNum(&buf->tag));
    3691           0 :     buf_state = pg_atomic_read_u32(&buf->state);
    3692             : 
    3693           0 :     result = psprintf("[%03d] (rel=%s, blockNum=%u, flags=0x%x, refcount=%u %d)",
    3694             :                       buffer, path,
    3695             :                       buf->tag.blockNum, buf_state & BUF_FLAG_MASK,
    3696             :                       BUF_STATE_GET_REFCOUNT(buf_state), loccount);
    3697           0 :     pfree(path);
    3698           0 :     return result;
    3699             : }
    3700             : 
    3701             : /*
    3702             :  * CheckPointBuffers
    3703             :  *
    3704             :  * Flush all dirty blocks in buffer pool to disk at checkpoint time.
    3705             :  *
    3706             :  * Note: temporary relations do not participate in checkpoints, so they don't
    3707             :  * need to be flushed.
    3708             :  */
    3709             : void
    3710        2468 : CheckPointBuffers(int flags)
    3711             : {
    3712        2468 :     BufferSync(flags);
    3713        2468 : }
    3714             : 
    3715             : /*
    3716             :  * BufferGetBlockNumber
    3717             :  *      Returns the block number associated with a buffer.
    3718             :  *
    3719             :  * Note:
    3720             :  *      Assumes that the buffer is valid and pinned, else the
    3721             :  *      value may be obsolete immediately...
    3722             :  */
    3723             : BlockNumber
    3724    87156254 : BufferGetBlockNumber(Buffer buffer)
    3725             : {
    3726             :     BufferDesc *bufHdr;
    3727             : 
    3728             :     Assert(BufferIsPinned(buffer));
    3729             : 
    3730    87156254 :     if (BufferIsLocal(buffer))
    3731     3329332 :         bufHdr = GetLocalBufferDescriptor(-buffer - 1);
    3732             :     else
    3733    83826922 :         bufHdr = GetBufferDescriptor(buffer - 1);
    3734             : 
    3735             :     /* pinned, so OK to read tag without spinlock */
    3736    87156254 :     return bufHdr->tag.blockNum;
    3737             : }
    3738             : 
    3739             : /*
    3740             :  * BufferGetTag
    3741             :  *      Returns the relfilelocator, fork number and block number associated with
    3742             :  *      a buffer.
    3743             :  */
    3744             : void
    3745    28426344 : BufferGetTag(Buffer buffer, RelFileLocator *rlocator, ForkNumber *forknum,
    3746             :              BlockNumber *blknum)
    3747             : {
    3748             :     BufferDesc *bufHdr;
    3749             : 
    3750             :     /* Do the same checks as BufferGetBlockNumber. */
    3751             :     Assert(BufferIsPinned(buffer));
    3752             : 
    3753    28426344 :     if (BufferIsLocal(buffer))
    3754           0 :         bufHdr = GetLocalBufferDescriptor(-buffer - 1);
    3755             :     else
    3756    28426344 :         bufHdr = GetBufferDescriptor(buffer - 1);
    3757             : 
    3758             :     /* pinned, so OK to read tag without spinlock */
    3759    28426344 :     *rlocator = BufTagGetRelFileLocator(&bufHdr->tag);
    3760    28426344 :     *forknum = BufTagGetForkNum(&bufHdr->tag);
    3761    28426344 :     *blknum = bufHdr->tag.blockNum;
    3762    28426344 : }
    3763             : 
    3764             : /*
    3765             :  * FlushBuffer
    3766             :  *      Physically write out a shared buffer.
    3767             :  *
    3768             :  * NOTE: this actually just passes the buffer contents to the kernel; the
    3769             :  * real write to disk won't happen until the kernel feels like it.  This
    3770             :  * is okay from our point of view since we can redo the changes from WAL.
    3771             :  * However, we will need to force the changes to disk via fsync before
    3772             :  * we can checkpoint WAL.
    3773             :  *
    3774             :  * The caller must hold a pin on the buffer and have share-locked the
    3775             :  * buffer contents.  (Note: a share-lock does not prevent updates of
    3776             :  * hint bits in the buffer, so the page could change while the write
    3777             :  * is in progress, but we assume that that will not invalidate the data
    3778             :  * written.)
    3779             :  *
    3780             :  * If the caller has an smgr reference for the buffer's relation, pass it
    3781             :  * as the second parameter.  If not, pass NULL.
    3782             :  */
    3783             : static void
    3784      952604 : FlushBuffer(BufferDesc *buf, SMgrRelation reln, IOObject io_object,
    3785             :             IOContext io_context)
    3786             : {
    3787             :     XLogRecPtr  recptr;
    3788             :     ErrorContextCallback errcallback;
    3789             :     instr_time  io_start;
    3790             :     Block       bufBlock;
    3791             :     char       *bufToWrite;
    3792             :     uint32      buf_state;
    3793             : 
    3794             :     /*
    3795             :      * Try to start an I/O operation.  If StartBufferIO returns false, then
    3796             :      * someone else flushed the buffer before we could, so we need not do
    3797             :      * anything.
    3798             :      */
    3799      952604 :     if (!StartBufferIO(buf, false, false))
    3800          26 :         return;
    3801             : 
    3802             :     /* Setup error traceback support for ereport() */
    3803      952578 :     errcallback.callback = shared_buffer_write_error_callback;
    3804      952578 :     errcallback.arg = buf;
    3805      952578 :     errcallback.previous = error_context_stack;
    3806      952578 :     error_context_stack = &errcallback;
    3807             : 
    3808             :     /* Find smgr relation for buffer */
    3809      952578 :     if (reln == NULL)
    3810      945484 :         reln = smgropen(BufTagGetRelFileLocator(&buf->tag), INVALID_PROC_NUMBER);
    3811             : 
    3812             :     TRACE_POSTGRESQL_BUFFER_FLUSH_START(BufTagGetForkNum(&buf->tag),
    3813             :                                         buf->tag.blockNum,
    3814             :                                         reln->smgr_rlocator.locator.spcOid,
    3815             :                                         reln->smgr_rlocator.locator.dbOid,
    3816             :                                         reln->smgr_rlocator.locator.relNumber);
    3817             : 
    3818      952578 :     buf_state = LockBufHdr(buf);
    3819             : 
    3820             :     /*
    3821             :      * Run PageGetLSN while holding header lock, since we don't have the
    3822             :      * buffer locked exclusively in all cases.
    3823             :      */
    3824      952578 :     recptr = BufferGetLSN(buf);
    3825             : 
    3826             :     /* To check if block content changes while flushing. - vadim 01/17/97 */
    3827      952578 :     buf_state &= ~BM_JUST_DIRTIED;
    3828      952578 :     UnlockBufHdr(buf, buf_state);
    3829             : 
    3830             :     /*
    3831             :      * Force XLOG flush up to buffer's LSN.  This implements the basic WAL
    3832             :      * rule that log updates must hit disk before any of the data-file changes
    3833             :      * they describe do.
    3834             :      *
    3835             :      * However, this rule does not apply to unlogged relations, which will be
    3836             :      * lost after a crash anyway.  Most unlogged relation pages do not bear
    3837             :      * LSNs since we never emit WAL records for them, and therefore flushing
    3838             :      * up through the buffer LSN would be useless, but harmless.  However,
    3839             :      * GiST indexes use LSNs internally to track page-splits, and therefore
    3840             :      * unlogged GiST pages bear "fake" LSNs generated by
    3841             :      * GetFakeLSNForUnloggedRel.  It is unlikely but possible that the fake
    3842             :      * LSN counter could advance past the WAL insertion point; and if it did
    3843             :      * happen, attempting to flush WAL through that location would fail, with
    3844             :      * disastrous system-wide consequences.  To make sure that can't happen,
    3845             :      * skip the flush if the buffer isn't permanent.
    3846             :      */
    3847      952578 :     if (buf_state & BM_PERMANENT)
    3848      949024 :         XLogFlush(recptr);
    3849             : 
    3850             :     /*
    3851             :      * Now it's safe to write buffer to disk. Note that no one else should
    3852             :      * have been able to write it while we were busy with log flushing because
    3853             :      * only one process at a time can set the BM_IO_IN_PROGRESS bit.
    3854             :      */
    3855      952578 :     bufBlock = BufHdrGetBlock(buf);
    3856             : 
    3857             :     /*
    3858             :      * Update page checksum if desired.  Since we have only shared lock on the
    3859             :      * buffer, other processes might be updating hint bits in it, so we must
    3860             :      * copy the page to private storage if we do checksumming.
    3861             :      */
    3862      952578 :     bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
    3863             : 
    3864      952578 :     io_start = pgstat_prepare_io_time(track_io_timing);
    3865             : 
    3866             :     /*
    3867             :      * bufToWrite is either the shared buffer or a copy, as appropriate.
    3868             :      */
    3869      952578 :     smgrwrite(reln,
    3870      952578 :               BufTagGetForkNum(&buf->tag),
    3871             :               buf->tag.blockNum,
    3872             :               bufToWrite,
    3873             :               false);
    3874             : 
    3875             :     /*
    3876             :      * When a strategy is in use, only flushes of dirty buffers already in the
    3877             :      * strategy ring are counted as strategy writes (IOCONTEXT
    3878             :      * [BULKREAD|BULKWRITE|VACUUM] IOOP_WRITE) for the purpose of IO
    3879             :      * statistics tracking.
    3880             :      *
    3881             :      * If a shared buffer initially added to the ring must be flushed before
    3882             :      * being used, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE.
    3883             :      *
    3884             :      * If a shared buffer which was added to the ring later because the
    3885             :      * current strategy buffer is pinned or in use or because all strategy
    3886             :      * buffers were dirty and rejected (for BAS_BULKREAD operations only)
    3887             :      * requires flushing, this is counted as an IOCONTEXT_NORMAL IOOP_WRITE
    3888             :      * (from_ring will be false).
    3889             :      *
    3890             :      * When a strategy is not in use, the write can only be a "regular" write
    3891             :      * of a dirty shared buffer (IOCONTEXT_NORMAL IOOP_WRITE).
    3892             :      */
    3893      952578 :     pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
    3894             :                             IOOP_WRITE, io_start, 1);
    3895             : 
    3896      952578 :     pgBufferUsage.shared_blks_written++;
    3897             : 
    3898             :     /*
    3899             :      * Mark the buffer as clean (unless BM_JUST_DIRTIED has become set) and
    3900             :      * end the BM_IO_IN_PROGRESS state.
    3901             :      */
    3902      952578 :     TerminateBufferIO(buf, true, 0, true);
    3903             : 
    3904             :     TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(BufTagGetForkNum(&buf->tag),
    3905             :                                        buf->tag.blockNum,
    3906             :                                        reln->smgr_rlocator.locator.spcOid,
    3907             :                                        reln->smgr_rlocator.locator.dbOid,
    3908             :                                        reln->smgr_rlocator.locator.relNumber);
    3909             : 
    3910             :     /* Pop the error context stack */
    3911      952578 :     error_context_stack = errcallback.previous;
    3912             : }
    3913             : 
    3914             : /*
    3915             :  * RelationGetNumberOfBlocksInFork
    3916             :  *      Determines the current number of pages in the specified relation fork.
    3917             :  *
    3918             :  * Note that the accuracy of the result will depend on the details of the
    3919             :  * relation's storage. For builtin AMs it'll be accurate, but for external AMs
    3920             :  * it might not be.
    3921             :  */
    3922             : BlockNumber
    3923     3104170 : RelationGetNumberOfBlocksInFork(Relation relation, ForkNumber forkNum)
    3924             : {
    3925     3104170 :     if (RELKIND_HAS_TABLE_AM(relation->rd_rel->relkind))
    3926             :     {
    3927             :         /*
    3928             :          * Not every table AM uses BLCKSZ wide fixed size blocks. Therefore
    3929             :          * tableam returns the size in bytes - but for the purpose of this
    3930             :          * routine, we want the number of blocks. Therefore divide, rounding
    3931             :          * up.
    3932             :          */
    3933             :         uint64      szbytes;
    3934             : 
    3935     2319398 :         szbytes = table_relation_size(relation, forkNum);
    3936             : 
    3937     2319360 :         return (szbytes + (BLCKSZ - 1)) / BLCKSZ;
    3938             :     }
    3939      784772 :     else if (RELKIND_HAS_STORAGE(relation->rd_rel->relkind))
    3940             :     {
    3941      784772 :         return smgrnblocks(RelationGetSmgr(relation), forkNum);
    3942             :     }
    3943             :     else
    3944             :         Assert(false);
    3945             : 
    3946           0 :     return 0;                   /* keep compiler quiet */
    3947             : }
    3948             : 
    3949             : /*
    3950             :  * BufferIsPermanent
    3951             :  *      Determines whether a buffer will potentially still be around after
    3952             :  *      a crash.  Caller must hold a buffer pin.
    3953             :  */
    3954             : bool
    3955    18465460 : BufferIsPermanent(Buffer buffer)
    3956             : {
    3957             :     BufferDesc *bufHdr;
    3958             : 
    3959             :     /* Local buffers are used only for temp relations. */
    3960    18465460 :     if (BufferIsLocal(buffer))
    3961     1151336 :         return false;
    3962             : 
    3963             :     /* Make sure we've got a real buffer, and that we hold a pin on it. */
    3964             :     Assert(BufferIsValid(buffer));
    3965             :     Assert(BufferIsPinned(buffer));
    3966             : 
    3967             :     /*
    3968             :      * BM_PERMANENT can't be changed while we hold a pin on the buffer, so we
    3969             :      * need not bother with the buffer header spinlock.  Even if someone else
    3970             :      * changes the buffer header state while we're doing this, the state is
    3971             :      * changed atomically, so we'll read the old value or the new value, but
    3972             :      * not random garbage.
    3973             :      */
    3974    17314124 :     bufHdr = GetBufferDescriptor(buffer - 1);
    3975    17314124 :     return (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT) != 0;
    3976             : }
    3977             : 
    3978             : /*
    3979             :  * BufferGetLSNAtomic
    3980             :  *      Retrieves the LSN of the buffer atomically using a buffer header lock.
    3981             :  *      This is necessary for some callers who may not have an exclusive lock
    3982             :  *      on the buffer.
    3983             :  */
    3984             : XLogRecPtr
    3985    15569456 : BufferGetLSNAtomic(Buffer buffer)
    3986             : {
    3987    15569456 :     BufferDesc *bufHdr = GetBufferDescriptor(buffer - 1);
    3988    15569456 :     char       *page = BufferGetPage(buffer);
    3989             :     XLogRecPtr  lsn;
    3990             :     uint32      buf_state;
    3991             : 
    3992             :     /*
    3993             :      * If we don't need locking for correctness, fastpath out.
    3994             :      */
    3995    15569456 :     if (!XLogHintBitIsNeeded() || BufferIsLocal(buffer))
    3996      570420 :         return PageGetLSN(page);
    3997             : 
    3998             :     /* Make sure we've got a real buffer, and that we hold a pin on it. */
    3999             :     Assert(BufferIsValid(buffer));
    4000             :     Assert(BufferIsPinned(buffer));
    4001             : 
    4002    14999036 :     buf_state = LockBufHdr(bufHdr);
    4003    14999036 :     lsn = PageGetLSN(page);
    4004    14999036 :     UnlockBufHdr(bufHdr, buf_state);
    4005             : 
    4006    14999036 :     return lsn;
    4007             : }
    4008             : 
    4009             : /* ---------------------------------------------------------------------
    4010             :  *      DropRelationBuffers
    4011             :  *
    4012             :  *      This function removes from the buffer pool all the pages of the
    4013             :  *      specified relation forks that have block numbers >= firstDelBlock.
    4014             :  *      (In particular, with firstDelBlock = 0, all pages are removed.)
    4015             :  *      Dirty pages are simply dropped, without bothering to write them
    4016             :  *      out first.  Therefore, this is NOT rollback-able, and so should be
    4017             :  *      used only with extreme caution!
    4018             :  *
    4019             :  *      Currently, this is called only from smgr.c when the underlying file
    4020             :  *      is about to be deleted or truncated (firstDelBlock is needed for
    4021             :  *      the truncation case).  The data in the affected pages would therefore
    4022             :  *      be deleted momentarily anyway, and there is no point in writing it.
    4023             :  *      It is the responsibility of higher-level code to ensure that the
    4024             :  *      deletion or truncation does not lose any data that could be needed
    4025             :  *      later.  It is also the responsibility of higher-level code to ensure
    4026             :  *      that no other process could be trying to load more pages of the
    4027             :  *      relation into buffers.
    4028             :  * --------------------------------------------------------------------
    4029             :  */
    4030             : void
    4031        1152 : DropRelationBuffers(SMgrRelation smgr_reln, ForkNumber *forkNum,
    4032             :                     int nforks, BlockNumber *firstDelBlock)
    4033             : {
    4034             :     int         i;
    4035             :     int         j;
    4036             :     RelFileLocatorBackend rlocator;
    4037             :     BlockNumber nForkBlock[MAX_FORKNUM];
    4038        1152 :     uint64      nBlocksToInvalidate = 0;
    4039             : 
    4040        1152 :     rlocator = smgr_reln->smgr_rlocator;
    4041             : 
    4042             :     /* If it's a local relation, it's localbuf.c's problem. */
    4043        1152 :     if (RelFileLocatorBackendIsTemp(rlocator))
    4044             :     {
    4045         658 :         if (rlocator.backend == MyProcNumber)
    4046             :         {
    4047        1350 :             for (j = 0; j < nforks; j++)
    4048         692 :                 DropRelationLocalBuffers(rlocator.locator, forkNum[j],
    4049         692 :                                          firstDelBlock[j]);
    4050             :         }
    4051         730 :         return;
    4052             :     }
    4053             : 
    4054             :     /*
    4055             :      * To remove all the pages of the specified relation forks from the buffer
    4056             :      * pool, we need to scan the entire buffer pool but we can optimize it by
    4057             :      * finding the buffers from BufMapping table provided we know the exact
    4058             :      * size of each fork of the relation. The exact size is required to ensure
    4059             :      * that we don't leave any buffer for the relation being dropped as
    4060             :      * otherwise the background writer or checkpointer can lead to a PANIC
    4061             :      * error while flushing buffers corresponding to files that don't exist.
    4062             :      *
    4063             :      * To know the exact size, we rely on the size cached for each fork by us
    4064             :      * during recovery which limits the optimization to recovery and on
    4065             :      * standbys but we can easily extend it once we have shared cache for
    4066             :      * relation size.
    4067             :      *
    4068             :      * In recovery, we cache the value returned by the first lseek(SEEK_END)
    4069             :      * and the future writes keeps the cached value up-to-date. See
    4070             :      * smgrextend. It is possible that the value of the first lseek is smaller
    4071             :      * than the actual number of existing blocks in the file due to buggy
    4072             :      * Linux kernels that might not have accounted for the recent write. But
    4073             :      * that should be fine because there must not be any buffers after that
    4074             :      * file size.
    4075             :      */
    4076         662 :     for (i = 0; i < nforks; i++)
    4077             :     {
    4078             :         /* Get the number of blocks for a relation's fork */
    4079         572 :         nForkBlock[i] = smgrnblocks_cached(smgr_reln, forkNum[i]);
    4080             : 
    4081         572 :         if (nForkBlock[i] == InvalidBlockNumber)
    4082             :         {
    4083         404 :             nBlocksToInvalidate = InvalidBlockNumber;
    4084         404 :             break;
    4085             :         }
    4086             : 
    4087             :         /* calculate the number of blocks to be invalidated */
    4088         168 :         nBlocksToInvalidate += (nForkBlock[i] - firstDelBlock[i]);
    4089             :     }
    4090             : 
    4091             :     /*
    4092             :      * We apply the optimization iff the total number of blocks to invalidate
    4093             :      * is below the BUF_DROP_FULL_SCAN_THRESHOLD.
    4094             :      */
    4095         494 :     if (BlockNumberIsValid(nBlocksToInvalidate) &&
    4096          90 :         nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD)
    4097             :     {
    4098         198 :         for (j = 0; j < nforks; j++)
    4099         126 :             FindAndDropRelationBuffers(rlocator.locator, forkNum[j],
    4100         126 :                                        nForkBlock[j], firstDelBlock[j]);
    4101          72 :         return;
    4102             :     }
    4103             : 
    4104     5646502 :     for (i = 0; i < NBuffers; i++)
    4105             :     {
    4106     5646080 :         BufferDesc *bufHdr = GetBufferDescriptor(i);
    4107             :         uint32      buf_state;
    4108             : 
    4109             :         /*
    4110             :          * We can make this a tad faster by prechecking the buffer tag before
    4111             :          * we attempt to lock the buffer; this saves a lot of lock
    4112             :          * acquisitions in typical cases.  It should be safe because the
    4113             :          * caller must have AccessExclusiveLock on the relation, or some other
    4114             :          * reason to be certain that no one is loading new pages of the rel
    4115             :          * into the buffer pool.  (Otherwise we might well miss such pages
    4116             :          * entirely.)  Therefore, while the tag might be changing while we
    4117             :          * look at it, it can't be changing *to* a value we care about, only
    4118             :          * *away* from such a value.  So false negatives are impossible, and
    4119             :          * false positives are safe because we'll recheck after getting the
    4120             :          * buffer lock.
    4121             :          *
    4122             :          * We could check forkNum and blockNum as well as the rlocator, but
    4123             :          * the incremental win from doing so seems small.
    4124             :          */
    4125     5646080 :         if (!BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator))
    4126     5632852 :             continue;
    4127             : 
    4128       13228 :         buf_state = LockBufHdr(bufHdr);
    4129             : 
    4130       34452 :         for (j = 0; j < nforks; j++)
    4131             :         {
    4132       24094 :             if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator.locator) &&
    4133       24094 :                 BufTagGetForkNum(&bufHdr->tag) == forkNum[j] &&
    4134       13052 :                 bufHdr->tag.blockNum >= firstDelBlock[j])
    4135             :             {
    4136        2870 :                 InvalidateBuffer(bufHdr);   /* releases spinlock */
    4137        2870 :                 break;
    4138             :             }
    4139             :         }
    4140       13228 :         if (j >= nforks)
    4141       10358 :             UnlockBufHdr(bufHdr, buf_state);
    4142             :     }
    4143             : }
    4144             : 
    4145             : /* ---------------------------------------------------------------------
    4146             :  *      DropRelationsAllBuffers
    4147             :  *
    4148             :  *      This function removes from the buffer pool all the pages of all
    4149             :  *      forks of the specified relations.  It's equivalent to calling
    4150             :  *      DropRelationBuffers once per fork per relation with firstDelBlock = 0.
    4151             :  *      --------------------------------------------------------------------
    4152             :  */
    4153             : void
    4154       25368 : DropRelationsAllBuffers(SMgrRelation *smgr_reln, int nlocators)
    4155             : {
    4156             :     int         i;
    4157       25368 :     int         n = 0;
    4158             :     SMgrRelation *rels;
    4159             :     BlockNumber (*block)[MAX_FORKNUM + 1];
    4160       25368 :     uint64      nBlocksToInvalidate = 0;
    4161             :     RelFileLocator *locators;
    4162       25368 :     bool        cached = true;
    4163             :     bool        use_bsearch;
    4164             : 
    4165       25368 :     if (nlocators == 0)
    4166           0 :         return;
    4167             : 
    4168       25368 :     rels = palloc(sizeof(SMgrRelation) * nlocators);    /* non-local relations */
    4169             : 
    4170             :     /* If it's a local relation, it's localbuf.c's problem. */
    4171      112804 :     for (i = 0; i < nlocators; i++)
    4172             :     {
    4173       87436 :         if (RelFileLocatorBackendIsTemp(smgr_reln[i]->smgr_rlocator))
    4174             :         {
    4175        5998 :             if (smgr_reln[i]->smgr_rlocator.backend == MyProcNumber)
    4176        5998 :                 DropRelationAllLocalBuffers(smgr_reln[i]->smgr_rlocator.locator);
    4177             :         }
    4178             :         else
    4179       81438 :             rels[n++] = smgr_reln[i];
    4180             :     }
    4181             : 
    4182             :     /*
    4183             :      * If there are no non-local relations, then we're done. Release the
    4184             :      * memory and return.
    4185             :      */
    4186       25368 :     if (n == 0)
    4187             :     {
    4188        1498 :         pfree(rels);
    4189        1498 :         return;
    4190             :     }
    4191             : 
    4192             :     /*
    4193             :      * This is used to remember the number of blocks for all the relations
    4194             :      * forks.
    4195             :      */
    4196             :     block = (BlockNumber (*)[MAX_FORKNUM + 1])
    4197       23870 :         palloc(sizeof(BlockNumber) * n * (MAX_FORKNUM + 1));
    4198             : 
    4199             :     /*
    4200             :      * We can avoid scanning the entire buffer pool if we know the exact size
    4201             :      * of each of the given relation forks. See DropRelationBuffers.
    4202             :      */
    4203       50306 :     for (i = 0; i < n && cached; i++)
    4204             :     {
    4205       43086 :         for (int j = 0; j <= MAX_FORKNUM; j++)
    4206             :         {
    4207             :             /* Get the number of blocks for a relation's fork. */
    4208       38946 :             block[i][j] = smgrnblocks_cached(rels[i], j);
    4209             : 
    4210             :             /* We need to only consider the relation forks that exists. */
    4211       38946 :             if (block[i][j] == InvalidBlockNumber)
    4212             :             {
    4213       34446 :                 if (!smgrexists(rels[i], j))
    4214       12150 :                     continue;
    4215       22296 :                 cached = false;
    4216       22296 :                 break;
    4217             :             }
    4218             : 
    4219             :             /* calculate the total number of blocks to be invalidated */
    4220        4500 :             nBlocksToInvalidate += block[i][j];
    4221             :         }
    4222             :     }
    4223             : 
    4224             :     /*
    4225             :      * We apply the optimization iff the total number of blocks to invalidate
    4226             :      * is below the BUF_DROP_FULL_SCAN_THRESHOLD.
    4227             :      */
    4228       23870 :     if (cached && nBlocksToInvalidate < BUF_DROP_FULL_SCAN_THRESHOLD)
    4229             :     {
    4230        2568 :         for (i = 0; i < n; i++)
    4231             :         {
    4232        7050 :             for (int j = 0; j <= MAX_FORKNUM; j++)
    4233             :             {
    4234             :                 /* ignore relation forks that doesn't exist */
    4235        5640 :                 if (!BlockNumberIsValid(block[i][j]))
    4236        4212 :                     continue;
    4237             : 
    4238             :                 /* drop all the buffers for a particular relation fork */
    4239        1428 :                 FindAndDropRelationBuffers(rels[i]->smgr_rlocator.locator,
    4240        1428 :                                            j, block[i][j], 0);
    4241             :             }
    4242             :         }
    4243             : 
    4244        1158 :         pfree(block);
    4245        1158 :         pfree(rels);
    4246        1158 :         return;
    4247             :     }
    4248             : 
    4249       22712 :     pfree(block);
    4250       22712 :     locators = palloc(sizeof(RelFileLocator) * n);  /* non-local relations */
    4251      102740 :     for (i = 0; i < n; i++)
    4252       80028 :         locators[i] = rels[i]->smgr_rlocator.locator;
    4253             : 
    4254             :     /*
    4255             :      * For low number of relations to drop just use a simple walk through, to
    4256             :      * save the bsearch overhead. The threshold to use is rather a guess than
    4257             :      * an exactly determined value, as it depends on many factors (CPU and RAM
    4258             :      * speeds, amount of shared buffers etc.).
    4259             :      */
    4260       22712 :     use_bsearch = n > RELS_BSEARCH_THRESHOLD;
    4261             : 
    4262             :     /* sort the list of rlocators if necessary */
    4263       22712 :     if (use_bsearch)
    4264         334 :         qsort(locators, n, sizeof(RelFileLocator), rlocator_comparator);
    4265             : 
    4266   248362936 :     for (i = 0; i < NBuffers; i++)
    4267             :     {
    4268   248340224 :         RelFileLocator *rlocator = NULL;
    4269   248340224 :         BufferDesc *bufHdr = GetBufferDescriptor(i);
    4270             :         uint32      buf_state;
    4271             : 
    4272             :         /*
    4273             :          * As in DropRelationBuffers, an unlocked precheck should be safe and
    4274             :          * saves some cycles.
    4275             :          */
    4276             : 
    4277   248340224 :         if (!use_bsearch)
    4278             :         {
    4279             :             int         j;
    4280             : 
    4281  1001203630 :             for (j = 0; j < n; j++)
    4282             :             {
    4283   756547684 :                 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &locators[j]))
    4284             :                 {
    4285      162742 :                     rlocator = &locators[j];
    4286      162742 :                     break;
    4287             :                 }
    4288             :             }
    4289             :         }
    4290             :         else
    4291             :         {
    4292             :             RelFileLocator locator;
    4293             : 
    4294     3521536 :             locator = BufTagGetRelFileLocator(&bufHdr->tag);
    4295     3521536 :             rlocator = bsearch((const void *) &(locator),
    4296             :                                locators, n, sizeof(RelFileLocator),
    4297             :                                rlocator_comparator);
    4298             :         }
    4299             : 
    4300             :         /* buffer doesn't belong to any of the given relfilelocators; skip it */
    4301   248340224 :         if (rlocator == NULL)
    4302   248174070 :             continue;
    4303             : 
    4304      166154 :         buf_state = LockBufHdr(bufHdr);
    4305      166154 :         if (BufTagMatchesRelFileLocator(&bufHdr->tag, rlocator))
    4306      166154 :             InvalidateBuffer(bufHdr);   /* releases spinlock */
    4307             :         else
    4308           0 :             UnlockBufHdr(bufHdr, buf_state);
    4309             :     }
    4310             : 
    4311       22712 :     pfree(locators);
    4312       22712 :     pfree(rels);
    4313             : }
    4314             : 
    4315             : /* ---------------------------------------------------------------------
    4316             :  *      FindAndDropRelationBuffers
    4317             :  *
    4318             :  *      This function performs look up in BufMapping table and removes from the
    4319             :  *      buffer pool all the pages of the specified relation fork that has block
    4320             :  *      number >= firstDelBlock. (In particular, with firstDelBlock = 0, all
    4321             :  *      pages are removed.)
    4322             :  * --------------------------------------------------------------------
    4323             :  */
    4324             : static void
    4325        1554 : FindAndDropRelationBuffers(RelFileLocator rlocator, ForkNumber forkNum,
    4326             :                            BlockNumber nForkBlock,
    4327             :                            BlockNumber firstDelBlock)
    4328             : {
    4329             :     BlockNumber curBlock;
    4330             : 
    4331        3782 :     for (curBlock = firstDelBlock; curBlock < nForkBlock; curBlock++)
    4332             :     {
    4333             :         uint32      bufHash;    /* hash value for tag */
    4334             :         BufferTag   bufTag;     /* identity of requested block */
    4335             :         LWLock     *bufPartitionLock;   /* buffer partition lock for it */
    4336             :         int         buf_id;
    4337             :         BufferDesc *bufHdr;
    4338             :         uint32      buf_state;
    4339             : 
    4340             :         /* create a tag so we can lookup the buffer */
    4341        2228 :         InitBufferTag(&bufTag, &rlocator, forkNum, curBlock);
    4342             : 
    4343             :         /* determine its hash code and partition lock ID */
    4344        2228 :         bufHash = BufTableHashCode(&bufTag);
    4345        2228 :         bufPartitionLock = BufMappingPartitionLock(bufHash);
    4346             : 
    4347             :         /* Check that it is in the buffer pool. If not, do nothing. */
    4348        2228 :         LWLockAcquire(bufPartitionLock, LW_SHARED);
    4349        2228 :         buf_id = BufTableLookup(&bufTag, bufHash);
    4350        2228 :         LWLockRelease(bufPartitionLock);
    4351             : 
    4352        2228 :         if (buf_id < 0)
    4353         182 :             continue;
    4354             : 
    4355        2046 :         bufHdr = GetBufferDescriptor(buf_id);
    4356             : 
    4357             :         /*
    4358             :          * We need to lock the buffer header and recheck if the buffer is
    4359             :          * still associated with the same block because the buffer could be
    4360             :          * evicted by some other backend loading blocks for a different
    4361             :          * relation after we release lock on the BufMapping table.
    4362             :          */
    4363        2046 :         buf_state = LockBufHdr(bufHdr);
    4364             : 
    4365        4092 :         if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rlocator) &&
    4366        2046 :             BufTagGetForkNum(&bufHdr->tag) == forkNum &&
    4367        2046 :             bufHdr->tag.blockNum >= firstDelBlock)
    4368        2046 :             InvalidateBuffer(bufHdr);   /* releases spinlock */
    4369             :         else
    4370           0 :             UnlockBufHdr(bufHdr, buf_state);
    4371             :     }
    4372        1554 : }
    4373             : 
    4374             : /* ---------------------------------------------------------------------
    4375             :  *      DropDatabaseBuffers
    4376             :  *
    4377             :  *      This function removes all the buffers in the buffer cache for a
    4378             :  *      particular database.  Dirty pages are simply dropped, without
    4379             :  *      bothering to write them out first.  This is used when we destroy a
    4380             :  *      database, to avoid trying to flush data to disk when the directory
    4381             :  *      tree no longer exists.  Implementation is pretty similar to
    4382             :  *      DropRelationBuffers() which is for destroying just one relation.
    4383             :  * --------------------------------------------------------------------
    4384             :  */
    4385             : void
    4386         114 : DropDatabaseBuffers(Oid dbid)
    4387             : {
    4388             :     int         i;
    4389             : 
    4390             :     /*
    4391             :      * We needn't consider local buffers, since by assumption the target
    4392             :      * database isn't our own.
    4393             :      */
    4394             : 
    4395      632434 :     for (i = 0; i < NBuffers; i++)
    4396             :     {
    4397      632320 :         BufferDesc *bufHdr = GetBufferDescriptor(i);
    4398             :         uint32      buf_state;
    4399             : 
    4400             :         /*
    4401             :          * As in DropRelationBuffers, an unlocked precheck should be safe and
    4402             :          * saves some cycles.
    4403             :          */
    4404      632320 :         if (bufHdr->tag.dbOid != dbid)
    4405      606530 :             continue;
    4406             : 
    4407       25790 :         buf_state = LockBufHdr(bufHdr);
    4408       25790 :         if (bufHdr->tag.dbOid == dbid)
    4409       25790 :             InvalidateBuffer(bufHdr);   /* releases spinlock */
    4410             :         else
    4411           0 :             UnlockBufHdr(bufHdr, buf_state);
    4412             :     }
    4413         114 : }
    4414             : 
    4415             : /* -----------------------------------------------------------------
    4416             :  *      PrintBufferDescs
    4417             :  *
    4418             :  *      this function prints all the buffer descriptors, for debugging
    4419             :  *      use only.
    4420             :  * -----------------------------------------------------------------
    4421             :  */
    4422             : #ifdef NOT_USED
    4423             : void
    4424             : PrintBufferDescs(void)
    4425             : {
    4426             :     int         i;
    4427             : 
    4428             :     for (i = 0; i < NBuffers; ++i)
    4429             :     {
    4430             :         BufferDesc *buf = GetBufferDescriptor(i);
    4431             :         Buffer      b = BufferDescriptorGetBuffer(buf);
    4432             : 
    4433             :         /* theoretically we should lock the bufhdr here */
    4434             :         elog(LOG,
    4435             :              "[%02d] (freeNext=%d, rel=%s, "
    4436             :              "blockNum=%u, flags=0x%x, refcount=%u %d)",
    4437             :              i, buf->freeNext,
    4438             :              relpathbackend(BufTagGetRelFileLocator(&buf->tag),
    4439             :                             INVALID_PROC_NUMBER, BufTagGetForkNum(&buf->tag)),
    4440             :              buf->tag.blockNum, buf->flags,
    4441             :              buf->refcount, GetPrivateRefCount(b));
    4442             :     }
    4443             : }
    4444             : #endif
    4445             : 
    4446             : #ifdef NOT_USED
    4447             : void
    4448             : PrintPinnedBufs(void)
    4449             : {
    4450             :     int         i;
    4451             : 
    4452             :     for (i = 0; i < NBuffers; ++i)
    4453             :     {
    4454             :         BufferDesc *buf = GetBufferDescriptor(i);
    4455             :         Buffer      b = BufferDescriptorGetBuffer(buf);
    4456             : 
    4457             :         if (GetPrivateRefCount(b) > 0)
    4458             :         {
    4459             :             /* theoretically we should lock the bufhdr here */
    4460             :             elog(LOG,
    4461             :                  "[%02d] (freeNext=%d, rel=%s, "
    4462             :                  "blockNum=%u, flags=0x%x, refcount=%u %d)",
    4463             :                  i, buf->freeNext,
    4464             :                  relpathperm(BufTagGetRelFileLocator(&buf->tag),
    4465             :                              BufTagGetForkNum(&buf->tag)),
    4466             :                  buf->tag.blockNum, buf->flags,
    4467             :                  buf->refcount, GetPrivateRefCount(b));
    4468             :         }
    4469             :     }
    4470             : }
    4471             : #endif
    4472             : 
    4473             : /* ---------------------------------------------------------------------
    4474             :  *      FlushRelationBuffers
    4475             :  *
    4476             :  *      This function writes all dirty pages of a relation out to disk
    4477             :  *      (or more accurately, out to kernel disk buffers), ensuring that the
    4478             :  *      kernel has an up-to-date view of the relation.
    4479             :  *
    4480             :  *      Generally, the caller should be holding AccessExclusiveLock on the
    4481             :  *      target relation to ensure that no other backend is busy dirtying
    4482             :  *      more blocks of the relation; the effects can't be expected to last
    4483             :  *      after the lock is released.
    4484             :  *
    4485             :  *      XXX currently it sequentially searches the buffer pool, should be
    4486             :  *      changed to more clever ways of searching.  This routine is not
    4487             :  *      used in any performance-critical code paths, so it's not worth
    4488             :  *      adding additional overhead to normal paths to make it go faster.
    4489             :  * --------------------------------------------------------------------
    4490             :  */
    4491             : void
    4492         276 : FlushRelationBuffers(Relation rel)
    4493             : {
    4494             :     int         i;
    4495             :     BufferDesc *bufHdr;
    4496         276 :     SMgrRelation srel = RelationGetSmgr(rel);
    4497             : 
    4498         276 :     if (RelationUsesLocalBuffers(rel))
    4499             :     {
    4500        1818 :         for (i = 0; i < NLocBuffer; i++)
    4501             :         {
    4502             :             uint32      buf_state;
    4503             :             instr_time  io_start;
    4504             : 
    4505        1800 :             bufHdr = GetLocalBufferDescriptor(i);
    4506        1800 :             if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) &&
    4507         600 :                 ((buf_state = pg_atomic_read_u32(&bufHdr->state)) &
    4508             :                  (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
    4509             :             {
    4510             :                 ErrorContextCallback errcallback;
    4511             :                 Page        localpage;
    4512             : 
    4513         594 :                 localpage = (char *) LocalBufHdrGetBlock(bufHdr);
    4514             : 
    4515             :                 /* Setup error traceback support for ereport() */
    4516         594 :                 errcallback.callback = local_buffer_write_error_callback;
    4517         594 :                 errcallback.arg = bufHdr;
    4518         594 :                 errcallback.previous = error_context_stack;
    4519         594 :                 error_context_stack = &errcallback;
    4520             : 
    4521         594 :                 PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
    4522             : 
    4523         594 :                 io_start = pgstat_prepare_io_time(track_io_timing);
    4524             : 
    4525         594 :                 smgrwrite(srel,
    4526         594 :                           BufTagGetForkNum(&bufHdr->tag),
    4527             :                           bufHdr->tag.blockNum,
    4528             :                           localpage,
    4529             :                           false);
    4530             : 
    4531         594 :                 pgstat_count_io_op_time(IOOBJECT_TEMP_RELATION,
    4532             :                                         IOCONTEXT_NORMAL, IOOP_WRITE,
    4533             :                                         io_start, 1);
    4534             : 
    4535         594 :                 buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED);
    4536         594 :                 pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
    4537             : 
    4538         594 :                 pgBufferUsage.local_blks_written++;
    4539             : 
    4540             :                 /* Pop the error context stack */
    4541         594 :                 error_context_stack = errcallback.previous;
    4542             :             }
    4543             :         }
    4544             : 
    4545          18 :         return;
    4546             :     }
    4547             : 
    4548     3024386 :     for (i = 0; i < NBuffers; i++)
    4549             :     {
    4550             :         uint32      buf_state;
    4551             : 
    4552     3024128 :         bufHdr = GetBufferDescriptor(i);
    4553             : 
    4554             :         /*
    4555             :          * As in DropRelationBuffers, an unlocked precheck should be safe and
    4556             :          * saves some cycles.
    4557             :          */
    4558     3024128 :         if (!BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator))
    4559     3023682 :             continue;
    4560             : 
    4561             :         /* Make sure we can handle the pin */
    4562         446 :         ReservePrivateRefCountEntry();
    4563         446 :         ResourceOwnerEnlarge(CurrentResourceOwner);
    4564             : 
    4565         446 :         buf_state = LockBufHdr(bufHdr);
    4566         446 :         if (BufTagMatchesRelFileLocator(&bufHdr->tag, &rel->rd_locator) &&
    4567         446 :             (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
    4568             :         {
    4569         360 :             PinBuffer_Locked(bufHdr);
    4570         360 :             LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
    4571         360 :             FlushBuffer(bufHdr, srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
    4572         360 :             LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
    4573         360 :             UnpinBuffer(bufHdr);
    4574             :         }
    4575             :         else
    4576          86 :             UnlockBufHdr(bufHdr, buf_state);
    4577             :     }
    4578             : }
    4579             : 
    4580             : /* ---------------------------------------------------------------------
    4581             :  *      FlushRelationsAllBuffers
    4582             :  *
    4583             :  *      This function flushes out of the buffer pool all the pages of all
    4584             :  *      forks of the specified smgr relations.  It's equivalent to calling
    4585             :  *      FlushRelationBuffers once per relation.  The relations are assumed not
    4586             :  *      to use local buffers.
    4587             :  * --------------------------------------------------------------------
    4588             :  */
    4589             : void
    4590          18 : FlushRelationsAllBuffers(SMgrRelation *smgrs, int nrels)
    4591             : {
    4592             :     int         i;
    4593             :     SMgrSortArray *srels;
    4594             :     bool        use_bsearch;
    4595             : 
    4596          18 :     if (nrels == 0)
    4597           0 :         return;
    4598             : 
    4599             :     /* fill-in array for qsort */
    4600          18 :     srels = palloc(sizeof(SMgrSortArray) * nrels);
    4601             : 
    4602          36 :     for (i = 0; i < nrels; i++)
    4603             :     {
    4604             :         Assert(!RelFileLocatorBackendIsTemp(smgrs[i]->smgr_rlocator));
    4605             : 
    4606          18 :         srels[i].rlocator = smgrs[i]->smgr_rlocator.locator;
    4607          18 :         srels[i].srel = smgrs[i];
    4608             :     }
    4609             : 
    4610             :     /*
    4611             :      * Save the bsearch overhead for low number of relations to sync. See
    4612             :      * DropRelationsAllBuffers for details.
    4613             :      */
    4614          18 :     use_bsearch = nrels > RELS_BSEARCH_THRESHOLD;
    4615             : 
    4616             :     /* sort the list of SMgrRelations if necessary */
    4617          18 :     if (use_bsearch)
    4618           0 :         qsort(srels, nrels, sizeof(SMgrSortArray), rlocator_comparator);
    4619             : 
    4620      294930 :     for (i = 0; i < NBuffers; i++)
    4621             :     {
    4622      294912 :         SMgrSortArray *srelent = NULL;
    4623      294912 :         BufferDesc *bufHdr = GetBufferDescriptor(i);
    4624             :         uint32      buf_state;
    4625             : 
    4626             :         /*
    4627             :          * As in DropRelationBuffers, an unlocked precheck should be safe and
    4628             :          * saves some cycles.
    4629             :          */
    4630             : 
    4631      294912 :         if (!use_bsearch)
    4632             :         {
    4633             :             int         j;
    4634             : 
    4635      582188 :             for (j = 0; j < nrels; j++)
    4636             :             {
    4637      294912 :                 if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srels[j].rlocator))
    4638             :                 {
    4639        7636 :                     srelent = &srels[j];
    4640        7636 :                     break;
    4641             :                 }
    4642             :             }
    4643             :         }
    4644             :         else
    4645             :         {
    4646             :             RelFileLocator rlocator;
    4647             : 
    4648           0 :             rlocator = BufTagGetRelFileLocator(&bufHdr->tag);
    4649           0 :             srelent = bsearch((const void *) &(rlocator),
    4650             :                               srels, nrels, sizeof(SMgrSortArray),
    4651             :                               rlocator_comparator);
    4652             :         }
    4653             : 
    4654             :         /* buffer doesn't belong to any of the given relfilelocators; skip it */
    4655      294912 :         if (srelent == NULL)
    4656      287276 :             continue;
    4657             : 
    4658             :         /* Make sure we can handle the pin */
    4659        7636 :         ReservePrivateRefCountEntry();
    4660        7636 :         ResourceOwnerEnlarge(CurrentResourceOwner);
    4661             : 
    4662        7636 :         buf_state = LockBufHdr(bufHdr);
    4663        7636 :         if (BufTagMatchesRelFileLocator(&bufHdr->tag, &srelent->rlocator) &&
    4664        7636 :             (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
    4665             :         {
    4666        6734 :             PinBuffer_Locked(bufHdr);
    4667        6734 :             LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
    4668        6734 :             FlushBuffer(bufHdr, srelent->srel, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
    4669        6734 :             LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
    4670        6734 :             UnpinBuffer(bufHdr);
    4671             :         }
    4672             :         else
    4673         902 :             UnlockBufHdr(bufHdr, buf_state);
    4674             :     }
    4675             : 
    4676          18 :     pfree(srels);
    4677             : }
    4678             : 
    4679             : /* ---------------------------------------------------------------------
    4680             :  *      RelationCopyStorageUsingBuffer
    4681             :  *
    4682             :  *      Copy fork's data using bufmgr.  Same as RelationCopyStorage but instead
    4683             :  *      of using smgrread and smgrextend this will copy using bufmgr APIs.
    4684             :  *
    4685             :  *      Refer comments atop CreateAndCopyRelationData() for details about
    4686             :  *      'permanent' parameter.
    4687             :  * --------------------------------------------------------------------
    4688             :  */
    4689             : static void
    4690      128772 : RelationCopyStorageUsingBuffer(RelFileLocator srclocator,
    4691             :                                RelFileLocator dstlocator,
    4692             :                                ForkNumber forkNum, bool permanent)
    4693             : {
    4694             :     Buffer      srcBuf;
    4695             :     Buffer      dstBuf;
    4696             :     Page        srcPage;
    4697             :     Page        dstPage;
    4698             :     bool        use_wal;
    4699             :     BlockNumber nblocks;
    4700             :     BlockNumber blkno;
    4701             :     PGIOAlignedBlock buf;
    4702             :     BufferAccessStrategy bstrategy_src;
    4703             :     BufferAccessStrategy bstrategy_dst;
    4704             :     BlockRangeReadStreamPrivate p;
    4705             :     ReadStream *src_stream;
    4706             :     SMgrRelation src_smgr;
    4707             : 
    4708             :     /*
    4709             :      * In general, we want to write WAL whenever wal_level > 'minimal', but we
    4710             :      * can skip it when copying any fork of an unlogged relation other than
    4711             :      * the init fork.
    4712             :      */
    4713      128772 :     use_wal = XLogIsNeeded() && (permanent || forkNum == INIT_FORKNUM);
    4714             : 
    4715             :     /* Get number of blocks in the source relation. */
    4716      128772 :     nblocks = smgrnblocks(smgropen(srclocator, INVALID_PROC_NUMBER),
    4717             :                           forkNum);
    4718             : 
    4719             :     /* Nothing to copy; just return. */
    4720      128772 :     if (nblocks == 0)
    4721       22462 :         return;
    4722             : 
    4723             :     /*
    4724             :      * Bulk extend the destination relation of the same size as the source
    4725             :      * relation before starting to copy block by block.
    4726             :      */
    4727      106310 :     memset(buf.data, 0, BLCKSZ);
    4728      106310 :     smgrextend(smgropen(dstlocator, INVALID_PROC_NUMBER), forkNum, nblocks - 1,
    4729             :                buf.data, true);
    4730             : 
    4731             :     /* This is a bulk operation, so use buffer access strategies. */
    4732      106310 :     bstrategy_src = GetAccessStrategy(BAS_BULKREAD);
    4733      106310 :     bstrategy_dst = GetAccessStrategy(BAS_BULKWRITE);
    4734             : 
    4735             :     /* Initialize streaming read */
    4736      106310 :     p.current_blocknum = 0;
    4737      106310 :     p.last_exclusive = nblocks;
    4738      106310 :     src_smgr = smgropen(srclocator, INVALID_PROC_NUMBER);
    4739      106310 :     src_stream = read_stream_begin_smgr_relation(READ_STREAM_FULL,
    4740             :                                                  bstrategy_src,
    4741             :                                                  src_smgr,
    4742             :                                                  permanent ? RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED,
    4743             :                                                  forkNum,
    4744             :                                                  block_range_read_stream_cb,
    4745             :                                                  &p,
    4746             :                                                  0);
    4747             : 
    4748             :     /* Iterate over each block of the source relation file. */
    4749      507844 :     for (blkno = 0; blkno < nblocks; blkno++)
    4750             :     {
    4751      401534 :         CHECK_FOR_INTERRUPTS();
    4752             : 
    4753             :         /* Read block from source relation. */
    4754      401534 :         srcBuf = read_stream_next_buffer(src_stream, NULL);
    4755      401534 :         LockBuffer(srcBuf, BUFFER_LOCK_SHARE);
    4756      401534 :         srcPage = BufferGetPage(srcBuf);
    4757             : 
    4758      401534 :         dstBuf = ReadBufferWithoutRelcache(dstlocator, forkNum,
    4759             :                                            BufferGetBlockNumber(srcBuf),
    4760             :                                            RBM_ZERO_AND_LOCK, bstrategy_dst,
    4761             :                                            permanent);
    4762      401534 :         dstPage = BufferGetPage(dstBuf);
    4763             : 
    4764      401534 :         START_CRIT_SECTION();
    4765             : 
    4766             :         /* Copy page data from the source to the destination. */
    4767      401534 :         memcpy(dstPage, srcPage, BLCKSZ);
    4768      401534 :         MarkBufferDirty(dstBuf);
    4769             : 
    4770             :         /* WAL-log the copied page. */
    4771      401534 :         if (use_wal)
    4772      236154 :             log_newpage_buffer(dstBuf, true);
    4773             : 
    4774      401534 :         END_CRIT_SECTION();
    4775             : 
    4776      401534 :         UnlockReleaseBuffer(dstBuf);
    4777      401534 :         UnlockReleaseBuffer(srcBuf);
    4778             :     }
    4779             :     Assert(read_stream_next_buffer(src_stream, NULL) == InvalidBuffer);
    4780      106310 :     read_stream_end(src_stream);
    4781             : 
    4782      106310 :     FreeAccessStrategy(bstrategy_src);
    4783      106310 :     FreeAccessStrategy(bstrategy_dst);
    4784             : }
    4785             : 
    4786             : /* ---------------------------------------------------------------------
    4787             :  *      CreateAndCopyRelationData
    4788             :  *
    4789             :  *      Create destination relation storage and copy all forks from the
    4790             :  *      source relation to the destination.
    4791             :  *
    4792             :  *      Pass permanent as true for permanent relations and false for
    4793             :  *      unlogged relations.  Currently this API is not supported for
    4794             :  *      temporary relations.
    4795             :  * --------------------------------------------------------------------
    4796             :  */
    4797             : void
    4798       96800 : CreateAndCopyRelationData(RelFileLocator src_rlocator,
    4799             :                           RelFileLocator dst_rlocator, bool permanent)
    4800             : {
    4801             :     char        relpersistence;
    4802             :     SMgrRelation src_rel;
    4803             :     SMgrRelation dst_rel;
    4804             : 
    4805             :     /* Set the relpersistence. */
    4806       96800 :     relpersistence = permanent ?
    4807             :         RELPERSISTENCE_PERMANENT : RELPERSISTENCE_UNLOGGED;
    4808             : 
    4809       96800 :     src_rel = smgropen(src_rlocator, INVALID_PROC_NUMBER);
    4810       96800 :     dst_rel = smgropen(dst_rlocator, INVALID_PROC_NUMBER);
    4811             : 
    4812             :     /*
    4813             :      * Create and copy all forks of the relation.  During create database we
    4814             :      * have a separate cleanup mechanism which deletes complete database
    4815             :      * directory.  Therefore, each individual relation doesn't need to be
    4816             :      * registered for cleanup.
    4817             :      */
    4818       96800 :     RelationCreateStorage(dst_rlocator, relpersistence, false);
    4819             : 
    4820             :     /* copy main fork. */
    4821       96800 :     RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, MAIN_FORKNUM,
    4822             :                                    permanent);
    4823             : 
    4824             :     /* copy those extra forks that exist */
    4825      387200 :     for (ForkNumber forkNum = MAIN_FORKNUM + 1;
    4826      290400 :          forkNum <= MAX_FORKNUM; forkNum++)
    4827             :     {
    4828      290400 :         if (smgrexists(src_rel, forkNum))
    4829             :         {
    4830       31972 :             smgrcreate(dst_rel, forkNum, false);
    4831             : 
    4832             :             /*
    4833             :              * WAL log creation if the relation is persistent, or this is the
    4834             :              * init fork of an unlogged relation.
    4835             :              */
    4836       31972 :             if (permanent || forkNum == INIT_FORKNUM)
    4837       31972 :                 log_smgrcreate(&dst_rlocator, forkNum);
    4838             : 
    4839             :             /* Copy a fork's data, block by block. */
    4840       31972 :             RelationCopyStorageUsingBuffer(src_rlocator, dst_rlocator, forkNum,
    4841             :                                            permanent);
    4842             :         }
    4843             :     }
    4844       96800 : }
    4845             : 
    4846             : /* ---------------------------------------------------------------------
    4847             :  *      FlushDatabaseBuffers
    4848             :  *
    4849             :  *      This function writes all dirty pages of a database out to disk
    4850             :  *      (or more accurately, out to kernel disk buffers), ensuring that the
    4851             :  *      kernel has an up-to-date view of the database.
    4852             :  *
    4853             :  *      Generally, the caller should be holding an appropriate lock to ensure
    4854             :  *      no other backend is active in the target database; otherwise more
    4855             :  *      pages could get dirtied.
    4856             :  *
    4857             :  *      Note we don't worry about flushing any pages of temporary relations.
    4858             :  *      It's assumed these wouldn't be interesting.
    4859             :  * --------------------------------------------------------------------
    4860             :  */
    4861             : void
    4862           8 : FlushDatabaseBuffers(Oid dbid)
    4863             : {
    4864             :     int         i;
    4865             :     BufferDesc *bufHdr;
    4866             : 
    4867        1032 :     for (i = 0; i < NBuffers; i++)
    4868             :     {
    4869             :         uint32      buf_state;
    4870             : 
    4871        1024 :         bufHdr = GetBufferDescriptor(i);
    4872             : 
    4873             :         /*
    4874             :          * As in DropRelationBuffers, an unlocked precheck should be safe and
    4875             :          * saves some cycles.
    4876             :          */
    4877        1024 :         if (bufHdr->tag.dbOid != dbid)
    4878         640 :             continue;
    4879             : 
    4880             :         /* Make sure we can handle the pin */
    4881         384 :         ReservePrivateRefCountEntry();
    4882         384 :         ResourceOwnerEnlarge(CurrentResourceOwner);
    4883             : 
    4884         384 :         buf_state = LockBufHdr(bufHdr);
    4885         384 :         if (bufHdr->tag.dbOid == dbid &&
    4886         384 :             (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
    4887             :         {
    4888          72 :             PinBuffer_Locked(bufHdr);
    4889          72 :             LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
    4890          72 :             FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
    4891          72 :             LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
    4892          72 :             UnpinBuffer(bufHdr);
    4893             :         }
    4894             :         else
    4895         312 :             UnlockBufHdr(bufHdr, buf_state);
    4896             :     }
    4897           8 : }
    4898             : 
    4899             : /*
    4900             :  * Flush a previously, shared or exclusively, locked and pinned buffer to the
    4901             :  * OS.
    4902             :  */
    4903             : void
    4904          58 : FlushOneBuffer(Buffer buffer)
    4905             : {
    4906             :     BufferDesc *bufHdr;
    4907             : 
    4908             :     /* currently not needed, but no fundamental reason not to support */
    4909             :     Assert(!BufferIsLocal(buffer));
    4910             : 
    4911             :     Assert(BufferIsPinned(buffer));
    4912             : 
    4913          58 :     bufHdr = GetBufferDescriptor(buffer - 1);
    4914             : 
    4915             :     Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr)));
    4916             : 
    4917          58 :     FlushBuffer(bufHdr, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
    4918          58 : }
    4919             : 
    4920             : /*
    4921             :  * ReleaseBuffer -- release the pin on a buffer
    4922             :  */
    4923             : void
    4924   109564164 : ReleaseBuffer(Buffer buffer)
    4925             : {
    4926   109564164 :     if (!BufferIsValid(buffer))
    4927           0 :         elog(ERROR, "bad buffer ID: %d", buffer);
    4928             : 
    4929   109564164 :     if (BufferIsLocal(buffer))
    4930     2831678 :         UnpinLocalBuffer(buffer);
    4931             :     else
    4932   106732486 :         UnpinBuffer(GetBufferDescriptor(buffer - 1));
    4933   109564164 : }
    4934             : 
    4935             : /*
    4936             :  * UnlockReleaseBuffer -- release the content lock and pin on a buffer
    4937             :  *
    4938             :  * This is just a shorthand for a common combination.
    4939             :  */
    4940             : void
    4941    34732210 : UnlockReleaseBuffer(Buffer buffer)
    4942             : {
    4943    34732210 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    4944    34732210 :     ReleaseBuffer(buffer);
    4945    34732210 : }
    4946             : 
    4947             : /*
    4948             :  * IncrBufferRefCount
    4949             :  *      Increment the pin count on a buffer that we have *already* pinned
    4950             :  *      at least once.
    4951             :  *
    4952             :  *      This function cannot be used on a buffer we do not have pinned,
    4953             :  *      because it doesn't change the shared buffer state.
    4954             :  */
    4955             : void
    4956    19682490 : IncrBufferRefCount(Buffer buffer)
    4957             : {
    4958             :     Assert(BufferIsPinned(buffer));
    4959    19682490 :     ResourceOwnerEnlarge(CurrentResourceOwner);
    4960    19682490 :     if (BufferIsLocal(buffer))
    4961      697690 :         LocalRefCount[-buffer - 1]++;
    4962             :     else
    4963             :     {
    4964             :         PrivateRefCountEntry *ref;
    4965             : 
    4966    18984800 :         ref = GetPrivateRefCountEntry(buffer, true);
    4967             :         Assert(ref != NULL);
    4968    18984800 :         ref->refcount++;
    4969             :     }
    4970    19682490 :     ResourceOwnerRememberBuffer(CurrentResourceOwner, buffer);
    4971    19682490 : }
    4972             : 
    4973             : /*
    4974             :  * MarkBufferDirtyHint
    4975             :  *
    4976             :  *  Mark a buffer dirty for non-critical changes.
    4977             :  *
    4978             :  * This is essentially the same as MarkBufferDirty, except:
    4979             :  *
    4980             :  * 1. The caller does not write WAL; so if checksums are enabled, we may need
    4981             :  *    to write an XLOG_FPI_FOR_HINT WAL record to protect against torn pages.
    4982             :  * 2. The caller might have only share-lock instead of exclusive-lock on the
    4983             :  *    buffer's content lock.
    4984             :  * 3. This function does not guarantee that the buffer is always marked dirty
    4985             :  *    (due to a race condition), so it cannot be used for important changes.
    4986             :  */
    4987             : void
    4988    19336042 : MarkBufferDirtyHint(Buffer buffer, bool buffer_std)
    4989             : {
    4990             :     BufferDesc *bufHdr;
    4991    19336042 :     Page        page = BufferGetPage(buffer);
    4992             : 
    4993    19336042 :     if (!BufferIsValid(buffer))
    4994           0 :         elog(ERROR, "bad buffer ID: %d", buffer);
    4995             : 
    4996    19336042 :     if (BufferIsLocal(buffer))
    4997             :     {
    4998     1163426 :         MarkLocalBufferDirty(buffer);
    4999     1163426 :         return;
    5000             :     }
    5001             : 
    5002    18172616 :     bufHdr = GetBufferDescriptor(buffer - 1);
    5003             : 
    5004             :     Assert(GetPrivateRefCount(buffer) > 0);
    5005             :     /* here, either share or exclusive lock is OK */
    5006             :     Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr)));
    5007             : 
    5008             :     /*
    5009             :      * This routine might get called many times on the same page, if we are
    5010             :      * making the first scan after commit of an xact that added/deleted many
    5011             :      * tuples. So, be as quick as we can if the buffer is already dirty.  We
    5012             :      * do this by not acquiring spinlock if it looks like the status bits are
    5013             :      * already set.  Since we make this test unlocked, there's a chance we
    5014             :      * might fail to notice that the flags have just been cleared, and failed
    5015             :      * to reset them, due to memory-ordering issues.  But since this function
    5016             :      * is only intended to be used in cases where failing to write out the
    5017             :      * data would be harmless anyway, it doesn't really matter.
    5018             :      */
    5019    18172616 :     if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) !=
    5020             :         (BM_DIRTY | BM_JUST_DIRTIED))
    5021             :     {
    5022     2186210 :         XLogRecPtr  lsn = InvalidXLogRecPtr;
    5023     2186210 :         bool        dirtied = false;
    5024     2186210 :         bool        delayChkptFlags = false;
    5025             :         uint32      buf_state;
    5026             : 
    5027             :         /*
    5028             :          * If we need to protect hint bit updates from torn writes, WAL-log a
    5029             :          * full page image of the page. This full page image is only necessary
    5030             :          * if the hint bit update is the first change to the page since the
    5031             :          * last checkpoint.
    5032             :          *
    5033             :          * We don't check full_page_writes here because that logic is included
    5034             :          * when we call XLogInsert() since the value changes dynamically.
    5035             :          */
    5036     4370638 :         if (XLogHintBitIsNeeded() &&
    5037     2184428 :             (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT))
    5038             :         {
    5039             :             /*
    5040             :              * If we must not write WAL, due to a relfilelocator-specific
    5041             :              * condition or being in recovery, don't dirty the page.  We can
    5042             :              * set the hint, just not dirty the page as a result so the hint
    5043             :              * is lost when we evict the page or shutdown.
    5044             :              *
    5045             :              * See src/backend/storage/page/README for longer discussion.
    5046             :              */
    5047     2291334 :             if (RecoveryInProgress() ||
    5048      106960 :                 RelFileLocatorSkippingWAL(BufTagGetRelFileLocator(&bufHdr->tag)))
    5049     2080688 :                 return;
    5050             : 
    5051             :             /*
    5052             :              * If the block is already dirty because we either made a change
    5053             :              * or set a hint already, then we don't need to write a full page
    5054             :              * image.  Note that aggressive cleaning of blocks dirtied by hint
    5055             :              * bit setting would increase the call rate. Bulk setting of hint
    5056             :              * bits would reduce the call rate...
    5057             :              *
    5058             :              * We must issue the WAL record before we mark the buffer dirty.
    5059             :              * Otherwise we might write the page before we write the WAL. That
    5060             :              * causes a race condition, since a checkpoint might occur between
    5061             :              * writing the WAL record and marking the buffer dirty. We solve
    5062             :              * that with a kluge, but one that is already in use during
    5063             :              * transaction commit to prevent race conditions. Basically, we
    5064             :              * simply prevent the checkpoint WAL record from being written
    5065             :              * until we have marked the buffer dirty. We don't start the
    5066             :              * checkpoint flush until we have marked dirty, so our checkpoint
    5067             :              * must flush the change to disk successfully or the checkpoint
    5068             :              * never gets written, so crash recovery will fix.
    5069             :              *
    5070             :              * It's possible we may enter here without an xid, so it is
    5071             :              * essential that CreateCheckPoint waits for virtual transactions
    5072             :              * rather than full transactionids.
    5073             :              */
    5074             :             Assert((MyProc->delayChkptFlags & DELAY_CHKPT_START) == 0);
    5075      103686 :             MyProc->delayChkptFlags |= DELAY_CHKPT_START;
    5076      103686 :             delayChkptFlags = true;
    5077      103686 :             lsn = XLogSaveBufferForHint(buffer, buffer_std);
    5078             :         }
    5079             : 
    5080      105522 :         buf_state = LockBufHdr(bufHdr);
    5081             : 
    5082             :         Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    5083             : 
    5084      105522 :         if (!(buf_state & BM_DIRTY))
    5085             :         {
    5086      105458 :             dirtied = true;     /* Means "will be dirtied by this action" */
    5087             : 
    5088             :             /*
    5089             :              * Set the page LSN if we wrote a backup block. We aren't supposed
    5090             :              * to set this when only holding a share lock but as long as we
    5091             :              * serialise it somehow we're OK. We choose to set LSN while
    5092             :              * holding the buffer header lock, which causes any reader of an
    5093             :              * LSN who holds only a share lock to also obtain a buffer header
    5094             :              * lock before using PageGetLSN(), which is enforced in
    5095             :              * BufferGetLSNAtomic().
    5096             :              *
    5097             :              * If checksums are enabled, you might think we should reset the
    5098             :              * checksum here. That will happen when the page is written
    5099             :              * sometime later in this checkpoint cycle.
    5100             :              */
    5101      105458 :             if (!XLogRecPtrIsInvalid(lsn))
    5102       54120 :                 PageSetLSN(page, lsn);
    5103             :         }
    5104             : 
    5105      105522 :         buf_state |= BM_DIRTY | BM_JUST_DIRTIED;
    5106      105522 :         UnlockBufHdr(bufHdr, buf_state);
    5107             : 
    5108      105522 :         if (delayChkptFlags)
    5109      103686 :             MyProc->delayChkptFlags &= ~DELAY_CHKPT_START;
    5110             : 
    5111      105522 :         if (dirtied)
    5112             :         {
    5113      105458 :             pgBufferUsage.shared_blks_dirtied++;
    5114      105458 :             if (VacuumCostActive)
    5115         714 :                 VacuumCostBalance += VacuumCostPageDirty;
    5116             :         }
    5117             :     }
    5118             : }
    5119             : 
    5120             : /*
    5121             :  * Release buffer content locks for shared buffers.
    5122             :  *
    5123             :  * Used to clean up after errors.
    5124             :  *
    5125             :  * Currently, we can expect that lwlock.c's LWLockReleaseAll() took care
    5126             :  * of releasing buffer content locks per se; the only thing we need to deal
    5127             :  * with here is clearing any PIN_COUNT request that was in progress.
    5128             :  */
    5129             : void
    5130       91438 : UnlockBuffers(void)
    5131             : {
    5132       91438 :     BufferDesc *buf = PinCountWaitBuf;
    5133             : 
    5134       91438 :     if (buf)
    5135             :     {
    5136             :         uint32      buf_state;
    5137             : 
    5138           0 :         buf_state = LockBufHdr(buf);
    5139             : 
    5140             :         /*
    5141             :          * Don't complain if flag bit not set; it could have been reset but we
    5142             :          * got a cancel/die interrupt before getting the signal.
    5143             :          */
    5144           0 :         if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
    5145           0 :             buf->wait_backend_pgprocno == MyProcNumber)
    5146           0 :             buf_state &= ~BM_PIN_COUNT_WAITER;
    5147             : 
    5148           0 :         UnlockBufHdr(buf, buf_state);
    5149             : 
    5150           0 :         PinCountWaitBuf = NULL;
    5151             :     }
    5152       91438 : }
    5153             : 
    5154             : /*
    5155             :  * Acquire or release the content_lock for the buffer.
    5156             :  */
    5157             : void
    5158   312270944 : LockBuffer(Buffer buffer, int mode)
    5159             : {
    5160             :     BufferDesc *buf;
    5161             : 
    5162             :     Assert(BufferIsPinned(buffer));
    5163   312270944 :     if (BufferIsLocal(buffer))
    5164    18921570 :         return;                 /* local buffers need no lock */
    5165             : 
    5166   293349374 :     buf = GetBufferDescriptor(buffer - 1);
    5167             : 
    5168   293349374 :     if (mode == BUFFER_LOCK_UNLOCK)
    5169   148218810 :         LWLockRelease(BufferDescriptorGetContentLock(buf));
    5170   145130564 :     else if (mode == BUFFER_LOCK_SHARE)
    5171   101666600 :         LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED);
    5172    43463964 :     else if (mode == BUFFER_LOCK_EXCLUSIVE)
    5173    43463964 :         LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE);
    5174             :     else
    5175           0 :         elog(ERROR, "unrecognized buffer lock mode: %d", mode);
    5176             : }
    5177             : 
    5178             : /*
    5179             :  * Acquire the content_lock for the buffer, but only if we don't have to wait.
    5180             :  *
    5181             :  * This assumes the caller wants BUFFER_LOCK_EXCLUSIVE mode.
    5182             :  */
    5183             : bool
    5184     2386558 : ConditionalLockBuffer(Buffer buffer)
    5185             : {
    5186             :     BufferDesc *buf;
    5187             : 
    5188             :     Assert(BufferIsPinned(buffer));
    5189     2386558 :     if (BufferIsLocal(buffer))
    5190      129518 :         return true;            /* act as though we got it */
    5191             : 
    5192     2257040 :     buf = GetBufferDescriptor(buffer - 1);
    5193             : 
    5194     2257040 :     return LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf),
    5195             :                                     LW_EXCLUSIVE);
    5196             : }
    5197             : 
    5198             : /*
    5199             :  * Verify that this backend is pinning the buffer exactly once.
    5200             :  *
    5201             :  * NOTE: Like in BufferIsPinned(), what we check here is that *this* backend
    5202             :  * holds a pin on the buffer.  We do not care whether some other backend does.
    5203             :  */
    5204             : void
    5205     3446654 : CheckBufferIsPinnedOnce(Buffer buffer)
    5206             : {
    5207     3446654 :     if (BufferIsLocal(buffer))
    5208             :     {
    5209          32 :         if (LocalRefCount[-buffer - 1] != 1)
    5210           0 :             elog(ERROR, "incorrect local pin count: %d",
    5211             :                  LocalRefCount[-buffer - 1]);
    5212             :     }
    5213             :     else
    5214             :     {
    5215     3446622 :         if (GetPrivateRefCount(buffer) != 1)
    5216           0 :             elog(ERROR, "incorrect local pin count: %d",
    5217             :                  GetPrivateRefCount(buffer));
    5218             :     }
    5219     3446654 : }
    5220             : 
    5221             : /*
    5222             :  * LockBufferForCleanup - lock a buffer in preparation for deleting items
    5223             :  *
    5224             :  * Items may be deleted from a disk page only when the caller (a) holds an
    5225             :  * exclusive lock on the buffer and (b) has observed that no other backend
    5226             :  * holds a pin on the buffer.  If there is a pin, then the other backend
    5227             :  * might have a pointer into the buffer (for example, a heapscan reference
    5228             :  * to an item --- see README for more details).  It's OK if a pin is added
    5229             :  * after the cleanup starts, however; the newly-arrived backend will be
    5230             :  * unable to look at the page until we release the exclusive lock.
    5231             :  *
    5232             :  * To implement this protocol, a would-be deleter must pin the buffer and
    5233             :  * then call LockBufferForCleanup().  LockBufferForCleanup() is similar to
    5234             :  * LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE), except that it loops until
    5235             :  * it has successfully observed pin count = 1.
    5236             :  */
    5237             : void
    5238       34904 : LockBufferForCleanup(Buffer buffer)
    5239             : {
    5240             :     BufferDesc *bufHdr;
    5241       34904 :     TimestampTz waitStart = 0;
    5242       34904 :     bool        waiting = false;
    5243       34904 :     bool        logged_recovery_conflict = false;
    5244             : 
    5245             :     Assert(BufferIsPinned(buffer));
    5246             :     Assert(PinCountWaitBuf == NULL);
    5247             : 
    5248       34904 :     CheckBufferIsPinnedOnce(buffer);
    5249             : 
    5250             :     /* Nobody else to wait for */
    5251       34904 :     if (BufferIsLocal(buffer))
    5252          32 :         return;
    5253             : 
    5254       34872 :     bufHdr = GetBufferDescriptor(buffer - 1);
    5255             : 
    5256             :     for (;;)
    5257          20 :     {
    5258             :         uint32      buf_state;
    5259             : 
    5260             :         /* Try to acquire lock */
    5261       34892 :         LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    5262       34892 :         buf_state = LockBufHdr(bufHdr);
    5263             : 
    5264             :         Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    5265       34892 :         if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
    5266             :         {
    5267             :             /* Successfully acquired exclusive lock with pincount 1 */
    5268       34872 :             UnlockBufHdr(bufHdr, buf_state);
    5269             : 
    5270             :             /*
    5271             :              * Emit the log message if recovery conflict on buffer pin was
    5272             :              * resolved but the startup process waited longer than
    5273             :              * deadlock_timeout for it.
    5274             :              */
    5275       34872 :             if (logged_recovery_conflict)
    5276           4 :                 LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN,
    5277             :                                     waitStart, GetCurrentTimestamp(),
    5278             :                                     NULL, false);
    5279             : 
    5280       34872 :             if (waiting)
    5281             :             {
    5282             :                 /* reset ps display to remove the suffix if we added one */
    5283           4 :                 set_ps_display_remove_suffix();
    5284           4 :                 waiting = false;
    5285             :             }
    5286       34872 :             return;
    5287             :         }
    5288             :         /* Failed, so mark myself as waiting for pincount 1 */
    5289          20 :         if (buf_state & BM_PIN_COUNT_WAITER)
    5290             :         {
    5291           0 :             UnlockBufHdr(bufHdr, buf_state);
    5292           0 :             LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    5293           0 :             elog(ERROR, "multiple backends attempting to wait for pincount 1");
    5294             :         }
    5295          20 :         bufHdr->wait_backend_pgprocno = MyProcNumber;
    5296          20 :         PinCountWaitBuf = bufHdr;
    5297          20 :         buf_state |= BM_PIN_COUNT_WAITER;
    5298          20 :         UnlockBufHdr(bufHdr, buf_state);
    5299          20 :         LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    5300             : 
    5301             :         /* Wait to be signaled by UnpinBuffer() */
    5302          20 :         if (InHotStandby)
    5303             :         {
    5304          20 :             if (!waiting)
    5305             :             {
    5306             :                 /* adjust the process title to indicate that it's waiting */
    5307           4 :                 set_ps_display_suffix("waiting");
    5308           4 :                 waiting = true;
    5309             :             }
    5310             : 
    5311             :             /*
    5312             :              * Emit the log message if the startup process is waiting longer
    5313             :              * than deadlock_timeout for recovery conflict on buffer pin.
    5314             :              *
    5315             :              * Skip this if first time through because the startup process has
    5316             :              * not started waiting yet in this case. So, the wait start
    5317             :              * timestamp is set after this logic.
    5318             :              */
    5319          20 :             if (waitStart != 0 && !logged_recovery_conflict)
    5320             :             {
    5321           6 :                 TimestampTz now = GetCurrentTimestamp();
    5322             : 
    5323           6 :                 if (TimestampDifferenceExceeds(waitStart, now,
    5324             :                                                DeadlockTimeout))
    5325             :                 {
    5326           4 :                     LogRecoveryConflict(PROCSIG_RECOVERY_CONFLICT_BUFFERPIN,
    5327             :                                         waitStart, now, NULL, true);
    5328           4 :                     logged_recovery_conflict = true;
    5329             :                 }
    5330             :             }
    5331             : 
    5332             :             /*
    5333             :              * Set the wait start timestamp if logging is enabled and first
    5334             :              * time through.
    5335             :              */
    5336          20 :             if (log_recovery_conflict_waits && waitStart == 0)
    5337           4 :                 waitStart = GetCurrentTimestamp();
    5338             : 
    5339             :             /* Publish the bufid that Startup process waits on */
    5340          20 :             SetStartupBufferPinWaitBufId(buffer - 1);
    5341             :             /* Set alarm and then wait to be signaled by UnpinBuffer() */
    5342          20 :             ResolveRecoveryConflictWithBufferPin();
    5343             :             /* Reset the published bufid */
    5344          20 :             SetStartupBufferPinWaitBufId(-1);
    5345             :         }
    5346             :         else
    5347           0 :             ProcWaitForSignal(WAIT_EVENT_BUFFER_PIN);
    5348             : 
    5349             :         /*
    5350             :          * Remove flag marking us as waiter. Normally this will not be set
    5351             :          * anymore, but ProcWaitForSignal() can return for other signals as
    5352             :          * well.  We take care to only reset the flag if we're the waiter, as
    5353             :          * theoretically another backend could have started waiting. That's
    5354             :          * impossible with the current usages due to table level locking, but
    5355             :          * better be safe.
    5356             :          */
    5357          20 :         buf_state = LockBufHdr(bufHdr);
    5358          20 :         if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
    5359          16 :             bufHdr->wait_backend_pgprocno == MyProcNumber)
    5360          16 :             buf_state &= ~BM_PIN_COUNT_WAITER;
    5361          20 :         UnlockBufHdr(bufHdr, buf_state);
    5362             : 
    5363          20 :         PinCountWaitBuf = NULL;
    5364             :         /* Loop back and try again */
    5365             :     }
    5366             : }
    5367             : 
    5368             : /*
    5369             :  * Check called from ProcessRecoveryConflictInterrupts() when Startup process
    5370             :  * requests cancellation of all pin holders that are blocking it.
    5371             :  */
    5372             : bool
    5373           8 : HoldingBufferPinThatDelaysRecovery(void)
    5374             : {
    5375           8 :     int         bufid = GetStartupBufferPinWaitBufId();
    5376             : 
    5377             :     /*
    5378             :      * If we get woken slowly then it's possible that the Startup process was
    5379             :      * already woken by other backends before we got here. Also possible that
    5380             :      * we get here by multiple interrupts or interrupts at inappropriate
    5381             :      * times, so make sure we do nothing if the bufid is not set.
    5382             :      */
    5383           8 :     if (bufid < 0)
    5384           4 :         return false;
    5385             : 
    5386           4 :     if (GetPrivateRefCount(bufid + 1) > 0)
    5387           4 :         return true;
    5388             : 
    5389           0 :     return false;
    5390             : }
    5391             : 
    5392             : /*
    5393             :  * ConditionalLockBufferForCleanup - as above, but don't wait to get the lock
    5394             :  *
    5395             :  * We won't loop, but just check once to see if the pin count is OK.  If
    5396             :  * not, return false with no lock held.
    5397             :  */
    5398             : bool
    5399      496172 : ConditionalLockBufferForCleanup(Buffer buffer)
    5400             : {
    5401             :     BufferDesc *bufHdr;
    5402             :     uint32      buf_state,
    5403             :                 refcount;
    5404             : 
    5405             :     Assert(BufferIsValid(buffer));
    5406             : 
    5407      496172 :     if (BufferIsLocal(buffer))
    5408             :     {
    5409        1582 :         refcount = LocalRefCount[-buffer - 1];
    5410             :         /* There should be exactly one pin */
    5411             :         Assert(refcount > 0);
    5412        1582 :         if (refcount != 1)
    5413          42 :             return false;
    5414             :         /* Nobody else to wait for */
    5415        1540 :         return true;
    5416             :     }
    5417             : 
    5418             :     /* There should be exactly one local pin */
    5419      494590 :     refcount = GetPrivateRefCount(buffer);
    5420             :     Assert(refcount);
    5421      494590 :     if (refcount != 1)
    5422         456 :         return false;
    5423             : 
    5424             :     /* Try to acquire lock */
    5425      494134 :     if (!ConditionalLockBuffer(buffer))
    5426          42 :         return false;
    5427             : 
    5428      494092 :     bufHdr = GetBufferDescriptor(buffer - 1);
    5429      494092 :     buf_state = LockBufHdr(bufHdr);
    5430      494092 :     refcount = BUF_STATE_GET_REFCOUNT(buf_state);
    5431             : 
    5432             :     Assert(refcount > 0);
    5433      494092 :     if (refcount == 1)
    5434             :     {
    5435             :         /* Successfully acquired exclusive lock with pincount 1 */
    5436      493886 :         UnlockBufHdr(bufHdr, buf_state);
    5437      493886 :         return true;
    5438             :     }
    5439             : 
    5440             :     /* Failed, so release the lock */
    5441         206 :     UnlockBufHdr(bufHdr, buf_state);
    5442         206 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    5443         206 :     return false;
    5444             : }
    5445             : 
    5446             : /*
    5447             :  * IsBufferCleanupOK - as above, but we already have the lock
    5448             :  *
    5449             :  * Check whether it's OK to perform cleanup on a buffer we've already
    5450             :  * locked.  If we observe that the pin count is 1, our exclusive lock
    5451             :  * happens to be a cleanup lock, and we can proceed with anything that
    5452             :  * would have been allowable had we sought a cleanup lock originally.
    5453             :  */
    5454             : bool
    5455        4014 : IsBufferCleanupOK(Buffer buffer)
    5456             : {
    5457             :     BufferDesc *bufHdr;
    5458             :     uint32      buf_state;
    5459             : 
    5460             :     Assert(BufferIsValid(buffer));
    5461             : 
    5462        4014 :     if (BufferIsLocal(buffer))
    5463             :     {
    5464             :         /* There should be exactly one pin */
    5465           0 :         if (LocalRefCount[-buffer - 1] != 1)
    5466           0 :             return false;
    5467             :         /* Nobody else to wait for */
    5468           0 :         return true;
    5469             :     }
    5470             : 
    5471             :     /* There should be exactly one local pin */
    5472        4014 :     if (GetPrivateRefCount(buffer) != 1)
    5473           0 :         return false;
    5474             : 
    5475        4014 :     bufHdr = GetBufferDescriptor(buffer - 1);
    5476             : 
    5477             :     /* caller must hold exclusive lock on buffer */
    5478             :     Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
    5479             :                                 LW_EXCLUSIVE));
    5480             : 
    5481        4014 :     buf_state = LockBufHdr(bufHdr);
    5482             : 
    5483             :     Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
    5484        4014 :     if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
    5485             :     {
    5486             :         /* pincount is OK. */
    5487        4014 :         UnlockBufHdr(bufHdr, buf_state);
    5488        4014 :         return true;
    5489             :     }
    5490             : 
    5491           0 :     UnlockBufHdr(bufHdr, buf_state);
    5492           0 :     return false;
    5493             : }
    5494             : 
    5495             : 
    5496             : /*
    5497             :  *  Functions for buffer I/O handling
    5498             :  *
    5499             :  *  Note: We assume that nested buffer I/O never occurs.
    5500             :  *  i.e at most one BM_IO_IN_PROGRESS bit is set per proc.
    5501             :  *
    5502             :  *  Also note that these are used only for shared buffers, not local ones.
    5503             :  */
    5504             : 
    5505             : /*
    5506             :  * WaitIO -- Block until the IO_IN_PROGRESS flag on 'buf' is cleared.
    5507             :  */
    5508             : static void
    5509        3134 : WaitIO(BufferDesc *buf)
    5510             : {
    5511        3134 :     ConditionVariable *cv = BufferDescriptorGetIOCV(buf);
    5512             : 
    5513        3134 :     ConditionVariablePrepareToSleep(cv);
    5514             :     for (;;)
    5515        3112 :     {
    5516             :         uint32      buf_state;
    5517             : 
    5518             :         /*
    5519             :          * It may not be necessary to acquire the spinlock to check the flag
    5520             :          * here, but since this test is essential for correctness, we'd better
    5521             :          * play it safe.
    5522             :          */
    5523        6246 :         buf_state = LockBufHdr(buf);
    5524        6246 :         UnlockBufHdr(buf, buf_state);
    5525             : 
    5526        6246 :         if (!(buf_state & BM_IO_IN_PROGRESS))
    5527        3134 :             break;
    5528        3112 :         ConditionVariableSleep(cv, WAIT_EVENT_BUFFER_IO);
    5529             :     }
    5530        3134 :     ConditionVariableCancelSleep();
    5531        3134 : }
    5532             : 
    5533             : /*
    5534             :  * StartBufferIO: begin I/O on this buffer
    5535             :  *  (Assumptions)
    5536             :  *  My process is executing no IO
    5537             :  *  The buffer is Pinned
    5538             :  *
    5539             :  * In some scenarios there are race conditions in which multiple backends
    5540             :  * could attempt the same I/O operation concurrently.  If someone else
    5541             :  * has already started I/O on this buffer then we will block on the
    5542             :  * I/O condition variable until he's done.
    5543             :  *
    5544             :  * Input operations are only attempted on buffers that are not BM_VALID,
    5545             :  * and output operations only on buffers that are BM_VALID and BM_DIRTY,
    5546             :  * so we can always tell if the work is already done.
    5547             :  *
    5548             :  * Returns true if we successfully marked the buffer as I/O busy,
    5549             :  * false if someone else already did the work.
    5550             :  *
    5551             :  * If nowait is true, then we don't wait for an I/O to be finished by another
    5552             :  * backend.  In that case, false indicates either that the I/O was already
    5553             :  * finished, or is still in progress.  This is useful for callers that want to
    5554             :  * find out if they can perform the I/O as part of a larger operation, without
    5555             :  * waiting for the answer or distinguishing the reasons why not.
    5556             :  */
    5557             : static bool
    5558     4355360 : StartBufferIO(BufferDesc *buf, bool forInput, bool nowait)
    5559             : {
    5560             :     uint32      buf_state;
    5561             : 
    5562     4355360 :     ResourceOwnerEnlarge(CurrentResourceOwner);
    5563             : 
    5564             :     for (;;)
    5565             :     {
    5566     4358488 :         buf_state = LockBufHdr(buf);
    5567             : 
    5568     4358488 :         if (!(buf_state & BM_IO_IN_PROGRESS))
    5569     4355360 :             break;
    5570        3128 :         UnlockBufHdr(buf, buf_state);
    5571        3128 :         if (nowait)
    5572           0 :             return false;
    5573        3128 :         WaitIO(buf);
    5574             :     }
    5575             : 
    5576             :     /* Once we get here, there is definitely no I/O active on this buffer */
    5577             : 
    5578     4355360 :     if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY))
    5579             :     {
    5580             :         /* someone else already did the I/O */
    5581        3782 :         UnlockBufHdr(buf, buf_state);
    5582        3782 :         return false;
    5583             :     }
    5584             : 
    5585     4351578 :     buf_state |= BM_IO_IN_PROGRESS;
    5586     4351578 :     UnlockBufHdr(buf, buf_state);
    5587             : 
    5588     4351578 :     ResourceOwnerRememberBufferIO(CurrentResourceOwner,
    5589             :                                   BufferDescriptorGetBuffer(buf));
    5590             : 
    5591     4351578 :     return true;
    5592             : }
    5593             : 
    5594             : /*
    5595             :  * TerminateBufferIO: release a buffer we were doing I/O on
    5596             :  *  (Assumptions)
    5597             :  *  My process is executing IO for the buffer
    5598             :  *  BM_IO_IN_PROGRESS bit is set for the buffer
    5599             :  *  The buffer is Pinned
    5600             :  *
    5601             :  * If clear_dirty is true and BM_JUST_DIRTIED is not set, we clear the
    5602             :  * buffer's BM_DIRTY flag.  This is appropriate when terminating a
    5603             :  * successful write.  The check on BM_JUST_DIRTIED is necessary to avoid
    5604             :  * marking the buffer clean if it was re-dirtied while we were writing.
    5605             :  *
    5606             :  * set_flag_bits gets ORed into the buffer's flags.  It must include
    5607             :  * BM_IO_ERROR in a failure case.  For successful completion it could
    5608             :  * be 0, or BM_VALID if we just finished reading in the page.
    5609             :  *
    5610             :  * If forget_owner is true, we release the buffer I/O from the current
    5611             :  * resource owner. (forget_owner=false is used when the resource owner itself
    5612             :  * is being released)
    5613             :  */
    5614             : static void
    5615     4351578 : TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits,
    5616             :                   bool forget_owner)
    5617             : {
    5618             :     uint32      buf_state;
    5619             : 
    5620     4351578 :     buf_state = LockBufHdr(buf);
    5621             : 
    5622             :     Assert(buf_state & BM_IO_IN_PROGRESS);
    5623             : 
    5624     4351578 :     buf_state &= ~(BM_IO_IN_PROGRESS | BM_IO_ERROR);
    5625     4351578 :     if (clear_dirty && !(buf_state & BM_JUST_DIRTIED))
    5626      952520 :         buf_state &= ~(BM_DIRTY | BM_CHECKPOINT_NEEDED);
    5627             : 
    5628     4351578 :     buf_state |= set_flag_bits;
    5629     4351578 :     UnlockBufHdr(buf, buf_state);
    5630             : 
    5631     4351578 :     if (forget_owner)
    5632     4351548 :         ResourceOwnerForgetBufferIO(CurrentResourceOwner,
    5633             :                                     BufferDescriptorGetBuffer(buf));
    5634             : 
    5635     4351578 :     ConditionVariableBroadcast(BufferDescriptorGetIOCV(buf));
    5636     4351578 : }
    5637             : 
    5638             : /*
    5639             :  * AbortBufferIO: Clean up active buffer I/O after an error.
    5640             :  *
    5641             :  *  All LWLocks we might have held have been released,
    5642             :  *  but we haven't yet released buffer pins, so the buffer is still pinned.
    5643             :  *
    5644             :  *  If I/O was in progress, we always set BM_IO_ERROR, even though it's
    5645             :  *  possible the error condition wasn't related to the I/O.
    5646             :  *
    5647             :  *  Note: this does not remove the buffer I/O from the resource owner.
    5648             :  *  That's correct when we're releasing the whole resource owner, but
    5649             :  *  beware if you use this in other contexts.
    5650             :  */
    5651             : static void
    5652          30 : AbortBufferIO(Buffer buffer)
    5653             : {
    5654          30 :     BufferDesc *buf_hdr = GetBufferDescriptor(buffer - 1);
    5655             :     uint32      buf_state;
    5656             : 
    5657          30 :     buf_state = LockBufHdr(buf_hdr);
    5658             :     Assert(buf_state & (BM_IO_IN_PROGRESS | BM_TAG_VALID));
    5659             : 
    5660          30 :     if (!(buf_state & BM_VALID))
    5661             :     {
    5662             :         Assert(!(buf_state & BM_DIRTY));
    5663          30 :         UnlockBufHdr(buf_hdr, buf_state);
    5664             :     }
    5665             :     else
    5666             :     {
    5667             :         Assert(buf_state & BM_DIRTY);
    5668           0 :         UnlockBufHdr(buf_hdr, buf_state);
    5669             : 
    5670             :         /* Issue notice if this is not the first failure... */
    5671           0 :         if (buf_state & BM_IO_ERROR)
    5672             :         {
    5673             :             /* Buffer is pinned, so we can read tag without spinlock */
    5674             :             char       *path;
    5675             : 
    5676           0 :             path = relpathperm(BufTagGetRelFileLocator(&buf_hdr->tag),
    5677             :                                BufTagGetForkNum(&buf_hdr->tag));
    5678           0 :             ereport(WARNING,
    5679             :                     (errcode(ERRCODE_IO_ERROR),
    5680             :                      errmsg("could not write block %u of %s",
    5681             :                             buf_hdr->tag.blockNum, path),
    5682             :                      errdetail("Multiple failures --- write error might be permanent.")));
    5683           0 :             pfree(path);
    5684             :         }
    5685             :     }
    5686             : 
    5687          30 :     TerminateBufferIO(buf_hdr, false, BM_IO_ERROR, false);
    5688          30 : }
    5689             : 
    5690             : /*
    5691             :  * Error context callback for errors occurring during shared buffer writes.
    5692             :  */
    5693             : static void
    5694          76 : shared_buffer_write_error_callback(void *arg)
    5695             : {
    5696          76 :     BufferDesc *bufHdr = (BufferDesc *) arg;
    5697             : 
    5698             :     /* Buffer is pinned, so we can read the tag without locking the spinlock */
    5699          76 :     if (bufHdr != NULL)
    5700             :     {
    5701          76 :         char       *path = relpathperm(BufTagGetRelFileLocator(&bufHdr->tag),
    5702             :                                        BufTagGetForkNum(&bufHdr->tag));
    5703             : 
    5704          76 :         errcontext("writing block %u of relation %s",
    5705             :                    bufHdr->tag.blockNum, path);
    5706          76 :         pfree(path);
    5707             :     }
    5708          76 : }
    5709             : 
    5710             : /*
    5711             :  * Error context callback for errors occurring during local buffer writes.
    5712             :  */
    5713             : static void
    5714           0 : local_buffer_write_error_callback(void *arg)
    5715             : {
    5716           0 :     BufferDesc *bufHdr = (BufferDesc *) arg;
    5717             : 
    5718           0 :     if (bufHdr != NULL)
    5719             :     {
    5720           0 :         char       *path = relpathbackend(BufTagGetRelFileLocator(&bufHdr->tag),
    5721             :                                           MyProcNumber,
    5722             :                                           BufTagGetForkNum(&bufHdr->tag));
    5723             : 
    5724           0 :         errcontext("writing block %u of relation %s",
    5725             :                    bufHdr->tag.blockNum, path);
    5726           0 :         pfree(path);
    5727             :     }
    5728           0 : }
    5729             : 
    5730             : /*
    5731             :  * RelFileLocator qsort/bsearch comparator; see RelFileLocatorEquals.
    5732             :  */
    5733             : static int
    5734    18433260 : rlocator_comparator(const void *p1, const void *p2)
    5735             : {
    5736    18433260 :     RelFileLocator n1 = *(const RelFileLocator *) p1;
    5737    18433260 :     RelFileLocator n2 = *(const RelFileLocator *) p2;
    5738             : 
    5739    18433260 :     if (n1.relNumber < n2.relNumber)
    5740    18362800 :         return -1;
    5741       70460 :     else if (n1.relNumber > n2.relNumber)
    5742       67048 :         return 1;
    5743             : 
    5744        3412 :     if (n1.dbOid < n2.dbOid)
    5745           0 :         return -1;
    5746        3412 :     else if (n1.dbOid > n2.dbOid)
    5747           0 :         return 1;
    5748             : 
    5749        3412 :     if (n1.spcOid < n2.spcOid)
    5750           0 :         return -1;
    5751        3412 :     else if (n1.spcOid > n2.spcOid)
    5752           0 :         return 1;
    5753             :     else
    5754        3412 :         return 0;
    5755             : }
    5756             : 
    5757             : /*
    5758             :  * Lock buffer header - set BM_LOCKED in buffer state.
    5759             :  */
    5760             : uint32
    5761    63932552 : LockBufHdr(BufferDesc *desc)
    5762             : {
    5763             :     SpinDelayStatus delayStatus;
    5764             :     uint32      old_buf_state;
    5765             : 
    5766             :     Assert(!BufferIsLocal(BufferDescriptorGetBuffer(desc)));
    5767             : 
    5768    63932552 :     init_local_spin_delay(&delayStatus);
    5769             : 
    5770             :     while (true)
    5771             :     {
    5772             :         /* set BM_LOCKED flag */
    5773    63972234 :         old_buf_state = pg_atomic_fetch_or_u32(&desc->state, BM_LOCKED);
    5774             :         /* if it wasn't set before we're OK */
    5775    63972234 :         if (!(old_buf_state & BM_LOCKED))
    5776    63932552 :             break;
    5777       39682 :         perform_spin_delay(&delayStatus);
    5778             :     }
    5779    63932552 :     finish_spin_delay(&delayStatus);
    5780    63932552 :     return old_buf_state | BM_LOCKED;
    5781             : }
    5782             : 
    5783             : /*
    5784             :  * Wait until the BM_LOCKED flag isn't set anymore and return the buffer's
    5785             :  * state at that point.
    5786             :  *
    5787             :  * Obviously the buffer could be locked by the time the value is returned, so
    5788             :  * this is primarily useful in CAS style loops.
    5789             :  */
    5790             : static uint32
    5791        4358 : WaitBufHdrUnlocked(BufferDesc *buf)
    5792             : {
    5793             :     SpinDelayStatus delayStatus;
    5794             :     uint32      buf_state;
    5795             : 
    5796        4358 :     init_local_spin_delay(&delayStatus);
    5797             : 
    5798        4358 :     buf_state = pg_atomic_read_u32(&buf->state);
    5799             : 
    5800       13372 :     while (buf_state & BM_LOCKED)
    5801             :     {
    5802        9014 :         perform_spin_delay(&delayStatus);
    5803        9014 :         buf_state = pg_atomic_read_u32(&buf->state);
    5804             :     }
    5805             : 
    5806        4358 :     finish_spin_delay(&delayStatus);
    5807             : 
    5808        4358 :     return buf_state;
    5809             : }
    5810             : 
    5811             : /*
    5812             :  * BufferTag comparator.
    5813             :  */
    5814             : static inline int
    5815           0 : buffertag_comparator(const BufferTag *ba, const BufferTag *bb)
    5816             : {
    5817             :     int         ret;
    5818             :     RelFileLocator rlocatora;
    5819             :     RelFileLocator rlocatorb;
    5820             : 
    5821           0 :     rlocatora = BufTagGetRelFileLocator(ba);
    5822           0 :     rlocatorb = BufTagGetRelFileLocator(bb);
    5823             : 
    5824           0 :     ret = rlocator_comparator(&rlocatora, &rlocatorb);
    5825             : 
    5826           0 :     if (ret != 0)
    5827           0 :         return ret;
    5828             : 
    5829           0 :     if (BufTagGetForkNum(ba) < BufTagGetForkNum(bb))
    5830           0 :         return -1;
    5831           0 :     if (BufTagGetForkNum(ba) > BufTagGetForkNum(bb))
    5832           0 :         return 1;
    5833             : 
    5834           0 :     if (ba->blockNum < bb->blockNum)
    5835           0 :         return -1;
    5836           0 :     if (ba->blockNum > bb->blockNum)
    5837           0 :         return 1;
    5838             : 
    5839           0 :     return 0;
    5840             : }
    5841             : 
    5842             : /*
    5843             :  * Comparator determining the writeout order in a checkpoint.
    5844             :  *
    5845             :  * It is important that tablespaces are compared first, the logic balancing
    5846             :  * writes between tablespaces relies on it.
    5847             :  */
    5848             : static inline int
    5849     5134704 : ckpt_buforder_comparator(const CkptSortItem *a, const CkptSortItem *b)
    5850             : {
    5851             :     /* compare tablespace */
    5852     5134704 :     if (a->tsId < b->tsId)
    5853       10300 :         return -1;
    5854     5124404 :     else if (a->tsId > b->tsId)
    5855       37664 :         return 1;
    5856             :     /* compare relation */
    5857     5086740 :     if (a->relNumber < b->relNumber)
    5858     1393292 :         return -1;
    5859     3693448 :     else if (a->relNumber > b->relNumber)
    5860     1384994 :         return 1;
    5861             :     /* compare fork */
    5862     2308454 :     else if (a->forkNum < b->forkNum)
    5863       96178 :         return -1;
    5864     2212276 :     else if (a->forkNum > b->forkNum)
    5865       96182 :         return 1;
    5866             :     /* compare block number */
    5867     2116094 :     else if (a->blockNum < b->blockNum)
    5868     1035814 :         return -1;
    5869     1080280 :     else if (a->blockNum > b->blockNum)
    5870     1018734 :         return 1;
    5871             :     /* equal page IDs are unlikely, but not impossible */
    5872       61546 :     return 0;
    5873             : }
    5874             : 
    5875             : /*
    5876             :  * Comparator for a Min-Heap over the per-tablespace checkpoint completion
    5877             :  * progress.
    5878             :  */
    5879             : static int
    5880      404260 : ts_ckpt_progress_comparator(Datum a, Datum b, void *arg)
    5881             : {
    5882      404260 :     CkptTsStatus *sa = (CkptTsStatus *) a;
    5883      404260 :     CkptTsStatus *sb = (CkptTsStatus *) b;
    5884             : 
    5885             :     /* we want a min-heap, so return 1 for the a < b */
    5886      404260 :     if (sa->progress < sb->progress)
    5887      384164 :         return 1;
    5888       20096 :     else if (sa->progress == sb->progress)
    5889        1298 :         return 0;
    5890             :     else
    5891       18798 :         return -1;
    5892             : }
    5893             : 
    5894             : /*
    5895             :  * Initialize a writeback context, discarding potential previous state.
    5896             :  *
    5897             :  * *max_pending is a pointer instead of an immediate value, so the coalesce
    5898             :  * limits can easily changed by the GUC mechanism, and so calling code does
    5899             :  * not have to check the current configuration. A value of 0 means that no
    5900             :  * writeback control will be performed.
    5901             :  */
    5902             : void
    5903        4660 : WritebackContextInit(WritebackContext *context, int *max_pending)
    5904             : {
    5905             :     Assert(*max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
    5906             : 
    5907        4660 :     context->max_pending = max_pending;
    5908        4660 :     context->nr_pending = 0;
    5909        4660 : }
    5910             : 
    5911             : /*
    5912             :  * Add buffer to list of pending writeback requests.
    5913             :  */
    5914             : void
    5915      945380 : ScheduleBufferTagForWriteback(WritebackContext *wb_context, IOContext io_context,
    5916             :                               BufferTag *tag)
    5917             : {
    5918             :     PendingWriteback *pending;
    5919             : 
    5920             :     /*
    5921             :      * As pg_flush_data() doesn't do anything with fsync disabled, there's no
    5922             :      * point in tracking in that case.
    5923             :      */
    5924      945380 :     if (io_direct_flags & IO_DIRECT_DATA ||
    5925      944304 :         !enableFsync)
    5926      945376 :         return;
    5927             : 
    5928             :     /*
    5929             :      * Add buffer to the pending writeback array, unless writeback control is
    5930             :      * disabled.
    5931             :      */
    5932           4 :     if (*wb_context->max_pending > 0)
    5933             :     {
    5934             :         Assert(*wb_context->max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
    5935             : 
    5936           0 :         pending = &wb_context->pending_writebacks[wb_context->nr_pending++];
    5937             : 
    5938           0 :         pending->tag = *tag;
    5939             :     }
    5940             : 
    5941             :     /*
    5942             :      * Perform pending flushes if the writeback limit is exceeded. This
    5943             :      * includes the case where previously an item has been added, but control
    5944             :      * is now disabled.
    5945             :      */
    5946           4 :     if (wb_context->nr_pending >= *wb_context->max_pending)
    5947           4 :         IssuePendingWritebacks(wb_context, io_context);
    5948             : }
    5949             : 
    5950             : #define ST_SORT sort_pending_writebacks
    5951             : #define ST_ELEMENT_TYPE PendingWriteback
    5952             : #define ST_COMPARE(a, b) buffertag_comparator(&a->tag, &b->tag)
    5953             : #define ST_SCOPE static
    5954             : #define ST_DEFINE
    5955             : #include <lib/sort_template.h>
    5956             : 
    5957             : /*
    5958             :  * Issue all pending writeback requests, previously scheduled with
    5959             :  * ScheduleBufferTagForWriteback, to the OS.
    5960             :  *
    5961             :  * Because this is only used to improve the OSs IO scheduling we try to never
    5962             :  * error out - it's just a hint.
    5963             :  */
    5964             : void
    5965        1868 : IssuePendingWritebacks(WritebackContext *wb_context, IOContext io_context)
    5966             : {
    5967             :     instr_time  io_start;
    5968             :     int         i;
    5969             : 
    5970        1868 :     if (wb_context->nr_pending == 0)
    5971        1868 :         return;
    5972             : 
    5973             :     /*
    5974             :      * Executing the writes in-order can make them a lot faster, and allows to
    5975             :      * merge writeback requests to consecutive blocks into larger writebacks.
    5976             :      */
    5977           0 :     sort_pending_writebacks(wb_context->pending_writebacks,
    5978           0 :                             wb_context->nr_pending);
    5979             : 
    5980           0 :     io_start = pgstat_prepare_io_time(track_io_timing);
    5981             : 
    5982             :     /*
    5983             :      * Coalesce neighbouring writes, but nothing else. For that we iterate
    5984             :      * through the, now sorted, array of pending flushes, and look forward to
    5985             :      * find all neighbouring (or identical) writes.
    5986             :      */
    5987           0 :     for (i = 0; i < wb_context->nr_pending; i++)
    5988             :     {
    5989             :         PendingWriteback *cur;
    5990             :         PendingWriteback *next;
    5991             :         SMgrRelation reln;
    5992             :         int         ahead;
    5993             :         BufferTag   tag;
    5994             :         RelFileLocator currlocator;
    5995           0 :         Size        nblocks = 1;
    5996             : 
    5997           0 :         cur = &wb_context->pending_writebacks[i];
    5998           0 :         tag = cur->tag;
    5999           0 :         currlocator = BufTagGetRelFileLocator(&tag);
    6000             : 
    6001             :         /*
    6002             :          * Peek ahead, into following writeback requests, to see if they can
    6003             :          * be combined with the current one.
    6004             :          */
    6005           0 :         for (ahead = 0; i + ahead + 1 < wb_context->nr_pending; ahead++)
    6006             :         {
    6007             : 
    6008           0 :             next = &wb_context->pending_writebacks[i + ahead + 1];
    6009             : 
    6010             :             /* different file, stop */
    6011           0 :             if (!RelFileLocatorEquals(currlocator,
    6012           0 :                                       BufTagGetRelFileLocator(&next->tag)) ||
    6013           0 :                 BufTagGetForkNum(&cur->tag) != BufTagGetForkNum(&next->tag))
    6014             :                 break;
    6015             : 
    6016             :             /* ok, block queued twice, skip */
    6017           0 :             if (cur->tag.blockNum == next->tag.blockNum)
    6018           0 :                 continue;
    6019             : 
    6020             :             /* only merge consecutive writes */
    6021           0 :             if (cur->tag.blockNum + 1 != next->tag.blockNum)
    6022           0 :                 break;
    6023             : 
    6024           0 :             nblocks++;
    6025           0 :             cur = next;
    6026             :         }
    6027             : 
    6028           0 :         i += ahead;
    6029             : 
    6030             :         /* and finally tell the kernel to write the data to storage */
    6031           0 :         reln = smgropen(currlocator, INVALID_PROC_NUMBER);
    6032           0 :         smgrwriteback(reln, BufTagGetForkNum(&tag), tag.blockNum, nblocks);
    6033             :     }
    6034             : 
    6035             :     /*
    6036             :      * Assume that writeback requests are only issued for buffers containing
    6037             :      * blocks of permanent relations.
    6038             :      */
    6039           0 :     pgstat_count_io_op_time(IOOBJECT_RELATION, io_context,
    6040           0 :                             IOOP_WRITEBACK, io_start, wb_context->nr_pending);
    6041             : 
    6042           0 :     wb_context->nr_pending = 0;
    6043             : }
    6044             : 
    6045             : /* ResourceOwner callbacks */
    6046             : 
    6047             : static void
    6048          30 : ResOwnerReleaseBufferIO(Datum res)
    6049             : {
    6050          30 :     Buffer      buffer = DatumGetInt32(res);
    6051             : 
    6052          30 :     AbortBufferIO(buffer);
    6053          30 : }
    6054             : 
    6055             : static char *
    6056           0 : ResOwnerPrintBufferIO(Datum res)
    6057             : {
    6058           0 :     Buffer      buffer = DatumGetInt32(res);
    6059             : 
    6060           0 :     return psprintf("lost track of buffer IO on buffer %d", buffer);
    6061             : }
    6062             : 
    6063             : static void
    6064        8916 : ResOwnerReleaseBufferPin(Datum res)
    6065             : {
    6066        8916 :     Buffer      buffer = DatumGetInt32(res);
    6067             : 
    6068             :     /* Like ReleaseBuffer, but don't call ResourceOwnerForgetBuffer */
    6069        8916 :     if (!BufferIsValid(buffer))
    6070           0 :         elog(ERROR, "bad buffer ID: %d", buffer);
    6071             : 
    6072        8916 :     if (BufferIsLocal(buffer))
    6073         754 :         UnpinLocalBufferNoOwner(buffer);
    6074             :     else
    6075        8162 :         UnpinBufferNoOwner(GetBufferDescriptor(buffer - 1));
    6076        8916 : }
    6077             : 
    6078             : static char *
    6079           0 : ResOwnerPrintBufferPin(Datum res)
    6080             : {
    6081           0 :     return DebugPrintBufferRefcount(DatumGetInt32(res));
    6082             : }
    6083             : 
    6084             : /*
    6085             :  * Try to evict the current block in a shared buffer.
    6086             :  *
    6087             :  * This function is intended for testing/development use only!
    6088             :  *
    6089             :  * To succeed, the buffer must not be pinned on entry, so if the caller had a
    6090             :  * particular block in mind, it might already have been replaced by some other
    6091             :  * block by the time this function runs.  It's also unpinned on return, so the
    6092             :  * buffer might be occupied again by the time control is returned, potentially
    6093             :  * even by the same block.  This inherent raciness without other interlocking
    6094             :  * makes the function unsuitable for non-testing usage.
    6095             :  *
    6096             :  * Returns true if the buffer was valid and it has now been made invalid.
    6097             :  * Returns false if it wasn't valid, if it couldn't be evicted due to a pin,
    6098             :  * or if the buffer becomes dirty again while we're trying to write it out.
    6099             :  */
    6100             : bool
    6101           0 : EvictUnpinnedBuffer(Buffer buf)
    6102             : {
    6103             :     BufferDesc *desc;
    6104             :     uint32      buf_state;
    6105             :     bool        result;
    6106             : 
    6107             :     /* Make sure we can pin the buffer. */
    6108           0 :     ResourceOwnerEnlarge(CurrentResourceOwner);
    6109           0 :     ReservePrivateRefCountEntry();
    6110             : 
    6111             :     Assert(!BufferIsLocal(buf));
    6112           0 :     desc = GetBufferDescriptor(buf - 1);
    6113             : 
    6114             :     /* Lock the header and check if it's valid. */
    6115           0 :     buf_state = LockBufHdr(desc);
    6116           0 :     if ((buf_state & BM_VALID) == 0)
    6117             :     {
    6118           0 :         UnlockBufHdr(desc, buf_state);
    6119           0 :         return false;
    6120             :     }
    6121             : 
    6122             :     /* Check that it's not pinned already. */
    6123           0 :     if (BUF_STATE_GET_REFCOUNT(buf_state) > 0)
    6124             :     {
    6125           0 :         UnlockBufHdr(desc, buf_state);
    6126           0 :         return false;
    6127             :     }
    6128             : 
    6129           0 :     PinBuffer_Locked(desc);     /* releases spinlock */
    6130             : 
    6131             :     /* If it was dirty, try to clean it once. */
    6132           0 :     if (buf_state & BM_DIRTY)
    6133             :     {
    6134           0 :         LWLockAcquire(BufferDescriptorGetContentLock(desc), LW_SHARED);
    6135           0 :         FlushBuffer(desc, NULL, IOOBJECT_RELATION, IOCONTEXT_NORMAL);
    6136           0 :         LWLockRelease(BufferDescriptorGetContentLock(desc));
    6137             :     }
    6138             : 
    6139             :     /* This will return false if it becomes dirty or someone else pins it. */
    6140           0 :     result = InvalidateVictimBuffer(desc);
    6141             : 
    6142           0 :     UnpinBuffer(desc);
    6143             : 
    6144           0 :     return result;
    6145             : }

Generated by: LCOV version 1.14