LCOV - code coverage report
Current view: top level - src/backend/access/heap - heapam.c (source / functions) Hit Total Coverage
Test: PostgreSQL 18devel Lines: 2499 2730 91.5 %
Date: 2025-04-01 15:15:16 Functions: 81 81 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * heapam.c
       4             :  *    heap access method code
       5             :  *
       6             :  * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
       7             :  * Portions Copyright (c) 1994, Regents of the University of California
       8             :  *
       9             :  *
      10             :  * IDENTIFICATION
      11             :  *    src/backend/access/heap/heapam.c
      12             :  *
      13             :  *
      14             :  * INTERFACE ROUTINES
      15             :  *      heap_beginscan  - begin relation scan
      16             :  *      heap_rescan     - restart a relation scan
      17             :  *      heap_endscan    - end relation scan
      18             :  *      heap_getnext    - retrieve next tuple in scan
      19             :  *      heap_fetch      - retrieve tuple with given tid
      20             :  *      heap_insert     - insert tuple into a relation
      21             :  *      heap_multi_insert - insert multiple tuples into a relation
      22             :  *      heap_delete     - delete a tuple from a relation
      23             :  *      heap_update     - replace a tuple in a relation with another tuple
      24             :  *
      25             :  * NOTES
      26             :  *    This file contains the heap_ routines which implement
      27             :  *    the POSTGRES heap access method used for all POSTGRES
      28             :  *    relations.
      29             :  *
      30             :  *-------------------------------------------------------------------------
      31             :  */
      32             : #include "postgres.h"
      33             : 
      34             : #include "access/heapam.h"
      35             : #include "access/heaptoast.h"
      36             : #include "access/hio.h"
      37             : #include "access/multixact.h"
      38             : #include "access/subtrans.h"
      39             : #include "access/syncscan.h"
      40             : #include "access/valid.h"
      41             : #include "access/visibilitymap.h"
      42             : #include "access/xloginsert.h"
      43             : #include "catalog/pg_database.h"
      44             : #include "catalog/pg_database_d.h"
      45             : #include "commands/vacuum.h"
      46             : #include "pgstat.h"
      47             : #include "port/pg_bitutils.h"
      48             : #include "storage/lmgr.h"
      49             : #include "storage/predicate.h"
      50             : #include "storage/procarray.h"
      51             : #include "utils/datum.h"
      52             : #include "utils/injection_point.h"
      53             : #include "utils/inval.h"
      54             : #include "utils/spccache.h"
      55             : #include "utils/syscache.h"
      56             : 
      57             : 
      58             : static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup,
      59             :                                      TransactionId xid, CommandId cid, int options);
      60             : static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf,
      61             :                                   Buffer newbuf, HeapTuple oldtup,
      62             :                                   HeapTuple newtup, HeapTuple old_key_tuple,
      63             :                                   bool all_visible_cleared, bool new_all_visible_cleared);
      64             : #ifdef USE_ASSERT_CHECKING
      65             : static void check_lock_if_inplace_updateable_rel(Relation relation,
      66             :                                                  ItemPointer otid,
      67             :                                                  HeapTuple newtup);
      68             : static void check_inplace_rel_lock(HeapTuple oldtup);
      69             : #endif
      70             : static Bitmapset *HeapDetermineColumnsInfo(Relation relation,
      71             :                                            Bitmapset *interesting_cols,
      72             :                                            Bitmapset *external_cols,
      73             :                                            HeapTuple oldtup, HeapTuple newtup,
      74             :                                            bool *has_external);
      75             : static bool heap_acquire_tuplock(Relation relation, ItemPointer tid,
      76             :                                  LockTupleMode mode, LockWaitPolicy wait_policy,
      77             :                                  bool *have_tuple_lock);
      78             : static inline BlockNumber heapgettup_advance_block(HeapScanDesc scan,
      79             :                                                    BlockNumber block,
      80             :                                                    ScanDirection dir);
      81             : static pg_noinline BlockNumber heapgettup_initial_block(HeapScanDesc scan,
      82             :                                                         ScanDirection dir);
      83             : static void compute_new_xmax_infomask(TransactionId xmax, uint16 old_infomask,
      84             :                                       uint16 old_infomask2, TransactionId add_to_xmax,
      85             :                                       LockTupleMode mode, bool is_update,
      86             :                                       TransactionId *result_xmax, uint16 *result_infomask,
      87             :                                       uint16 *result_infomask2);
      88             : static TM_Result heap_lock_updated_tuple(Relation rel, HeapTuple tuple,
      89             :                                          ItemPointer ctid, TransactionId xid,
      90             :                                          LockTupleMode mode);
      91             : static void GetMultiXactIdHintBits(MultiXactId multi, uint16 *new_infomask,
      92             :                                    uint16 *new_infomask2);
      93             : static TransactionId MultiXactIdGetUpdateXid(TransactionId xmax,
      94             :                                              uint16 t_infomask);
      95             : static bool DoesMultiXactIdConflict(MultiXactId multi, uint16 infomask,
      96             :                                     LockTupleMode lockmode, bool *current_is_member);
      97             : static void MultiXactIdWait(MultiXactId multi, MultiXactStatus status, uint16 infomask,
      98             :                             Relation rel, ItemPointer ctid, XLTW_Oper oper,
      99             :                             int *remaining);
     100             : static bool ConditionalMultiXactIdWait(MultiXactId multi, MultiXactStatus status,
     101             :                                        uint16 infomask, Relation rel, int *remaining,
     102             :                                        bool logLockFailure);
     103             : static void index_delete_sort(TM_IndexDeleteOp *delstate);
     104             : static int  bottomup_sort_and_shrink(TM_IndexDeleteOp *delstate);
     105             : static XLogRecPtr log_heap_new_cid(Relation relation, HeapTuple tup);
     106             : static HeapTuple ExtractReplicaIdentity(Relation relation, HeapTuple tp, bool key_required,
     107             :                                         bool *copy);
     108             : 
     109             : 
     110             : /*
     111             :  * Each tuple lock mode has a corresponding heavyweight lock, and one or two
     112             :  * corresponding MultiXactStatuses (one to merely lock tuples, another one to
     113             :  * update them).  This table (and the macros below) helps us determine the
     114             :  * heavyweight lock mode and MultiXactStatus values to use for any particular
     115             :  * tuple lock strength.
     116             :  *
     117             :  * These interact with InplaceUpdateTupleLock, an alias for ExclusiveLock.
     118             :  *
     119             :  * Don't look at lockstatus/updstatus directly!  Use get_mxact_status_for_lock
     120             :  * instead.
     121             :  */
     122             : static const struct
     123             : {
     124             :     LOCKMODE    hwlock;
     125             :     int         lockstatus;
     126             :     int         updstatus;
     127             : }
     128             : 
     129             :             tupleLockExtraInfo[MaxLockTupleMode + 1] =
     130             : {
     131             :     {                           /* LockTupleKeyShare */
     132             :         AccessShareLock,
     133             :         MultiXactStatusForKeyShare,
     134             :         -1                      /* KeyShare does not allow updating tuples */
     135             :     },
     136             :     {                           /* LockTupleShare */
     137             :         RowShareLock,
     138             :         MultiXactStatusForShare,
     139             :         -1                      /* Share does not allow updating tuples */
     140             :     },
     141             :     {                           /* LockTupleNoKeyExclusive */
     142             :         ExclusiveLock,
     143             :         MultiXactStatusForNoKeyUpdate,
     144             :         MultiXactStatusNoKeyUpdate
     145             :     },
     146             :     {                           /* LockTupleExclusive */
     147             :         AccessExclusiveLock,
     148             :         MultiXactStatusForUpdate,
     149             :         MultiXactStatusUpdate
     150             :     }
     151             : };
     152             : 
     153             : /* Get the LOCKMODE for a given MultiXactStatus */
     154             : #define LOCKMODE_from_mxstatus(status) \
     155             :             (tupleLockExtraInfo[TUPLOCK_from_mxstatus((status))].hwlock)
     156             : 
     157             : /*
     158             :  * Acquire heavyweight locks on tuples, using a LockTupleMode strength value.
     159             :  * This is more readable than having every caller translate it to lock.h's
     160             :  * LOCKMODE.
     161             :  */
     162             : #define LockTupleTuplock(rel, tup, mode) \
     163             :     LockTuple((rel), (tup), tupleLockExtraInfo[mode].hwlock)
     164             : #define UnlockTupleTuplock(rel, tup, mode) \
     165             :     UnlockTuple((rel), (tup), tupleLockExtraInfo[mode].hwlock)
     166             : #define ConditionalLockTupleTuplock(rel, tup, mode, log) \
     167             :     ConditionalLockTuple((rel), (tup), tupleLockExtraInfo[mode].hwlock, (log))
     168             : 
     169             : #ifdef USE_PREFETCH
     170             : /*
     171             :  * heap_index_delete_tuples and index_delete_prefetch_buffer use this
     172             :  * structure to coordinate prefetching activity
     173             :  */
     174             : typedef struct
     175             : {
     176             :     BlockNumber cur_hblkno;
     177             :     int         next_item;
     178             :     int         ndeltids;
     179             :     TM_IndexDelete *deltids;
     180             : } IndexDeletePrefetchState;
     181             : #endif
     182             : 
     183             : /* heap_index_delete_tuples bottom-up index deletion costing constants */
     184             : #define BOTTOMUP_MAX_NBLOCKS            6
     185             : #define BOTTOMUP_TOLERANCE_NBLOCKS      3
     186             : 
     187             : /*
     188             :  * heap_index_delete_tuples uses this when determining which heap blocks it
     189             :  * must visit to help its bottom-up index deletion caller
     190             :  */
     191             : typedef struct IndexDeleteCounts
     192             : {
     193             :     int16       npromisingtids; /* Number of "promising" TIDs in group */
     194             :     int16       ntids;          /* Number of TIDs in group */
     195             :     int16       ifirsttid;      /* Offset to group's first deltid */
     196             : } IndexDeleteCounts;
     197             : 
     198             : /*
     199             :  * This table maps tuple lock strength values for each particular
     200             :  * MultiXactStatus value.
     201             :  */
     202             : static const int MultiXactStatusLock[MaxMultiXactStatus + 1] =
     203             : {
     204             :     LockTupleKeyShare,          /* ForKeyShare */
     205             :     LockTupleShare,             /* ForShare */
     206             :     LockTupleNoKeyExclusive,    /* ForNoKeyUpdate */
     207             :     LockTupleExclusive,         /* ForUpdate */
     208             :     LockTupleNoKeyExclusive,    /* NoKeyUpdate */
     209             :     LockTupleExclusive          /* Update */
     210             : };
     211             : 
     212             : /* Get the LockTupleMode for a given MultiXactStatus */
     213             : #define TUPLOCK_from_mxstatus(status) \
     214             :             (MultiXactStatusLock[(status)])
     215             : 
     216             : /* ----------------------------------------------------------------
     217             :  *                       heap support routines
     218             :  * ----------------------------------------------------------------
     219             :  */
     220             : 
     221             : /*
     222             :  * Streaming read API callback for parallel sequential scans. Returns the next
     223             :  * block the caller wants from the read stream or InvalidBlockNumber when done.
     224             :  */
     225             : static BlockNumber
     226      201416 : heap_scan_stream_read_next_parallel(ReadStream *stream,
     227             :                                     void *callback_private_data,
     228             :                                     void *per_buffer_data)
     229             : {
     230      201416 :     HeapScanDesc scan = (HeapScanDesc) callback_private_data;
     231             : 
     232             :     Assert(ScanDirectionIsForward(scan->rs_dir));
     233             :     Assert(scan->rs_base.rs_parallel);
     234             : 
     235      201416 :     if (unlikely(!scan->rs_inited))
     236             :     {
     237             :         /* parallel scan */
     238        2836 :         table_block_parallelscan_startblock_init(scan->rs_base.rs_rd,
     239        2836 :                                                  scan->rs_parallelworkerdata,
     240        2836 :                                                  (ParallelBlockTableScanDesc) scan->rs_base.rs_parallel);
     241             : 
     242             :         /* may return InvalidBlockNumber if there are no more blocks */
     243        5672 :         scan->rs_prefetch_block = table_block_parallelscan_nextpage(scan->rs_base.rs_rd,
     244        2836 :                                                                     scan->rs_parallelworkerdata,
     245        2836 :                                                                     (ParallelBlockTableScanDesc) scan->rs_base.rs_parallel);
     246        2836 :         scan->rs_inited = true;
     247             :     }
     248             :     else
     249             :     {
     250      198580 :         scan->rs_prefetch_block = table_block_parallelscan_nextpage(scan->rs_base.rs_rd,
     251      198580 :                                                                     scan->rs_parallelworkerdata, (ParallelBlockTableScanDesc)
     252      198580 :                                                                     scan->rs_base.rs_parallel);
     253             :     }
     254             : 
     255      201416 :     return scan->rs_prefetch_block;
     256             : }
     257             : 
     258             : /*
     259             :  * Streaming read API callback for serial sequential and TID range scans.
     260             :  * Returns the next block the caller wants from the read stream or
     261             :  * InvalidBlockNumber when done.
     262             :  */
     263             : static BlockNumber
     264     7066924 : heap_scan_stream_read_next_serial(ReadStream *stream,
     265             :                                   void *callback_private_data,
     266             :                                   void *per_buffer_data)
     267             : {
     268     7066924 :     HeapScanDesc scan = (HeapScanDesc) callback_private_data;
     269             : 
     270     7066924 :     if (unlikely(!scan->rs_inited))
     271             :     {
     272     1839116 :         scan->rs_prefetch_block = heapgettup_initial_block(scan, scan->rs_dir);
     273     1839116 :         scan->rs_inited = true;
     274             :     }
     275             :     else
     276     5227808 :         scan->rs_prefetch_block = heapgettup_advance_block(scan,
     277             :                                                            scan->rs_prefetch_block,
     278             :                                                            scan->rs_dir);
     279             : 
     280     7066924 :     return scan->rs_prefetch_block;
     281             : }
     282             : 
     283             : /*
     284             :  * Read stream API callback for bitmap heap scans.
     285             :  * Returns the next block the caller wants from the read stream or
     286             :  * InvalidBlockNumber when done.
     287             :  */
     288             : static BlockNumber
     289      397238 : bitmapheap_stream_read_next(ReadStream *pgsr, void *private_data,
     290             :                             void *per_buffer_data)
     291             : {
     292      397238 :     TBMIterateResult *tbmres = per_buffer_data;
     293      397238 :     BitmapHeapScanDesc bscan = (BitmapHeapScanDesc) private_data;
     294      397238 :     HeapScanDesc hscan = (HeapScanDesc) bscan;
     295      397238 :     TableScanDesc sscan = &hscan->rs_base;
     296             : 
     297             :     for (;;)
     298             :     {
     299      422804 :         CHECK_FOR_INTERRUPTS();
     300             : 
     301             :         /* no more entries in the bitmap */
     302      422804 :         if (!tbm_iterate(&sscan->st.rs_tbmiterator, tbmres))
     303       25560 :             return InvalidBlockNumber;
     304             : 
     305             :         /*
     306             :          * Ignore any claimed entries past what we think is the end of the
     307             :          * relation. It may have been extended after the start of our scan (we
     308             :          * only hold an AccessShareLock, and it could be inserts from this
     309             :          * backend).  We don't take this optimization in SERIALIZABLE
     310             :          * isolation though, as we need to examine all invisible tuples
     311             :          * reachable by the index.
     312             :          */
     313      397244 :         if (!IsolationIsSerializable() &&
     314      397038 :             tbmres->blockno >= hscan->rs_nblocks)
     315           0 :             continue;
     316             : 
     317             :         /*
     318             :          * We can skip fetching the heap page if we don't need any fields from
     319             :          * the heap, the bitmap entries don't need rechecking, and all tuples
     320             :          * on the page are visible to our transaction.
     321             :          */
     322      397244 :         if (!(sscan->rs_flags & SO_NEED_TUPLES) &&
     323      100456 :             !tbmres->recheck &&
     324       58024 :             VM_ALL_VISIBLE(sscan->rs_rd, tbmres->blockno, &bscan->rs_vmbuffer))
     325             :         {
     326             :             OffsetNumber offsets[TBM_MAX_TUPLES_PER_PAGE];
     327             :             int         noffsets;
     328             : 
     329             :             /* can't be lossy in the skip_fetch case */
     330             :             Assert(!tbmres->lossy);
     331             :             Assert(bscan->rs_empty_tuples_pending >= 0);
     332             : 
     333             :             /*
     334             :              * We throw away the offsets, but this is the easiest way to get a
     335             :              * count of tuples.
     336             :              */
     337       25566 :             noffsets = tbm_extract_page_tuple(tbmres, offsets, TBM_MAX_TUPLES_PER_PAGE);
     338       25566 :             bscan->rs_empty_tuples_pending += noffsets;
     339       25566 :             continue;
     340             :         }
     341             : 
     342      371678 :         return tbmres->blockno;
     343             :     }
     344             : 
     345             :     /* not reachable */
     346             :     Assert(false);
     347             : }
     348             : 
     349             : /* ----------------
     350             :  *      initscan - scan code common to heap_beginscan and heap_rescan
     351             :  * ----------------
     352             :  */
     353             : static void
     354     1886080 : initscan(HeapScanDesc scan, ScanKey key, bool keep_startblock)
     355             : {
     356     1886080 :     ParallelBlockTableScanDesc bpscan = NULL;
     357             :     bool        allow_strat;
     358             :     bool        allow_sync;
     359             : 
     360             :     /*
     361             :      * Determine the number of blocks we have to scan.
     362             :      *
     363             :      * It is sufficient to do this once at scan start, since any tuples added
     364             :      * while the scan is in progress will be invisible to my snapshot anyway.
     365             :      * (That is not true when using a non-MVCC snapshot.  However, we couldn't
     366             :      * guarantee to return tuples added after scan start anyway, since they
     367             :      * might go into pages we already scanned.  To guarantee consistent
     368             :      * results for a non-MVCC snapshot, the caller must hold some higher-level
     369             :      * lock that ensures the interesting tuple(s) won't change.)
     370             :      */
     371     1886080 :     if (scan->rs_base.rs_parallel != NULL)
     372             :     {
     373        4024 :         bpscan = (ParallelBlockTableScanDesc) scan->rs_base.rs_parallel;
     374        4024 :         scan->rs_nblocks = bpscan->phs_nblocks;
     375             :     }
     376             :     else
     377     1882056 :         scan->rs_nblocks = RelationGetNumberOfBlocks(scan->rs_base.rs_rd);
     378             : 
     379             :     /*
     380             :      * If the table is large relative to NBuffers, use a bulk-read access
     381             :      * strategy and enable synchronized scanning (see syncscan.c).  Although
     382             :      * the thresholds for these features could be different, we make them the
     383             :      * same so that there are only two behaviors to tune rather than four.
     384             :      * (However, some callers need to be able to disable one or both of these
     385             :      * behaviors, independently of the size of the table; also there is a GUC
     386             :      * variable that can disable synchronized scanning.)
     387             :      *
     388             :      * Note that table_block_parallelscan_initialize has a very similar test;
     389             :      * if you change this, consider changing that one, too.
     390             :      */
     391     1886076 :     if (!RelationUsesLocalBuffers(scan->rs_base.rs_rd) &&
     392     1871904 :         scan->rs_nblocks > NBuffers / 4)
     393             :     {
     394       24394 :         allow_strat = (scan->rs_base.rs_flags & SO_ALLOW_STRAT) != 0;
     395       24394 :         allow_sync = (scan->rs_base.rs_flags & SO_ALLOW_SYNC) != 0;
     396             :     }
     397             :     else
     398     1861682 :         allow_strat = allow_sync = false;
     399             : 
     400     1886076 :     if (allow_strat)
     401             :     {
     402             :         /* During a rescan, keep the previous strategy object. */
     403       21866 :         if (scan->rs_strategy == NULL)
     404       21518 :             scan->rs_strategy = GetAccessStrategy(BAS_BULKREAD);
     405             :     }
     406             :     else
     407             :     {
     408     1864210 :         if (scan->rs_strategy != NULL)
     409           0 :             FreeAccessStrategy(scan->rs_strategy);
     410     1864210 :         scan->rs_strategy = NULL;
     411             :     }
     412             : 
     413     1886076 :     if (scan->rs_base.rs_parallel != NULL)
     414             :     {
     415             :         /* For parallel scan, believe whatever ParallelTableScanDesc says. */
     416        4024 :         if (scan->rs_base.rs_parallel->phs_syncscan)
     417           4 :             scan->rs_base.rs_flags |= SO_ALLOW_SYNC;
     418             :         else
     419        4020 :             scan->rs_base.rs_flags &= ~SO_ALLOW_SYNC;
     420             :     }
     421     1882052 :     else if (keep_startblock)
     422             :     {
     423             :         /*
     424             :          * When rescanning, we want to keep the previous startblock setting,
     425             :          * so that rewinding a cursor doesn't generate surprising results.
     426             :          * Reset the active syncscan setting, though.
     427             :          */
     428     1166666 :         if (allow_sync && synchronize_seqscans)
     429          40 :             scan->rs_base.rs_flags |= SO_ALLOW_SYNC;
     430             :         else
     431     1166626 :             scan->rs_base.rs_flags &= ~SO_ALLOW_SYNC;
     432             :     }
     433      715386 :     else if (allow_sync && synchronize_seqscans)
     434             :     {
     435         118 :         scan->rs_base.rs_flags |= SO_ALLOW_SYNC;
     436         118 :         scan->rs_startblock = ss_get_location(scan->rs_base.rs_rd, scan->rs_nblocks);
     437             :     }
     438             :     else
     439             :     {
     440      715268 :         scan->rs_base.rs_flags &= ~SO_ALLOW_SYNC;
     441      715268 :         scan->rs_startblock = 0;
     442             :     }
     443             : 
     444     1886076 :     scan->rs_numblocks = InvalidBlockNumber;
     445     1886076 :     scan->rs_inited = false;
     446     1886076 :     scan->rs_ctup.t_data = NULL;
     447     1886076 :     ItemPointerSetInvalid(&scan->rs_ctup.t_self);
     448     1886076 :     scan->rs_cbuf = InvalidBuffer;
     449     1886076 :     scan->rs_cblock = InvalidBlockNumber;
     450     1886076 :     scan->rs_ntuples = 0;
     451     1886076 :     scan->rs_cindex = 0;
     452             : 
     453             :     /*
     454             :      * Initialize to ForwardScanDirection because it is most common and
     455             :      * because heap scans go forward before going backward (e.g. CURSORs).
     456             :      */
     457     1886076 :     scan->rs_dir = ForwardScanDirection;
     458     1886076 :     scan->rs_prefetch_block = InvalidBlockNumber;
     459             : 
     460             :     /* page-at-a-time fields are always invalid when not rs_inited */
     461             : 
     462             :     /*
     463             :      * copy the scan key, if appropriate
     464             :      */
     465     1886076 :     if (key != NULL && scan->rs_base.rs_nkeys > 0)
     466      405734 :         memcpy(scan->rs_base.rs_key, key, scan->rs_base.rs_nkeys * sizeof(ScanKeyData));
     467             : 
     468             :     /*
     469             :      * Currently, we only have a stats counter for sequential heap scans (but
     470             :      * e.g for bitmap scans the underlying bitmap index scans will be counted,
     471             :      * and for sample scans we update stats for tuple fetches).
     472             :      */
     473     1886076 :     if (scan->rs_base.rs_flags & SO_TYPE_SEQSCAN)
     474     1841472 :         pgstat_count_heap_scan(scan->rs_base.rs_rd);
     475     1886076 : }
     476             : 
     477             : /*
     478             :  * heap_setscanlimits - restrict range of a heapscan
     479             :  *
     480             :  * startBlk is the page to start at
     481             :  * numBlks is number of pages to scan (InvalidBlockNumber means "all")
     482             :  */
     483             : void
     484        5606 : heap_setscanlimits(TableScanDesc sscan, BlockNumber startBlk, BlockNumber numBlks)
     485             : {
     486        5606 :     HeapScanDesc scan = (HeapScanDesc) sscan;
     487             : 
     488             :     Assert(!scan->rs_inited);    /* else too late to change */
     489             :     /* else rs_startblock is significant */
     490             :     Assert(!(scan->rs_base.rs_flags & SO_ALLOW_SYNC));
     491             : 
     492             :     /* Check startBlk is valid (but allow case of zero blocks...) */
     493             :     Assert(startBlk == 0 || startBlk < scan->rs_nblocks);
     494             : 
     495        5606 :     scan->rs_startblock = startBlk;
     496        5606 :     scan->rs_numblocks = numBlks;
     497        5606 : }
     498             : 
     499             : /*
     500             :  * Per-tuple loop for heap_prepare_pagescan(). Pulled out so it can be called
     501             :  * multiple times, with constant arguments for all_visible,
     502             :  * check_serializable.
     503             :  */
     504             : pg_attribute_always_inline
     505             : static int
     506     5204386 : page_collect_tuples(HeapScanDesc scan, Snapshot snapshot,
     507             :                     Page page, Buffer buffer,
     508             :                     BlockNumber block, int lines,
     509             :                     bool all_visible, bool check_serializable)
     510             : {
     511     5204386 :     int         ntup = 0;
     512             :     OffsetNumber lineoff;
     513             : 
     514   261307050 :     for (lineoff = FirstOffsetNumber; lineoff <= lines; lineoff++)
     515             :     {
     516   256102680 :         ItemId      lpp = PageGetItemId(page, lineoff);
     517             :         HeapTupleData loctup;
     518             :         bool        valid;
     519             : 
     520   256102680 :         if (!ItemIdIsNormal(lpp))
     521    54793258 :             continue;
     522             : 
     523   201309422 :         loctup.t_data = (HeapTupleHeader) PageGetItem(page, lpp);
     524   201309422 :         loctup.t_len = ItemIdGetLength(lpp);
     525   201309422 :         loctup.t_tableOid = RelationGetRelid(scan->rs_base.rs_rd);
     526   201309422 :         ItemPointerSet(&(loctup.t_self), block, lineoff);
     527             : 
     528   201309422 :         if (all_visible)
     529    77700442 :             valid = true;
     530             :         else
     531   123608980 :             valid = HeapTupleSatisfiesVisibility(&loctup, snapshot, buffer);
     532             : 
     533   201309422 :         if (check_serializable)
     534        2810 :             HeapCheckForSerializableConflictOut(valid, scan->rs_base.rs_rd,
     535             :                                                 &loctup, buffer, snapshot);
     536             : 
     537   201309406 :         if (valid)
     538             :         {
     539   185182116 :             scan->rs_vistuples[ntup] = lineoff;
     540   185182116 :             ntup++;
     541             :         }
     542             :     }
     543             : 
     544             :     Assert(ntup <= MaxHeapTuplesPerPage);
     545             : 
     546     5204370 :     return ntup;
     547             : }
     548             : 
     549             : /*
     550             :  * heap_prepare_pagescan - Prepare current scan page to be scanned in pagemode
     551             :  *
     552             :  * Preparation currently consists of 1. prune the scan's rs_cbuf page, and 2.
     553             :  * fill the rs_vistuples[] array with the OffsetNumbers of visible tuples.
     554             :  */
     555             : void
     556     5204386 : heap_prepare_pagescan(TableScanDesc sscan)
     557             : {
     558     5204386 :     HeapScanDesc scan = (HeapScanDesc) sscan;
     559     5204386 :     Buffer      buffer = scan->rs_cbuf;
     560     5204386 :     BlockNumber block = scan->rs_cblock;
     561             :     Snapshot    snapshot;
     562             :     Page        page;
     563             :     int         lines;
     564             :     bool        all_visible;
     565             :     bool        check_serializable;
     566             : 
     567             :     Assert(BufferGetBlockNumber(buffer) == block);
     568             : 
     569             :     /* ensure we're not accidentally being used when not in pagemode */
     570             :     Assert(scan->rs_base.rs_flags & SO_ALLOW_PAGEMODE);
     571     5204386 :     snapshot = scan->rs_base.rs_snapshot;
     572             : 
     573             :     /*
     574             :      * Prune and repair fragmentation for the whole page, if possible.
     575             :      */
     576     5204386 :     heap_page_prune_opt(scan->rs_base.rs_rd, buffer);
     577             : 
     578             :     /*
     579             :      * We must hold share lock on the buffer content while examining tuple
     580             :      * visibility.  Afterwards, however, the tuples we have found to be
     581             :      * visible are guaranteed good as long as we hold the buffer pin.
     582             :      */
     583     5204386 :     LockBuffer(buffer, BUFFER_LOCK_SHARE);
     584             : 
     585     5204386 :     page = BufferGetPage(buffer);
     586     5204386 :     lines = PageGetMaxOffsetNumber(page);
     587             : 
     588             :     /*
     589             :      * If the all-visible flag indicates that all tuples on the page are
     590             :      * visible to everyone, we can skip the per-tuple visibility tests.
     591             :      *
     592             :      * Note: In hot standby, a tuple that's already visible to all
     593             :      * transactions on the primary might still be invisible to a read-only
     594             :      * transaction in the standby. We partly handle this problem by tracking
     595             :      * the minimum xmin of visible tuples as the cut-off XID while marking a
     596             :      * page all-visible on the primary and WAL log that along with the
     597             :      * visibility map SET operation. In hot standby, we wait for (or abort)
     598             :      * all transactions that can potentially may not see one or more tuples on
     599             :      * the page. That's how index-only scans work fine in hot standby. A
     600             :      * crucial difference between index-only scans and heap scans is that the
     601             :      * index-only scan completely relies on the visibility map where as heap
     602             :      * scan looks at the page-level PD_ALL_VISIBLE flag. We are not sure if
     603             :      * the page-level flag can be trusted in the same way, because it might
     604             :      * get propagated somehow without being explicitly WAL-logged, e.g. via a
     605             :      * full page write. Until we can prove that beyond doubt, let's check each
     606             :      * tuple for visibility the hard way.
     607             :      */
     608     5204386 :     all_visible = PageIsAllVisible(page) && !snapshot->takenDuringRecovery;
     609             :     check_serializable =
     610     5204386 :         CheckForSerializableConflictOutNeeded(scan->rs_base.rs_rd, snapshot);
     611             : 
     612             :     /*
     613             :      * We call page_collect_tuples() with constant arguments, to get the
     614             :      * compiler to constant fold the constant arguments. Separate calls with
     615             :      * constant arguments, rather than variables, are needed on several
     616             :      * compilers to actually perform constant folding.
     617             :      */
     618     5204386 :     if (likely(all_visible))
     619             :     {
     620     1815780 :         if (likely(!check_serializable))
     621     1815780 :             scan->rs_ntuples = page_collect_tuples(scan, snapshot, page, buffer,
     622             :                                                    block, lines, true, false);
     623             :         else
     624           0 :             scan->rs_ntuples = page_collect_tuples(scan, snapshot, page, buffer,
     625             :                                                    block, lines, true, true);
     626             :     }
     627             :     else
     628             :     {
     629     3388606 :         if (likely(!check_serializable))
     630     3387368 :             scan->rs_ntuples = page_collect_tuples(scan, snapshot, page, buffer,
     631             :                                                    block, lines, false, false);
     632             :         else
     633        1238 :             scan->rs_ntuples = page_collect_tuples(scan, snapshot, page, buffer,
     634             :                                                    block, lines, false, true);
     635             :     }
     636             : 
     637     5204370 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
     638     5204370 : }
     639             : 
     640             : /*
     641             :  * heap_fetch_next_buffer - read and pin the next block from MAIN_FORKNUM.
     642             :  *
     643             :  * Read the next block of the scan relation from the read stream and save it
     644             :  * in the scan descriptor.  It is already pinned.
     645             :  */
     646             : static inline void
     647     6918848 : heap_fetch_next_buffer(HeapScanDesc scan, ScanDirection dir)
     648             : {
     649             :     Assert(scan->rs_read_stream);
     650             : 
     651             :     /* release previous scan buffer, if any */
     652     6918848 :     if (BufferIsValid(scan->rs_cbuf))
     653             :     {
     654     5076892 :         ReleaseBuffer(scan->rs_cbuf);
     655     5076892 :         scan->rs_cbuf = InvalidBuffer;
     656             :     }
     657             : 
     658             :     /*
     659             :      * Be sure to check for interrupts at least once per page.  Checks at
     660             :      * higher code levels won't be able to stop a seqscan that encounters many
     661             :      * pages' worth of consecutive dead tuples.
     662             :      */
     663     6918848 :     CHECK_FOR_INTERRUPTS();
     664             : 
     665             :     /*
     666             :      * If the scan direction is changing, reset the prefetch block to the
     667             :      * current block. Otherwise, we will incorrectly prefetch the blocks
     668             :      * between the prefetch block and the current block again before
     669             :      * prefetching blocks in the new, correct scan direction.
     670             :      */
     671     6918842 :     if (unlikely(scan->rs_dir != dir))
     672             :     {
     673         154 :         scan->rs_prefetch_block = scan->rs_cblock;
     674         154 :         read_stream_reset(scan->rs_read_stream);
     675             :     }
     676             : 
     677     6918842 :     scan->rs_dir = dir;
     678             : 
     679     6918842 :     scan->rs_cbuf = read_stream_next_buffer(scan->rs_read_stream, NULL);
     680     6918830 :     if (BufferIsValid(scan->rs_cbuf))
     681     5384878 :         scan->rs_cblock = BufferGetBlockNumber(scan->rs_cbuf);
     682     6918830 : }
     683             : 
     684             : /*
     685             :  * heapgettup_initial_block - return the first BlockNumber to scan
     686             :  *
     687             :  * Returns InvalidBlockNumber when there are no blocks to scan.  This can
     688             :  * occur with empty tables and in parallel scans when parallel workers get all
     689             :  * of the pages before we can get a chance to get our first page.
     690             :  */
     691             : static pg_noinline BlockNumber
     692     1839116 : heapgettup_initial_block(HeapScanDesc scan, ScanDirection dir)
     693             : {
     694             :     Assert(!scan->rs_inited);
     695             :     Assert(scan->rs_base.rs_parallel == NULL);
     696             : 
     697             :     /* When there are no pages to scan, return InvalidBlockNumber */
     698     1839116 :     if (scan->rs_nblocks == 0 || scan->rs_numblocks == 0)
     699      891356 :         return InvalidBlockNumber;
     700             : 
     701      947760 :     if (ScanDirectionIsForward(dir))
     702             :     {
     703      947696 :         return scan->rs_startblock;
     704             :     }
     705             :     else
     706             :     {
     707             :         /*
     708             :          * Disable reporting to syncscan logic in a backwards scan; it's not
     709             :          * very likely anyone else is doing the same thing at the same time,
     710             :          * and much more likely that we'll just bollix things for forward
     711             :          * scanners.
     712             :          */
     713          64 :         scan->rs_base.rs_flags &= ~SO_ALLOW_SYNC;
     714             : 
     715             :         /*
     716             :          * Start from last page of the scan.  Ensure we take into account
     717             :          * rs_numblocks if it's been adjusted by heap_setscanlimits().
     718             :          */
     719          64 :         if (scan->rs_numblocks != InvalidBlockNumber)
     720           6 :             return (scan->rs_startblock + scan->rs_numblocks - 1) % scan->rs_nblocks;
     721             : 
     722          58 :         if (scan->rs_startblock > 0)
     723           0 :             return scan->rs_startblock - 1;
     724             : 
     725          58 :         return scan->rs_nblocks - 1;
     726             :     }
     727             : }
     728             : 
     729             : 
     730             : /*
     731             :  * heapgettup_start_page - helper function for heapgettup()
     732             :  *
     733             :  * Return the next page to scan based on the scan->rs_cbuf and set *linesleft
     734             :  * to the number of tuples on this page.  Also set *lineoff to the first
     735             :  * offset to scan with forward scans getting the first offset and backward
     736             :  * getting the final offset on the page.
     737             :  */
     738             : static Page
     739      189042 : heapgettup_start_page(HeapScanDesc scan, ScanDirection dir, int *linesleft,
     740             :                       OffsetNumber *lineoff)
     741             : {
     742             :     Page        page;
     743             : 
     744             :     Assert(scan->rs_inited);
     745             :     Assert(BufferIsValid(scan->rs_cbuf));
     746             : 
     747             :     /* Caller is responsible for ensuring buffer is locked if needed */
     748      189042 :     page = BufferGetPage(scan->rs_cbuf);
     749             : 
     750      189042 :     *linesleft = PageGetMaxOffsetNumber(page) - FirstOffsetNumber + 1;
     751             : 
     752      189042 :     if (ScanDirectionIsForward(dir))
     753      189042 :         *lineoff = FirstOffsetNumber;
     754             :     else
     755           0 :         *lineoff = (OffsetNumber) (*linesleft);
     756             : 
     757             :     /* lineoff now references the physically previous or next tid */
     758      189042 :     return page;
     759             : }
     760             : 
     761             : 
     762             : /*
     763             :  * heapgettup_continue_page - helper function for heapgettup()
     764             :  *
     765             :  * Return the next page to scan based on the scan->rs_cbuf and set *linesleft
     766             :  * to the number of tuples left to scan on this page.  Also set *lineoff to
     767             :  * the next offset to scan according to the ScanDirection in 'dir'.
     768             :  */
     769             : static inline Page
     770    15655998 : heapgettup_continue_page(HeapScanDesc scan, ScanDirection dir, int *linesleft,
     771             :                          OffsetNumber *lineoff)
     772             : {
     773             :     Page        page;
     774             : 
     775             :     Assert(scan->rs_inited);
     776             :     Assert(BufferIsValid(scan->rs_cbuf));
     777             : 
     778             :     /* Caller is responsible for ensuring buffer is locked if needed */
     779    15655998 :     page = BufferGetPage(scan->rs_cbuf);
     780             : 
     781    15655998 :     if (ScanDirectionIsForward(dir))
     782             :     {
     783    15655998 :         *lineoff = OffsetNumberNext(scan->rs_coffset);
     784    15655998 :         *linesleft = PageGetMaxOffsetNumber(page) - (*lineoff) + 1;
     785             :     }
     786             :     else
     787             :     {
     788             :         /*
     789             :          * The previous returned tuple may have been vacuumed since the
     790             :          * previous scan when we use a non-MVCC snapshot, so we must
     791             :          * re-establish the lineoff <= PageGetMaxOffsetNumber(page) invariant
     792             :          */
     793           0 :         *lineoff = Min(PageGetMaxOffsetNumber(page), OffsetNumberPrev(scan->rs_coffset));
     794           0 :         *linesleft = *lineoff;
     795             :     }
     796             : 
     797             :     /* lineoff now references the physically previous or next tid */
     798    15655998 :     return page;
     799             : }
     800             : 
     801             : /*
     802             :  * heapgettup_advance_block - helper for heap_fetch_next_buffer()
     803             :  *
     804             :  * Given the current block number, the scan direction, and various information
     805             :  * contained in the scan descriptor, calculate the BlockNumber to scan next
     806             :  * and return it.  If there are no further blocks to scan, return
     807             :  * InvalidBlockNumber to indicate this fact to the caller.
     808             :  *
     809             :  * This should not be called to determine the initial block number -- only for
     810             :  * subsequent blocks.
     811             :  *
     812             :  * This also adjusts rs_numblocks when a limit has been imposed by
     813             :  * heap_setscanlimits().
     814             :  */
     815             : static inline BlockNumber
     816     5227808 : heapgettup_advance_block(HeapScanDesc scan, BlockNumber block, ScanDirection dir)
     817             : {
     818             :     Assert(scan->rs_base.rs_parallel == NULL);
     819             : 
     820     5227808 :     if (likely(ScanDirectionIsForward(dir)))
     821             :     {
     822     5227690 :         block++;
     823             : 
     824             :         /* wrap back to the start of the heap */
     825     5227690 :         if (block >= scan->rs_nblocks)
     826      762018 :             block = 0;
     827             : 
     828             :         /*
     829             :          * Report our new scan position for synchronization purposes. We don't
     830             :          * do that when moving backwards, however. That would just mess up any
     831             :          * other forward-moving scanners.
     832             :          *
     833             :          * Note: we do this before checking for end of scan so that the final
     834             :          * state of the position hint is back at the start of the rel.  That's
     835             :          * not strictly necessary, but otherwise when you run the same query
     836             :          * multiple times the starting position would shift a little bit
     837             :          * backwards on every invocation, which is confusing. We don't
     838             :          * guarantee any specific ordering in general, though.
     839             :          */
     840     5227690 :         if (scan->rs_base.rs_flags & SO_ALLOW_SYNC)
     841       17574 :             ss_report_location(scan->rs_base.rs_rd, block);
     842             : 
     843             :         /* we're done if we're back at where we started */
     844     5227690 :         if (block == scan->rs_startblock)
     845      761936 :             return InvalidBlockNumber;
     846             : 
     847             :         /* check if the limit imposed by heap_setscanlimits() is met */
     848     4465754 :         if (scan->rs_numblocks != InvalidBlockNumber)
     849             :         {
     850        4924 :             if (--scan->rs_numblocks == 0)
     851        3056 :                 return InvalidBlockNumber;
     852             :         }
     853             : 
     854     4462698 :         return block;
     855             :     }
     856             :     else
     857             :     {
     858             :         /* we're done if the last block is the start position */
     859         118 :         if (block == scan->rs_startblock)
     860         118 :             return InvalidBlockNumber;
     861             : 
     862             :         /* check if the limit imposed by heap_setscanlimits() is met */
     863           0 :         if (scan->rs_numblocks != InvalidBlockNumber)
     864             :         {
     865           0 :             if (--scan->rs_numblocks == 0)
     866           0 :                 return InvalidBlockNumber;
     867             :         }
     868             : 
     869             :         /* wrap to the end of the heap when the last page was page 0 */
     870           0 :         if (block == 0)
     871           0 :             block = scan->rs_nblocks;
     872             : 
     873           0 :         block--;
     874             : 
     875           0 :         return block;
     876             :     }
     877             : }
     878             : 
     879             : /* ----------------
     880             :  *      heapgettup - fetch next heap tuple
     881             :  *
     882             :  *      Initialize the scan if not already done; then advance to the next
     883             :  *      tuple as indicated by "dir"; return the next tuple in scan->rs_ctup,
     884             :  *      or set scan->rs_ctup.t_data = NULL if no more tuples.
     885             :  *
     886             :  * Note: the reason nkeys/key are passed separately, even though they are
     887             :  * kept in the scan descriptor, is that the caller may not want us to check
     888             :  * the scankeys.
     889             :  *
     890             :  * Note: when we fall off the end of the scan in either direction, we
     891             :  * reset rs_inited.  This means that a further request with the same
     892             :  * scan direction will restart the scan, which is a bit odd, but a
     893             :  * request with the opposite scan direction will start a fresh scan
     894             :  * in the proper direction.  The latter is required behavior for cursors,
     895             :  * while the former case is generally undefined behavior in Postgres
     896             :  * so we don't care too much.
     897             :  * ----------------
     898             :  */
     899             : static void
     900    15696076 : heapgettup(HeapScanDesc scan,
     901             :            ScanDirection dir,
     902             :            int nkeys,
     903             :            ScanKey key)
     904             : {
     905    15696076 :     HeapTuple   tuple = &(scan->rs_ctup);
     906             :     Page        page;
     907             :     OffsetNumber lineoff;
     908             :     int         linesleft;
     909             : 
     910    15696076 :     if (likely(scan->rs_inited))
     911             :     {
     912             :         /* continue from previously returned page/tuple */
     913    15655998 :         LockBuffer(scan->rs_cbuf, BUFFER_LOCK_SHARE);
     914    15655998 :         page = heapgettup_continue_page(scan, dir, &linesleft, &lineoff);
     915    15655998 :         goto continue_page;
     916             :     }
     917             : 
     918             :     /*
     919             :      * advance the scan until we find a qualifying tuple or run out of stuff
     920             :      * to scan
     921             :      */
     922             :     while (true)
     923             :     {
     924      228820 :         heap_fetch_next_buffer(scan, dir);
     925             : 
     926             :         /* did we run out of blocks to scan? */
     927      228820 :         if (!BufferIsValid(scan->rs_cbuf))
     928       39778 :             break;
     929             : 
     930             :         Assert(BufferGetBlockNumber(scan->rs_cbuf) == scan->rs_cblock);
     931             : 
     932      189042 :         LockBuffer(scan->rs_cbuf, BUFFER_LOCK_SHARE);
     933      189042 :         page = heapgettup_start_page(scan, dir, &linesleft, &lineoff);
     934    15845040 : continue_page:
     935             : 
     936             :         /*
     937             :          * Only continue scanning the page while we have lines left.
     938             :          *
     939             :          * Note that this protects us from accessing line pointers past
     940             :          * PageGetMaxOffsetNumber(); both for forward scans when we resume the
     941             :          * table scan, and for when we start scanning a new page.
     942             :          */
     943    15928798 :         for (; linesleft > 0; linesleft--, lineoff += dir)
     944             :         {
     945             :             bool        visible;
     946    15740056 :             ItemId      lpp = PageGetItemId(page, lineoff);
     947             : 
     948    15740056 :             if (!ItemIdIsNormal(lpp))
     949       73302 :                 continue;
     950             : 
     951    15666754 :             tuple->t_data = (HeapTupleHeader) PageGetItem(page, lpp);
     952    15666754 :             tuple->t_len = ItemIdGetLength(lpp);
     953    15666754 :             ItemPointerSet(&(tuple->t_self), scan->rs_cblock, lineoff);
     954             : 
     955    15666754 :             visible = HeapTupleSatisfiesVisibility(tuple,
     956             :                                                    scan->rs_base.rs_snapshot,
     957             :                                                    scan->rs_cbuf);
     958             : 
     959    15666754 :             HeapCheckForSerializableConflictOut(visible, scan->rs_base.rs_rd,
     960             :                                                 tuple, scan->rs_cbuf,
     961             :                                                 scan->rs_base.rs_snapshot);
     962             : 
     963             :             /* skip tuples not visible to this snapshot */
     964    15666754 :             if (!visible)
     965       10456 :                 continue;
     966             : 
     967             :             /* skip any tuples that don't match the scan key */
     968    15656298 :             if (key != NULL &&
     969           0 :                 !HeapKeyTest(tuple, RelationGetDescr(scan->rs_base.rs_rd),
     970             :                              nkeys, key))
     971           0 :                 continue;
     972             : 
     973    15656298 :             LockBuffer(scan->rs_cbuf, BUFFER_LOCK_UNLOCK);
     974    15656298 :             scan->rs_coffset = lineoff;
     975    15656298 :             return;
     976             :         }
     977             : 
     978             :         /*
     979             :          * if we get here, it means we've exhausted the items on this page and
     980             :          * it's time to move to the next.
     981             :          */
     982      188742 :         LockBuffer(scan->rs_cbuf, BUFFER_LOCK_UNLOCK);
     983             :     }
     984             : 
     985             :     /* end of scan */
     986       39778 :     if (BufferIsValid(scan->rs_cbuf))
     987           0 :         ReleaseBuffer(scan->rs_cbuf);
     988             : 
     989       39778 :     scan->rs_cbuf = InvalidBuffer;
     990       39778 :     scan->rs_cblock = InvalidBlockNumber;
     991       39778 :     scan->rs_prefetch_block = InvalidBlockNumber;
     992       39778 :     tuple->t_data = NULL;
     993       39778 :     scan->rs_inited = false;
     994             : }
     995             : 
     996             : /* ----------------
     997             :  *      heapgettup_pagemode - fetch next heap tuple in page-at-a-time mode
     998             :  *
     999             :  *      Same API as heapgettup, but used in page-at-a-time mode
    1000             :  *
    1001             :  * The internal logic is much the same as heapgettup's too, but there are some
    1002             :  * differences: we do not take the buffer content lock (that only needs to
    1003             :  * happen inside heap_prepare_pagescan), and we iterate through just the
    1004             :  * tuples listed in rs_vistuples[] rather than all tuples on the page.  Notice
    1005             :  * that lineindex is 0-based, where the corresponding loop variable lineoff in
    1006             :  * heapgettup is 1-based.
    1007             :  * ----------------
    1008             :  */
    1009             : static void
    1010    93098646 : heapgettup_pagemode(HeapScanDesc scan,
    1011             :                     ScanDirection dir,
    1012             :                     int nkeys,
    1013             :                     ScanKey key)
    1014             : {
    1015    93098646 :     HeapTuple   tuple = &(scan->rs_ctup);
    1016             :     Page        page;
    1017             :     uint32      lineindex;
    1018             :     uint32      linesleft;
    1019             : 
    1020    93098646 :     if (likely(scan->rs_inited))
    1021             :     {
    1022             :         /* continue from previously returned page/tuple */
    1023    91296768 :         page = BufferGetPage(scan->rs_cbuf);
    1024             : 
    1025    91296768 :         lineindex = scan->rs_cindex + dir;
    1026    91296768 :         if (ScanDirectionIsForward(dir))
    1027    91296110 :             linesleft = scan->rs_ntuples - lineindex;
    1028             :         else
    1029         658 :             linesleft = scan->rs_cindex;
    1030             :         /* lineindex now references the next or previous visible tid */
    1031             : 
    1032    91296768 :         goto continue_page;
    1033             :     }
    1034             : 
    1035             :     /*
    1036             :      * advance the scan until we find a qualifying tuple or run out of stuff
    1037             :      * to scan
    1038             :      */
    1039             :     while (true)
    1040             :     {
    1041     6690028 :         heap_fetch_next_buffer(scan, dir);
    1042             : 
    1043             :         /* did we run out of blocks to scan? */
    1044     6690010 :         if (!BufferIsValid(scan->rs_cbuf))
    1045     1494174 :             break;
    1046             : 
    1047             :         Assert(BufferGetBlockNumber(scan->rs_cbuf) == scan->rs_cblock);
    1048             : 
    1049             :         /* prune the page and determine visible tuple offsets */
    1050     5195836 :         heap_prepare_pagescan((TableScanDesc) scan);
    1051     5195820 :         page = BufferGetPage(scan->rs_cbuf);
    1052     5195820 :         linesleft = scan->rs_ntuples;
    1053     5195820 :         lineindex = ScanDirectionIsForward(dir) ? 0 : linesleft - 1;
    1054             : 
    1055             :         /* block is the same for all tuples, set it once outside the loop */
    1056     5195820 :         ItemPointerSetBlockNumber(&tuple->t_self, scan->rs_cblock);
    1057             : 
    1058             :         /* lineindex now references the next or previous visible tid */
    1059    96492588 : continue_page:
    1060             : 
    1061   183854540 :         for (; linesleft > 0; linesleft--, lineindex += dir)
    1062             :         {
    1063             :             ItemId      lpp;
    1064             :             OffsetNumber lineoff;
    1065             : 
    1066             :             Assert(lineindex <= scan->rs_ntuples);
    1067   178966390 :             lineoff = scan->rs_vistuples[lineindex];
    1068   178966390 :             lpp = PageGetItemId(page, lineoff);
    1069             :             Assert(ItemIdIsNormal(lpp));
    1070             : 
    1071   178966390 :             tuple->t_data = (HeapTupleHeader) PageGetItem(page, lpp);
    1072   178966390 :             tuple->t_len = ItemIdGetLength(lpp);
    1073   178966390 :             ItemPointerSetOffsetNumber(&tuple->t_self, lineoff);
    1074             : 
    1075             :             /* skip any tuples that don't match the scan key */
    1076   178966390 :             if (key != NULL &&
    1077    88055254 :                 !HeapKeyTest(tuple, RelationGetDescr(scan->rs_base.rs_rd),
    1078             :                              nkeys, key))
    1079    87361952 :                 continue;
    1080             : 
    1081    91604438 :             scan->rs_cindex = lineindex;
    1082    91604438 :             return;
    1083             :         }
    1084             :     }
    1085             : 
    1086             :     /* end of scan */
    1087     1494174 :     if (BufferIsValid(scan->rs_cbuf))
    1088           0 :         ReleaseBuffer(scan->rs_cbuf);
    1089     1494174 :     scan->rs_cbuf = InvalidBuffer;
    1090     1494174 :     scan->rs_cblock = InvalidBlockNumber;
    1091     1494174 :     scan->rs_prefetch_block = InvalidBlockNumber;
    1092     1494174 :     tuple->t_data = NULL;
    1093     1494174 :     scan->rs_inited = false;
    1094             : }
    1095             : 
    1096             : 
    1097             : /* ----------------------------------------------------------------
    1098             :  *                   heap access method interface
    1099             :  * ----------------------------------------------------------------
    1100             :  */
    1101             : 
    1102             : 
    1103             : TableScanDesc
    1104      719306 : heap_beginscan(Relation relation, Snapshot snapshot,
    1105             :                int nkeys, ScanKey key,
    1106             :                ParallelTableScanDesc parallel_scan,
    1107             :                uint32 flags)
    1108             : {
    1109             :     HeapScanDesc scan;
    1110             : 
    1111             :     /*
    1112             :      * increment relation ref count while scanning relation
    1113             :      *
    1114             :      * This is just to make really sure the relcache entry won't go away while
    1115             :      * the scan has a pointer to it.  Caller should be holding the rel open
    1116             :      * anyway, so this is redundant in all normal scenarios...
    1117             :      */
    1118      719306 :     RelationIncrementReferenceCount(relation);
    1119             : 
    1120             :     /*
    1121             :      * allocate and initialize scan descriptor
    1122             :      */
    1123      719306 :     if (flags & SO_TYPE_BITMAPSCAN)
    1124             :     {
    1125       21346 :         BitmapHeapScanDesc bscan = palloc(sizeof(BitmapHeapScanDescData));
    1126             : 
    1127       21346 :         bscan->rs_vmbuffer = InvalidBuffer;
    1128       21346 :         bscan->rs_empty_tuples_pending = 0;
    1129       21346 :         scan = (HeapScanDesc) bscan;
    1130             :     }
    1131             :     else
    1132      697960 :         scan = (HeapScanDesc) palloc(sizeof(HeapScanDescData));
    1133             : 
    1134      719306 :     scan->rs_base.rs_rd = relation;
    1135      719306 :     scan->rs_base.rs_snapshot = snapshot;
    1136      719306 :     scan->rs_base.rs_nkeys = nkeys;
    1137      719306 :     scan->rs_base.rs_flags = flags;
    1138      719306 :     scan->rs_base.rs_parallel = parallel_scan;
    1139      719306 :     scan->rs_strategy = NULL;    /* set in initscan */
    1140      719306 :     scan->rs_cbuf = InvalidBuffer;
    1141             : 
    1142             :     /*
    1143             :      * Disable page-at-a-time mode if it's not a MVCC-safe snapshot.
    1144             :      */
    1145      719306 :     if (!(snapshot && IsMVCCSnapshot(snapshot)))
    1146       56270 :         scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE;
    1147             : 
    1148             :     /*
    1149             :      * For seqscan and sample scans in a serializable transaction, acquire a
    1150             :      * predicate lock on the entire relation. This is required not only to
    1151             :      * lock all the matching tuples, but also to conflict with new insertions
    1152             :      * into the table. In an indexscan, we take page locks on the index pages
    1153             :      * covering the range specified in the scan qual, but in a heap scan there
    1154             :      * is nothing more fine-grained to lock. A bitmap scan is a different
    1155             :      * story, there we have already scanned the index and locked the index
    1156             :      * pages covering the predicate. But in that case we still have to lock
    1157             :      * any matching heap tuples. For sample scan we could optimize the locking
    1158             :      * to be at least page-level granularity, but we'd need to add per-tuple
    1159             :      * locking for that.
    1160             :      */
    1161      719306 :     if (scan->rs_base.rs_flags & (SO_TYPE_SEQSCAN | SO_TYPE_SAMPLESCAN))
    1162             :     {
    1163             :         /*
    1164             :          * Ensure a missing snapshot is noticed reliably, even if the
    1165             :          * isolation mode means predicate locking isn't performed (and
    1166             :          * therefore the snapshot isn't used here).
    1167             :          */
    1168             :         Assert(snapshot);
    1169      679196 :         PredicateLockRelation(relation, snapshot);
    1170             :     }
    1171             : 
    1172             :     /* we only need to set this up once */
    1173      719306 :     scan->rs_ctup.t_tableOid = RelationGetRelid(relation);
    1174             : 
    1175             :     /*
    1176             :      * Allocate memory to keep track of page allocation for parallel workers
    1177             :      * when doing a parallel scan.
    1178             :      */
    1179      719306 :     if (parallel_scan != NULL)
    1180        3916 :         scan->rs_parallelworkerdata = palloc(sizeof(ParallelBlockTableScanWorkerData));
    1181             :     else
    1182      715390 :         scan->rs_parallelworkerdata = NULL;
    1183             : 
    1184             :     /*
    1185             :      * we do this here instead of in initscan() because heap_rescan also calls
    1186             :      * initscan() and we don't want to allocate memory again
    1187             :      */
    1188      719306 :     if (nkeys > 0)
    1189      405734 :         scan->rs_base.rs_key = (ScanKey) palloc(sizeof(ScanKeyData) * nkeys);
    1190             :     else
    1191      313572 :         scan->rs_base.rs_key = NULL;
    1192             : 
    1193      719306 :     initscan(scan, key, false);
    1194             : 
    1195      719302 :     scan->rs_read_stream = NULL;
    1196             : 
    1197             :     /*
    1198             :      * Set up a read stream for sequential scans and TID range scans. This
    1199             :      * should be done after initscan() because initscan() allocates the
    1200             :      * BufferAccessStrategy object passed to the read stream API.
    1201             :      */
    1202      719302 :     if (scan->rs_base.rs_flags & SO_TYPE_SEQSCAN ||
    1203       40256 :         scan->rs_base.rs_flags & SO_TYPE_TIDRANGESCAN)
    1204      680892 :     {
    1205             :         ReadStreamBlockNumberCB cb;
    1206             : 
    1207      680892 :         if (scan->rs_base.rs_parallel)
    1208        3916 :             cb = heap_scan_stream_read_next_parallel;
    1209             :         else
    1210      676976 :             cb = heap_scan_stream_read_next_serial;
    1211             : 
    1212             :         /* ---
    1213             :          * It is safe to use batchmode as the only locks taken by `cb`
    1214             :          * are never taken while waiting for IO:
    1215             :          * - SyncScanLock is used in the non-parallel case
    1216             :          * - in the parallel case, only spinlocks and atomics are used
    1217             :          * ---
    1218             :          */
    1219      680892 :         scan->rs_read_stream = read_stream_begin_relation(READ_STREAM_SEQUENTIAL |
    1220             :                                                           READ_STREAM_USE_BATCHING,
    1221             :                                                           scan->rs_strategy,
    1222             :                                                           scan->rs_base.rs_rd,
    1223             :                                                           MAIN_FORKNUM,
    1224             :                                                           cb,
    1225             :                                                           scan,
    1226             :                                                           0);
    1227             :     }
    1228       38410 :     else if (scan->rs_base.rs_flags & SO_TYPE_BITMAPSCAN)
    1229             :     {
    1230             :         /*
    1231             :          * Currently we can't trivially use batching, due to the
    1232             :          * VM_ALL_VISIBLE check in bitmapheap_stream_read_next. While that
    1233             :          * could be made safe, we are about to remove the all-visible logic
    1234             :          * from bitmap scans due to its unsoundness.
    1235             :          */
    1236       21346 :         scan->rs_read_stream = read_stream_begin_relation(READ_STREAM_DEFAULT,
    1237             :                                                           scan->rs_strategy,
    1238             :                                                           scan->rs_base.rs_rd,
    1239             :                                                           MAIN_FORKNUM,
    1240             :                                                           bitmapheap_stream_read_next,
    1241             :                                                           scan,
    1242             :                                                           sizeof(TBMIterateResult));
    1243             :     }
    1244             : 
    1245             : 
    1246      719302 :     return (TableScanDesc) scan;
    1247             : }
    1248             : 
    1249             : void
    1250     1166774 : heap_rescan(TableScanDesc sscan, ScanKey key, bool set_params,
    1251             :             bool allow_strat, bool allow_sync, bool allow_pagemode)
    1252             : {
    1253     1166774 :     HeapScanDesc scan = (HeapScanDesc) sscan;
    1254             : 
    1255     1166774 :     if (set_params)
    1256             :     {
    1257          30 :         if (allow_strat)
    1258          30 :             scan->rs_base.rs_flags |= SO_ALLOW_STRAT;
    1259             :         else
    1260           0 :             scan->rs_base.rs_flags &= ~SO_ALLOW_STRAT;
    1261             : 
    1262          30 :         if (allow_sync)
    1263          12 :             scan->rs_base.rs_flags |= SO_ALLOW_SYNC;
    1264             :         else
    1265          18 :             scan->rs_base.rs_flags &= ~SO_ALLOW_SYNC;
    1266             : 
    1267          30 :         if (allow_pagemode && scan->rs_base.rs_snapshot &&
    1268          30 :             IsMVCCSnapshot(scan->rs_base.rs_snapshot))
    1269          30 :             scan->rs_base.rs_flags |= SO_ALLOW_PAGEMODE;
    1270             :         else
    1271           0 :             scan->rs_base.rs_flags &= ~SO_ALLOW_PAGEMODE;
    1272             :     }
    1273             : 
    1274             :     /*
    1275             :      * unpin scan buffers
    1276             :      */
    1277     1166774 :     if (BufferIsValid(scan->rs_cbuf))
    1278             :     {
    1279        7272 :         ReleaseBuffer(scan->rs_cbuf);
    1280        7272 :         scan->rs_cbuf = InvalidBuffer;
    1281             :     }
    1282             : 
    1283     1166774 :     if (scan->rs_base.rs_flags & SO_TYPE_BITMAPSCAN)
    1284             :     {
    1285        4246 :         BitmapHeapScanDesc bscan = (BitmapHeapScanDesc) scan;
    1286             : 
    1287             :         /*
    1288             :          * Reset empty_tuples_pending, a field only used by bitmap heap scan,
    1289             :          * to avoid incorrectly emitting NULL-filled tuples from a previous
    1290             :          * scan on rescan.
    1291             :          */
    1292        4246 :         bscan->rs_empty_tuples_pending = 0;
    1293             : 
    1294        4246 :         if (BufferIsValid(bscan->rs_vmbuffer))
    1295             :         {
    1296           0 :             ReleaseBuffer(bscan->rs_vmbuffer);
    1297           0 :             bscan->rs_vmbuffer = InvalidBuffer;
    1298             :         }
    1299             :     }
    1300             : 
    1301             :     /*
    1302             :      * The read stream is reset on rescan. This must be done before
    1303             :      * initscan(), as some state referred to by read_stream_reset() is reset
    1304             :      * in initscan().
    1305             :      */
    1306     1166774 :     if (scan->rs_read_stream)
    1307     1166738 :         read_stream_reset(scan->rs_read_stream);
    1308             : 
    1309             :     /*
    1310             :      * reinitialize scan descriptor
    1311             :      */
    1312     1166774 :     initscan(scan, key, true);
    1313     1166774 : }
    1314             : 
    1315             : void
    1316      714764 : heap_endscan(TableScanDesc sscan)
    1317             : {
    1318      714764 :     HeapScanDesc scan = (HeapScanDesc) sscan;
    1319             : 
    1320             :     /* Note: no locking manipulations needed */
    1321             : 
    1322             :     /*
    1323             :      * unpin scan buffers
    1324             :      */
    1325      714764 :     if (BufferIsValid(scan->rs_cbuf))
    1326      297556 :         ReleaseBuffer(scan->rs_cbuf);
    1327             : 
    1328      714764 :     if (scan->rs_base.rs_flags & SO_TYPE_BITMAPSCAN)
    1329             :     {
    1330       21244 :         BitmapHeapScanDesc bscan = (BitmapHeapScanDesc) sscan;
    1331             : 
    1332       21244 :         bscan->rs_empty_tuples_pending = 0;
    1333       21244 :         if (BufferIsValid(bscan->rs_vmbuffer))
    1334           0 :             ReleaseBuffer(bscan->rs_vmbuffer);
    1335             :     }
    1336             : 
    1337             :     /*
    1338             :      * Must free the read stream before freeing the BufferAccessStrategy.
    1339             :      */
    1340      714764 :     if (scan->rs_read_stream)
    1341      697790 :         read_stream_end(scan->rs_read_stream);
    1342             : 
    1343             :     /*
    1344             :      * decrement relation reference count and free scan descriptor storage
    1345             :      */
    1346      714764 :     RelationDecrementReferenceCount(scan->rs_base.rs_rd);
    1347             : 
    1348      714764 :     if (scan->rs_base.rs_key)
    1349      405674 :         pfree(scan->rs_base.rs_key);
    1350             : 
    1351      714764 :     if (scan->rs_strategy != NULL)
    1352       21498 :         FreeAccessStrategy(scan->rs_strategy);
    1353             : 
    1354      714764 :     if (scan->rs_parallelworkerdata != NULL)
    1355        3916 :         pfree(scan->rs_parallelworkerdata);
    1356             : 
    1357      714764 :     if (scan->rs_base.rs_flags & SO_TEMP_SNAPSHOT)
    1358       70756 :         UnregisterSnapshot(scan->rs_base.rs_snapshot);
    1359             : 
    1360      714764 :     pfree(scan);
    1361      714764 : }
    1362             : 
    1363             : HeapTuple
    1364    18925336 : heap_getnext(TableScanDesc sscan, ScanDirection direction)
    1365             : {
    1366    18925336 :     HeapScanDesc scan = (HeapScanDesc) sscan;
    1367             : 
    1368             :     /*
    1369             :      * This is still widely used directly, without going through table AM, so
    1370             :      * add a safety check.  It's possible we should, at a later point,
    1371             :      * downgrade this to an assert. The reason for checking the AM routine,
    1372             :      * rather than the AM oid, is that this allows to write regression tests
    1373             :      * that create another AM reusing the heap handler.
    1374             :      */
    1375    18925336 :     if (unlikely(sscan->rs_rd->rd_tableam != GetHeapamTableAmRoutine()))
    1376           0 :         ereport(ERROR,
    1377             :                 (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
    1378             :                  errmsg_internal("only heap AM is supported")));
    1379             : 
    1380             :     /*
    1381             :      * We don't expect direct calls to heap_getnext with valid CheckXidAlive
    1382             :      * for catalog or regular tables.  See detailed comments in xact.c where
    1383             :      * these variables are declared.  Normally we have such a check at tableam
    1384             :      * level API but this is called from many places so we need to ensure it
    1385             :      * here.
    1386             :      */
    1387    18925336 :     if (unlikely(TransactionIdIsValid(CheckXidAlive) && !bsysscan))
    1388           0 :         elog(ERROR, "unexpected heap_getnext call during logical decoding");
    1389             : 
    1390             :     /* Note: no locking manipulations needed */
    1391             : 
    1392    18925336 :     if (scan->rs_base.rs_flags & SO_ALLOW_PAGEMODE)
    1393     4220838 :         heapgettup_pagemode(scan, direction,
    1394     4220838 :                             scan->rs_base.rs_nkeys, scan->rs_base.rs_key);
    1395             :     else
    1396    14704498 :         heapgettup(scan, direction,
    1397    14704498 :                    scan->rs_base.rs_nkeys, scan->rs_base.rs_key);
    1398             : 
    1399    18925336 :     if (scan->rs_ctup.t_data == NULL)
    1400      121112 :         return NULL;
    1401             : 
    1402             :     /*
    1403             :      * if we get here it means we have a new current scan tuple, so point to
    1404             :      * the proper return buffer and return the tuple.
    1405             :      */
    1406             : 
    1407    18804224 :     pgstat_count_heap_getnext(scan->rs_base.rs_rd);
    1408             : 
    1409    18804224 :     return &scan->rs_ctup;
    1410             : }
    1411             : 
    1412             : bool
    1413    89861514 : heap_getnextslot(TableScanDesc sscan, ScanDirection direction, TupleTableSlot *slot)
    1414             : {
    1415    89861514 :     HeapScanDesc scan = (HeapScanDesc) sscan;
    1416             : 
    1417             :     /* Note: no locking manipulations needed */
    1418             : 
    1419    89861514 :     if (sscan->rs_flags & SO_ALLOW_PAGEMODE)
    1420    88869936 :         heapgettup_pagemode(scan, direction, sscan->rs_nkeys, sscan->rs_key);
    1421             :     else
    1422      991578 :         heapgettup(scan, direction, sscan->rs_nkeys, sscan->rs_key);
    1423             : 
    1424    89861492 :     if (scan->rs_ctup.t_data == NULL)
    1425             :     {
    1426     1412746 :         ExecClearTuple(slot);
    1427     1412746 :         return false;
    1428             :     }
    1429             : 
    1430             :     /*
    1431             :      * if we get here it means we have a new current scan tuple, so point to
    1432             :      * the proper return buffer and return the tuple.
    1433             :      */
    1434             : 
    1435    88448746 :     pgstat_count_heap_getnext(scan->rs_base.rs_rd);
    1436             : 
    1437    88448746 :     ExecStoreBufferHeapTuple(&scan->rs_ctup, slot,
    1438             :                              scan->rs_cbuf);
    1439    88448746 :     return true;
    1440             : }
    1441             : 
    1442             : void
    1443        1912 : heap_set_tidrange(TableScanDesc sscan, ItemPointer mintid,
    1444             :                   ItemPointer maxtid)
    1445             : {
    1446        1912 :     HeapScanDesc scan = (HeapScanDesc) sscan;
    1447             :     BlockNumber startBlk;
    1448             :     BlockNumber numBlks;
    1449             :     ItemPointerData highestItem;
    1450             :     ItemPointerData lowestItem;
    1451             : 
    1452             :     /*
    1453             :      * For relations without any pages, we can simply leave the TID range
    1454             :      * unset.  There will be no tuples to scan, therefore no tuples outside
    1455             :      * the given TID range.
    1456             :      */
    1457        1912 :     if (scan->rs_nblocks == 0)
    1458          48 :         return;
    1459             : 
    1460             :     /*
    1461             :      * Set up some ItemPointers which point to the first and last possible
    1462             :      * tuples in the heap.
    1463             :      */
    1464        1900 :     ItemPointerSet(&highestItem, scan->rs_nblocks - 1, MaxOffsetNumber);
    1465        1900 :     ItemPointerSet(&lowestItem, 0, FirstOffsetNumber);
    1466             : 
    1467             :     /*
    1468             :      * If the given maximum TID is below the highest possible TID in the
    1469             :      * relation, then restrict the range to that, otherwise we scan to the end
    1470             :      * of the relation.
    1471             :      */
    1472        1900 :     if (ItemPointerCompare(maxtid, &highestItem) < 0)
    1473         132 :         ItemPointerCopy(maxtid, &highestItem);
    1474             : 
    1475             :     /*
    1476             :      * If the given minimum TID is above the lowest possible TID in the
    1477             :      * relation, then restrict the range to only scan for TIDs above that.
    1478             :      */
    1479        1900 :     if (ItemPointerCompare(mintid, &lowestItem) > 0)
    1480        1762 :         ItemPointerCopy(mintid, &lowestItem);
    1481             : 
    1482             :     /*
    1483             :      * Check for an empty range and protect from would be negative results
    1484             :      * from the numBlks calculation below.
    1485             :      */
    1486        1900 :     if (ItemPointerCompare(&highestItem, &lowestItem) < 0)
    1487             :     {
    1488             :         /* Set an empty range of blocks to scan */
    1489          36 :         heap_setscanlimits(sscan, 0, 0);
    1490          36 :         return;
    1491             :     }
    1492             : 
    1493             :     /*
    1494             :      * Calculate the first block and the number of blocks we must scan. We
    1495             :      * could be more aggressive here and perform some more validation to try
    1496             :      * and further narrow the scope of blocks to scan by checking if the
    1497             :      * lowestItem has an offset above MaxOffsetNumber.  In this case, we could
    1498             :      * advance startBlk by one.  Likewise, if highestItem has an offset of 0
    1499             :      * we could scan one fewer blocks.  However, such an optimization does not
    1500             :      * seem worth troubling over, currently.
    1501             :      */
    1502        1864 :     startBlk = ItemPointerGetBlockNumberNoCheck(&lowestItem);
    1503             : 
    1504        1864 :     numBlks = ItemPointerGetBlockNumberNoCheck(&highestItem) -
    1505        1864 :         ItemPointerGetBlockNumberNoCheck(&lowestItem) + 1;
    1506             : 
    1507             :     /* Set the start block and number of blocks to scan */
    1508        1864 :     heap_setscanlimits(sscan, startBlk, numBlks);
    1509             : 
    1510             :     /* Finally, set the TID range in sscan */
    1511        1864 :     ItemPointerCopy(&lowestItem, &sscan->st.tidrange.rs_mintid);
    1512        1864 :     ItemPointerCopy(&highestItem, &sscan->st.tidrange.rs_maxtid);
    1513             : }
    1514             : 
    1515             : bool
    1516        7686 : heap_getnextslot_tidrange(TableScanDesc sscan, ScanDirection direction,
    1517             :                           TupleTableSlot *slot)
    1518             : {
    1519        7686 :     HeapScanDesc scan = (HeapScanDesc) sscan;
    1520        7686 :     ItemPointer mintid = &sscan->st.tidrange.rs_mintid;
    1521        7686 :     ItemPointer maxtid = &sscan->st.tidrange.rs_maxtid;
    1522             : 
    1523             :     /* Note: no locking manipulations needed */
    1524             :     for (;;)
    1525             :     {
    1526        7872 :         if (sscan->rs_flags & SO_ALLOW_PAGEMODE)
    1527        7872 :             heapgettup_pagemode(scan, direction, sscan->rs_nkeys, sscan->rs_key);
    1528             :         else
    1529           0 :             heapgettup(scan, direction, sscan->rs_nkeys, sscan->rs_key);
    1530             : 
    1531        7860 :         if (scan->rs_ctup.t_data == NULL)
    1532             :         {
    1533          94 :             ExecClearTuple(slot);
    1534          94 :             return false;
    1535             :         }
    1536             : 
    1537             :         /*
    1538             :          * heap_set_tidrange will have used heap_setscanlimits to limit the
    1539             :          * range of pages we scan to only ones that can contain the TID range
    1540             :          * we're scanning for.  Here we must filter out any tuples from these
    1541             :          * pages that are outside of that range.
    1542             :          */
    1543        7766 :         if (ItemPointerCompare(&scan->rs_ctup.t_self, mintid) < 0)
    1544             :         {
    1545         186 :             ExecClearTuple(slot);
    1546             : 
    1547             :             /*
    1548             :              * When scanning backwards, the TIDs will be in descending order.
    1549             :              * Future tuples in this direction will be lower still, so we can
    1550             :              * just return false to indicate there will be no more tuples.
    1551             :              */
    1552         186 :             if (ScanDirectionIsBackward(direction))
    1553           0 :                 return false;
    1554             : 
    1555         186 :             continue;
    1556             :         }
    1557             : 
    1558             :         /*
    1559             :          * Likewise for the final page, we must filter out TIDs greater than
    1560             :          * maxtid.
    1561             :          */
    1562        7580 :         if (ItemPointerCompare(&scan->rs_ctup.t_self, maxtid) > 0)
    1563             :         {
    1564          72 :             ExecClearTuple(slot);
    1565             : 
    1566             :             /*
    1567             :              * When scanning forward, the TIDs will be in ascending order.
    1568             :              * Future tuples in this direction will be higher still, so we can
    1569             :              * just return false to indicate there will be no more tuples.
    1570             :              */
    1571          72 :             if (ScanDirectionIsForward(direction))
    1572          72 :                 return false;
    1573           0 :             continue;
    1574             :         }
    1575             : 
    1576        7508 :         break;
    1577             :     }
    1578             : 
    1579             :     /*
    1580             :      * if we get here it means we have a new current scan tuple, so point to
    1581             :      * the proper return buffer and return the tuple.
    1582             :      */
    1583        7508 :     pgstat_count_heap_getnext(scan->rs_base.rs_rd);
    1584             : 
    1585        7508 :     ExecStoreBufferHeapTuple(&scan->rs_ctup, slot, scan->rs_cbuf);
    1586        7508 :     return true;
    1587             : }
    1588             : 
    1589             : /*
    1590             :  *  heap_fetch      - retrieve tuple with given tid
    1591             :  *
    1592             :  * On entry, tuple->t_self is the TID to fetch.  We pin the buffer holding
    1593             :  * the tuple, fill in the remaining fields of *tuple, and check the tuple
    1594             :  * against the specified snapshot.
    1595             :  *
    1596             :  * If successful (tuple found and passes snapshot time qual), then *userbuf
    1597             :  * is set to the buffer holding the tuple and true is returned.  The caller
    1598             :  * must unpin the buffer when done with the tuple.
    1599             :  *
    1600             :  * If the tuple is not found (ie, item number references a deleted slot),
    1601             :  * then tuple->t_data is set to NULL, *userbuf is set to InvalidBuffer,
    1602             :  * and false is returned.
    1603             :  *
    1604             :  * If the tuple is found but fails the time qual check, then the behavior
    1605             :  * depends on the keep_buf parameter.  If keep_buf is false, the results
    1606             :  * are the same as for the tuple-not-found case.  If keep_buf is true,
    1607             :  * then tuple->t_data and *userbuf are returned as for the success case,
    1608             :  * and again the caller must unpin the buffer; but false is returned.
    1609             :  *
    1610             :  * heap_fetch does not follow HOT chains: only the exact TID requested will
    1611             :  * be fetched.
    1612             :  *
    1613             :  * It is somewhat inconsistent that we ereport() on invalid block number but
    1614             :  * return false on invalid item number.  There are a couple of reasons though.
    1615             :  * One is that the caller can relatively easily check the block number for
    1616             :  * validity, but cannot check the item number without reading the page
    1617             :  * himself.  Another is that when we are following a t_ctid link, we can be
    1618             :  * reasonably confident that the page number is valid (since VACUUM shouldn't
    1619             :  * truncate off the destination page without having killed the referencing
    1620             :  * tuple first), but the item number might well not be good.
    1621             :  */
    1622             : bool
    1623      353378 : heap_fetch(Relation relation,
    1624             :            Snapshot snapshot,
    1625             :            HeapTuple tuple,
    1626             :            Buffer *userbuf,
    1627             :            bool keep_buf)
    1628             : {
    1629      353378 :     ItemPointer tid = &(tuple->t_self);
    1630             :     ItemId      lp;
    1631             :     Buffer      buffer;
    1632             :     Page        page;
    1633             :     OffsetNumber offnum;
    1634             :     bool        valid;
    1635             : 
    1636             :     /*
    1637             :      * Fetch and pin the appropriate page of the relation.
    1638             :      */
    1639      353378 :     buffer = ReadBuffer(relation, ItemPointerGetBlockNumber(tid));
    1640             : 
    1641             :     /*
    1642             :      * Need share lock on buffer to examine tuple commit status.
    1643             :      */
    1644      353378 :     LockBuffer(buffer, BUFFER_LOCK_SHARE);
    1645      353378 :     page = BufferGetPage(buffer);
    1646             : 
    1647             :     /*
    1648             :      * We'd better check for out-of-range offnum in case of VACUUM since the
    1649             :      * TID was obtained.
    1650             :      */
    1651      353378 :     offnum = ItemPointerGetOffsetNumber(tid);
    1652      353378 :     if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page))
    1653             :     {
    1654           6 :         LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    1655           6 :         ReleaseBuffer(buffer);
    1656           6 :         *userbuf = InvalidBuffer;
    1657           6 :         tuple->t_data = NULL;
    1658           6 :         return false;
    1659             :     }
    1660             : 
    1661             :     /*
    1662             :      * get the item line pointer corresponding to the requested tid
    1663             :      */
    1664      353372 :     lp = PageGetItemId(page, offnum);
    1665             : 
    1666             :     /*
    1667             :      * Must check for deleted tuple.
    1668             :      */
    1669      353372 :     if (!ItemIdIsNormal(lp))
    1670             :     {
    1671         650 :         LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    1672         650 :         ReleaseBuffer(buffer);
    1673         650 :         *userbuf = InvalidBuffer;
    1674         650 :         tuple->t_data = NULL;
    1675         650 :         return false;
    1676             :     }
    1677             : 
    1678             :     /*
    1679             :      * fill in *tuple fields
    1680             :      */
    1681      352722 :     tuple->t_data = (HeapTupleHeader) PageGetItem(page, lp);
    1682      352722 :     tuple->t_len = ItemIdGetLength(lp);
    1683      352722 :     tuple->t_tableOid = RelationGetRelid(relation);
    1684             : 
    1685             :     /*
    1686             :      * check tuple visibility, then release lock
    1687             :      */
    1688      352722 :     valid = HeapTupleSatisfiesVisibility(tuple, snapshot, buffer);
    1689             : 
    1690      352722 :     if (valid)
    1691      352632 :         PredicateLockTID(relation, &(tuple->t_self), snapshot,
    1692      352632 :                          HeapTupleHeaderGetXmin(tuple->t_data));
    1693             : 
    1694      352722 :     HeapCheckForSerializableConflictOut(valid, relation, tuple, buffer, snapshot);
    1695             : 
    1696      352722 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    1697             : 
    1698      352722 :     if (valid)
    1699             :     {
    1700             :         /*
    1701             :          * All checks passed, so return the tuple as valid. Caller is now
    1702             :          * responsible for releasing the buffer.
    1703             :          */
    1704      352632 :         *userbuf = buffer;
    1705             : 
    1706      352632 :         return true;
    1707             :     }
    1708             : 
    1709             :     /* Tuple failed time qual, but maybe caller wants to see it anyway. */
    1710          90 :     if (keep_buf)
    1711          58 :         *userbuf = buffer;
    1712             :     else
    1713             :     {
    1714          32 :         ReleaseBuffer(buffer);
    1715          32 :         *userbuf = InvalidBuffer;
    1716          32 :         tuple->t_data = NULL;
    1717             :     }
    1718             : 
    1719          90 :     return false;
    1720             : }
    1721             : 
    1722             : /*
    1723             :  *  heap_hot_search_buffer  - search HOT chain for tuple satisfying snapshot
    1724             :  *
    1725             :  * On entry, *tid is the TID of a tuple (either a simple tuple, or the root
    1726             :  * of a HOT chain), and buffer is the buffer holding this tuple.  We search
    1727             :  * for the first chain member satisfying the given snapshot.  If one is
    1728             :  * found, we update *tid to reference that tuple's offset number, and
    1729             :  * return true.  If no match, return false without modifying *tid.
    1730             :  *
    1731             :  * heapTuple is a caller-supplied buffer.  When a match is found, we return
    1732             :  * the tuple here, in addition to updating *tid.  If no match is found, the
    1733             :  * contents of this buffer on return are undefined.
    1734             :  *
    1735             :  * If all_dead is not NULL, we check non-visible tuples to see if they are
    1736             :  * globally dead; *all_dead is set true if all members of the HOT chain
    1737             :  * are vacuumable, false if not.
    1738             :  *
    1739             :  * Unlike heap_fetch, the caller must already have pin and (at least) share
    1740             :  * lock on the buffer; it is still pinned/locked at exit.
    1741             :  */
    1742             : bool
    1743    41689684 : heap_hot_search_buffer(ItemPointer tid, Relation relation, Buffer buffer,
    1744             :                        Snapshot snapshot, HeapTuple heapTuple,
    1745             :                        bool *all_dead, bool first_call)
    1746             : {
    1747    41689684 :     Page        page = BufferGetPage(buffer);
    1748    41689684 :     TransactionId prev_xmax = InvalidTransactionId;
    1749             :     BlockNumber blkno;
    1750             :     OffsetNumber offnum;
    1751             :     bool        at_chain_start;
    1752             :     bool        valid;
    1753             :     bool        skip;
    1754    41689684 :     GlobalVisState *vistest = NULL;
    1755             : 
    1756             :     /* If this is not the first call, previous call returned a (live!) tuple */
    1757    41689684 :     if (all_dead)
    1758    35839132 :         *all_dead = first_call;
    1759             : 
    1760    41689684 :     blkno = ItemPointerGetBlockNumber(tid);
    1761    41689684 :     offnum = ItemPointerGetOffsetNumber(tid);
    1762    41689684 :     at_chain_start = first_call;
    1763    41689684 :     skip = !first_call;
    1764             : 
    1765             :     /* XXX: we should assert that a snapshot is pushed or registered */
    1766             :     Assert(TransactionIdIsValid(RecentXmin));
    1767             :     Assert(BufferGetBlockNumber(buffer) == blkno);
    1768             : 
    1769             :     /* Scan through possible multiple members of HOT-chain */
    1770             :     for (;;)
    1771     2976110 :     {
    1772             :         ItemId      lp;
    1773             : 
    1774             :         /* check for bogus TID */
    1775    44665794 :         if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page))
    1776             :             break;
    1777             : 
    1778    44665794 :         lp = PageGetItemId(page, offnum);
    1779             : 
    1780             :         /* check for unused, dead, or redirected items */
    1781    44665794 :         if (!ItemIdIsNormal(lp))
    1782             :         {
    1783             :             /* We should only see a redirect at start of chain */
    1784     1655842 :             if (ItemIdIsRedirected(lp) && at_chain_start)
    1785             :             {
    1786             :                 /* Follow the redirect */
    1787      877780 :                 offnum = ItemIdGetRedirect(lp);
    1788      877780 :                 at_chain_start = false;
    1789      877780 :                 continue;
    1790             :             }
    1791             :             /* else must be end of chain */
    1792      778062 :             break;
    1793             :         }
    1794             : 
    1795             :         /*
    1796             :          * Update heapTuple to point to the element of the HOT chain we're
    1797             :          * currently investigating. Having t_self set correctly is important
    1798             :          * because the SSI checks and the *Satisfies routine for historical
    1799             :          * MVCC snapshots need the correct tid to decide about the visibility.
    1800             :          */
    1801    43009952 :         heapTuple->t_data = (HeapTupleHeader) PageGetItem(page, lp);
    1802    43009952 :         heapTuple->t_len = ItemIdGetLength(lp);
    1803    43009952 :         heapTuple->t_tableOid = RelationGetRelid(relation);
    1804    43009952 :         ItemPointerSet(&heapTuple->t_self, blkno, offnum);
    1805             : 
    1806             :         /*
    1807             :          * Shouldn't see a HEAP_ONLY tuple at chain start.
    1808             :          */
    1809    43009952 :         if (at_chain_start && HeapTupleIsHeapOnly(heapTuple))
    1810           0 :             break;
    1811             : 
    1812             :         /*
    1813             :          * The xmin should match the previous xmax value, else chain is
    1814             :          * broken.
    1815             :          */
    1816    45108282 :         if (TransactionIdIsValid(prev_xmax) &&
    1817     2098330 :             !TransactionIdEquals(prev_xmax,
    1818             :                                  HeapTupleHeaderGetXmin(heapTuple->t_data)))
    1819           0 :             break;
    1820             : 
    1821             :         /*
    1822             :          * When first_call is true (and thus, skip is initially false) we'll
    1823             :          * return the first tuple we find.  But on later passes, heapTuple
    1824             :          * will initially be pointing to the tuple we returned last time.
    1825             :          * Returning it again would be incorrect (and would loop forever), so
    1826             :          * we skip it and return the next match we find.
    1827             :          */
    1828    43009952 :         if (!skip)
    1829             :         {
    1830             :             /* If it's visible per the snapshot, we must return it */
    1831    42851814 :             valid = HeapTupleSatisfiesVisibility(heapTuple, snapshot, buffer);
    1832    42851814 :             HeapCheckForSerializableConflictOut(valid, relation, heapTuple,
    1833             :                                                 buffer, snapshot);
    1834             : 
    1835    42851804 :             if (valid)
    1836             :             {
    1837    28564690 :                 ItemPointerSetOffsetNumber(tid, offnum);
    1838    28564690 :                 PredicateLockTID(relation, &heapTuple->t_self, snapshot,
    1839    28564690 :                                  HeapTupleHeaderGetXmin(heapTuple->t_data));
    1840    28564690 :                 if (all_dead)
    1841    23279812 :                     *all_dead = false;
    1842    28564690 :                 return true;
    1843             :             }
    1844             :         }
    1845    14445252 :         skip = false;
    1846             : 
    1847             :         /*
    1848             :          * If we can't see it, maybe no one else can either.  At caller
    1849             :          * request, check whether all chain members are dead to all
    1850             :          * transactions.
    1851             :          *
    1852             :          * Note: if you change the criterion here for what is "dead", fix the
    1853             :          * planner's get_actual_variable_range() function to match.
    1854             :          */
    1855    14445252 :         if (all_dead && *all_dead)
    1856             :         {
    1857    12779070 :             if (!vistest)
    1858    12553490 :                 vistest = GlobalVisTestFor(relation);
    1859             : 
    1860    12779070 :             if (!HeapTupleIsSurelyDead(heapTuple, vistest))
    1861    12082684 :                 *all_dead = false;
    1862             :         }
    1863             : 
    1864             :         /*
    1865             :          * Check to see if HOT chain continues past this tuple; if so fetch
    1866             :          * the next offnum and loop around.
    1867             :          */
    1868    14445252 :         if (HeapTupleIsHotUpdated(heapTuple))
    1869             :         {
    1870             :             Assert(ItemPointerGetBlockNumber(&heapTuple->t_data->t_ctid) ==
    1871             :                    blkno);
    1872     2098330 :             offnum = ItemPointerGetOffsetNumber(&heapTuple->t_data->t_ctid);
    1873     2098330 :             at_chain_start = false;
    1874     2098330 :             prev_xmax = HeapTupleHeaderGetUpdateXid(heapTuple->t_data);
    1875             :         }
    1876             :         else
    1877    12346922 :             break;              /* end of chain */
    1878             :     }
    1879             : 
    1880    13124984 :     return false;
    1881             : }
    1882             : 
    1883             : /*
    1884             :  *  heap_get_latest_tid -  get the latest tid of a specified tuple
    1885             :  *
    1886             :  * Actually, this gets the latest version that is visible according to the
    1887             :  * scan's snapshot.  Create a scan using SnapshotDirty to get the very latest,
    1888             :  * possibly uncommitted version.
    1889             :  *
    1890             :  * *tid is both an input and an output parameter: it is updated to
    1891             :  * show the latest version of the row.  Note that it will not be changed
    1892             :  * if no version of the row passes the snapshot test.
    1893             :  */
    1894             : void
    1895         300 : heap_get_latest_tid(TableScanDesc sscan,
    1896             :                     ItemPointer tid)
    1897             : {
    1898         300 :     Relation    relation = sscan->rs_rd;
    1899         300 :     Snapshot    snapshot = sscan->rs_snapshot;
    1900             :     ItemPointerData ctid;
    1901             :     TransactionId priorXmax;
    1902             : 
    1903             :     /*
    1904             :      * table_tuple_get_latest_tid() verified that the passed in tid is valid.
    1905             :      * Assume that t_ctid links are valid however - there shouldn't be invalid
    1906             :      * ones in the table.
    1907             :      */
    1908             :     Assert(ItemPointerIsValid(tid));
    1909             : 
    1910             :     /*
    1911             :      * Loop to chase down t_ctid links.  At top of loop, ctid is the tuple we
    1912             :      * need to examine, and *tid is the TID we will return if ctid turns out
    1913             :      * to be bogus.
    1914             :      *
    1915             :      * Note that we will loop until we reach the end of the t_ctid chain.
    1916             :      * Depending on the snapshot passed, there might be at most one visible
    1917             :      * version of the row, but we don't try to optimize for that.
    1918             :      */
    1919         300 :     ctid = *tid;
    1920         300 :     priorXmax = InvalidTransactionId;   /* cannot check first XMIN */
    1921             :     for (;;)
    1922          90 :     {
    1923             :         Buffer      buffer;
    1924             :         Page        page;
    1925             :         OffsetNumber offnum;
    1926             :         ItemId      lp;
    1927             :         HeapTupleData tp;
    1928             :         bool        valid;
    1929             : 
    1930             :         /*
    1931             :          * Read, pin, and lock the page.
    1932             :          */
    1933         390 :         buffer = ReadBuffer(relation, ItemPointerGetBlockNumber(&ctid));
    1934         390 :         LockBuffer(buffer, BUFFER_LOCK_SHARE);
    1935         390 :         page = BufferGetPage(buffer);
    1936             : 
    1937             :         /*
    1938             :          * Check for bogus item number.  This is not treated as an error
    1939             :          * condition because it can happen while following a t_ctid link. We
    1940             :          * just assume that the prior tid is OK and return it unchanged.
    1941             :          */
    1942         390 :         offnum = ItemPointerGetOffsetNumber(&ctid);
    1943         390 :         if (offnum < FirstOffsetNumber || offnum > PageGetMaxOffsetNumber(page))
    1944             :         {
    1945           0 :             UnlockReleaseBuffer(buffer);
    1946           0 :             break;
    1947             :         }
    1948         390 :         lp = PageGetItemId(page, offnum);
    1949         390 :         if (!ItemIdIsNormal(lp))
    1950             :         {
    1951           0 :             UnlockReleaseBuffer(buffer);
    1952           0 :             break;
    1953             :         }
    1954             : 
    1955             :         /* OK to access the tuple */
    1956         390 :         tp.t_self = ctid;
    1957         390 :         tp.t_data = (HeapTupleHeader) PageGetItem(page, lp);
    1958         390 :         tp.t_len = ItemIdGetLength(lp);
    1959         390 :         tp.t_tableOid = RelationGetRelid(relation);
    1960             : 
    1961             :         /*
    1962             :          * After following a t_ctid link, we might arrive at an unrelated
    1963             :          * tuple.  Check for XMIN match.
    1964             :          */
    1965         480 :         if (TransactionIdIsValid(priorXmax) &&
    1966          90 :             !TransactionIdEquals(priorXmax, HeapTupleHeaderGetXmin(tp.t_data)))
    1967             :         {
    1968           0 :             UnlockReleaseBuffer(buffer);
    1969           0 :             break;
    1970             :         }
    1971             : 
    1972             :         /*
    1973             :          * Check tuple visibility; if visible, set it as the new result
    1974             :          * candidate.
    1975             :          */
    1976         390 :         valid = HeapTupleSatisfiesVisibility(&tp, snapshot, buffer);
    1977         390 :         HeapCheckForSerializableConflictOut(valid, relation, &tp, buffer, snapshot);
    1978         390 :         if (valid)
    1979         276 :             *tid = ctid;
    1980             : 
    1981             :         /*
    1982             :          * If there's a valid t_ctid link, follow it, else we're done.
    1983             :          */
    1984         552 :         if ((tp.t_data->t_infomask & HEAP_XMAX_INVALID) ||
    1985         276 :             HeapTupleHeaderIsOnlyLocked(tp.t_data) ||
    1986         228 :             HeapTupleHeaderIndicatesMovedPartitions(tp.t_data) ||
    1987         114 :             ItemPointerEquals(&tp.t_self, &tp.t_data->t_ctid))
    1988             :         {
    1989         300 :             UnlockReleaseBuffer(buffer);
    1990         300 :             break;
    1991             :         }
    1992             : 
    1993          90 :         ctid = tp.t_data->t_ctid;
    1994          90 :         priorXmax = HeapTupleHeaderGetUpdateXid(tp.t_data);
    1995          90 :         UnlockReleaseBuffer(buffer);
    1996             :     }                           /* end of loop */
    1997         300 : }
    1998             : 
    1999             : 
    2000             : /*
    2001             :  * UpdateXmaxHintBits - update tuple hint bits after xmax transaction ends
    2002             :  *
    2003             :  * This is called after we have waited for the XMAX transaction to terminate.
    2004             :  * If the transaction aborted, we guarantee the XMAX_INVALID hint bit will
    2005             :  * be set on exit.  If the transaction committed, we set the XMAX_COMMITTED
    2006             :  * hint bit if possible --- but beware that that may not yet be possible,
    2007             :  * if the transaction committed asynchronously.
    2008             :  *
    2009             :  * Note that if the transaction was a locker only, we set HEAP_XMAX_INVALID
    2010             :  * even if it commits.
    2011             :  *
    2012             :  * Hence callers should look only at XMAX_INVALID.
    2013             :  *
    2014             :  * Note this is not allowed for tuples whose xmax is a multixact.
    2015             :  */
    2016             : static void
    2017         350 : UpdateXmaxHintBits(HeapTupleHeader tuple, Buffer buffer, TransactionId xid)
    2018             : {
    2019             :     Assert(TransactionIdEquals(HeapTupleHeaderGetRawXmax(tuple), xid));
    2020             :     Assert(!(tuple->t_infomask & HEAP_XMAX_IS_MULTI));
    2021             : 
    2022         350 :     if (!(tuple->t_infomask & (HEAP_XMAX_COMMITTED | HEAP_XMAX_INVALID)))
    2023             :     {
    2024         634 :         if (!HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_infomask) &&
    2025         284 :             TransactionIdDidCommit(xid))
    2026         232 :             HeapTupleSetHintBits(tuple, buffer, HEAP_XMAX_COMMITTED,
    2027             :                                  xid);
    2028             :         else
    2029         118 :             HeapTupleSetHintBits(tuple, buffer, HEAP_XMAX_INVALID,
    2030             :                                  InvalidTransactionId);
    2031             :     }
    2032         350 : }
    2033             : 
    2034             : 
    2035             : /*
    2036             :  * GetBulkInsertState - prepare status object for a bulk insert
    2037             :  */
    2038             : BulkInsertState
    2039        4468 : GetBulkInsertState(void)
    2040             : {
    2041             :     BulkInsertState bistate;
    2042             : 
    2043        4468 :     bistate = (BulkInsertState) palloc(sizeof(BulkInsertStateData));
    2044        4468 :     bistate->strategy = GetAccessStrategy(BAS_BULKWRITE);
    2045        4468 :     bistate->current_buf = InvalidBuffer;
    2046        4468 :     bistate->next_free = InvalidBlockNumber;
    2047        4468 :     bistate->last_free = InvalidBlockNumber;
    2048        4468 :     bistate->already_extended_by = 0;
    2049        4468 :     return bistate;
    2050             : }
    2051             : 
    2052             : /*
    2053             :  * FreeBulkInsertState - clean up after finishing a bulk insert
    2054             :  */
    2055             : void
    2056        4190 : FreeBulkInsertState(BulkInsertState bistate)
    2057             : {
    2058        4190 :     if (bistate->current_buf != InvalidBuffer)
    2059        3450 :         ReleaseBuffer(bistate->current_buf);
    2060        4190 :     FreeAccessStrategy(bistate->strategy);
    2061        4190 :     pfree(bistate);
    2062        4190 : }
    2063             : 
    2064             : /*
    2065             :  * ReleaseBulkInsertStatePin - release a buffer currently held in bistate
    2066             :  */
    2067             : void
    2068      161512 : ReleaseBulkInsertStatePin(BulkInsertState bistate)
    2069             : {
    2070      161512 :     if (bistate->current_buf != InvalidBuffer)
    2071       60042 :         ReleaseBuffer(bistate->current_buf);
    2072      161512 :     bistate->current_buf = InvalidBuffer;
    2073             : 
    2074             :     /*
    2075             :      * Despite the name, we also reset bulk relation extension state.
    2076             :      * Otherwise we can end up erroring out due to looking for free space in
    2077             :      * ->next_free of one partition, even though ->next_free was set when
    2078             :      * extending another partition. It could obviously also be bad for
    2079             :      * efficiency to look at existing blocks at offsets from another
    2080             :      * partition, even if we don't error out.
    2081             :      */
    2082      161512 :     bistate->next_free = InvalidBlockNumber;
    2083      161512 :     bistate->last_free = InvalidBlockNumber;
    2084      161512 : }
    2085             : 
    2086             : 
    2087             : /*
    2088             :  *  heap_insert     - insert tuple into a heap
    2089             :  *
    2090             :  * The new tuple is stamped with current transaction ID and the specified
    2091             :  * command ID.
    2092             :  *
    2093             :  * See table_tuple_insert for comments about most of the input flags, except
    2094             :  * that this routine directly takes a tuple rather than a slot.
    2095             :  *
    2096             :  * There's corresponding HEAP_INSERT_ options to all the TABLE_INSERT_
    2097             :  * options, and there additionally is HEAP_INSERT_SPECULATIVE which is used to
    2098             :  * implement table_tuple_insert_speculative().
    2099             :  *
    2100             :  * On return the header fields of *tup are updated to match the stored tuple;
    2101             :  * in particular tup->t_self receives the actual TID where the tuple was
    2102             :  * stored.  But note that any toasting of fields within the tuple data is NOT
    2103             :  * reflected into *tup.
    2104             :  */
    2105             : void
    2106    15846670 : heap_insert(Relation relation, HeapTuple tup, CommandId cid,
    2107             :             int options, BulkInsertState bistate)
    2108             : {
    2109    15846670 :     TransactionId xid = GetCurrentTransactionId();
    2110             :     HeapTuple   heaptup;
    2111             :     Buffer      buffer;
    2112    15846660 :     Buffer      vmbuffer = InvalidBuffer;
    2113    15846660 :     bool        all_visible_cleared = false;
    2114             : 
    2115             :     /* Cheap, simplistic check that the tuple matches the rel's rowtype. */
    2116             :     Assert(HeapTupleHeaderGetNatts(tup->t_data) <=
    2117             :            RelationGetNumberOfAttributes(relation));
    2118             : 
    2119             :     /*
    2120             :      * Fill in tuple header fields and toast the tuple if necessary.
    2121             :      *
    2122             :      * Note: below this point, heaptup is the data we actually intend to store
    2123             :      * into the relation; tup is the caller's original untoasted data.
    2124             :      */
    2125    15846660 :     heaptup = heap_prepare_insert(relation, tup, xid, cid, options);
    2126             : 
    2127             :     /*
    2128             :      * Find buffer to insert this tuple into.  If the page is all visible,
    2129             :      * this will also pin the requisite visibility map page.
    2130             :      */
    2131    15846660 :     buffer = RelationGetBufferForTuple(relation, heaptup->t_len,
    2132             :                                        InvalidBuffer, options, bistate,
    2133             :                                        &vmbuffer, NULL,
    2134             :                                        0);
    2135             : 
    2136             :     /*
    2137             :      * We're about to do the actual insert -- but check for conflict first, to
    2138             :      * avoid possibly having to roll back work we've just done.
    2139             :      *
    2140             :      * This is safe without a recheck as long as there is no possibility of
    2141             :      * another process scanning the page between this check and the insert
    2142             :      * being visible to the scan (i.e., an exclusive buffer content lock is
    2143             :      * continuously held from this point until the tuple insert is visible).
    2144             :      *
    2145             :      * For a heap insert, we only need to check for table-level SSI locks. Our
    2146             :      * new tuple can't possibly conflict with existing tuple locks, and heap
    2147             :      * page locks are only consolidated versions of tuple locks; they do not
    2148             :      * lock "gaps" as index page locks do.  So we don't need to specify a
    2149             :      * buffer when making the call, which makes for a faster check.
    2150             :      */
    2151    15846660 :     CheckForSerializableConflictIn(relation, NULL, InvalidBlockNumber);
    2152             : 
    2153             :     /* NO EREPORT(ERROR) from here till changes are logged */
    2154    15846636 :     START_CRIT_SECTION();
    2155             : 
    2156    15846636 :     RelationPutHeapTuple(relation, buffer, heaptup,
    2157    15846636 :                          (options & HEAP_INSERT_SPECULATIVE) != 0);
    2158             : 
    2159    15846636 :     if (PageIsAllVisible(BufferGetPage(buffer)))
    2160             :     {
    2161       13358 :         all_visible_cleared = true;
    2162       13358 :         PageClearAllVisible(BufferGetPage(buffer));
    2163       13358 :         visibilitymap_clear(relation,
    2164       13358 :                             ItemPointerGetBlockNumber(&(heaptup->t_self)),
    2165             :                             vmbuffer, VISIBILITYMAP_VALID_BITS);
    2166             :     }
    2167             : 
    2168             :     /*
    2169             :      * XXX Should we set PageSetPrunable on this page ?
    2170             :      *
    2171             :      * The inserting transaction may eventually abort thus making this tuple
    2172             :      * DEAD and hence available for pruning. Though we don't want to optimize
    2173             :      * for aborts, if no other tuple in this page is UPDATEd/DELETEd, the
    2174             :      * aborted tuple will never be pruned until next vacuum is triggered.
    2175             :      *
    2176             :      * If you do add PageSetPrunable here, add it in heap_xlog_insert too.
    2177             :      */
    2178             : 
    2179    15846636 :     MarkBufferDirty(buffer);
    2180             : 
    2181             :     /* XLOG stuff */
    2182    15846636 :     if (RelationNeedsWAL(relation))
    2183             :     {
    2184             :         xl_heap_insert xlrec;
    2185             :         xl_heap_header xlhdr;
    2186             :         XLogRecPtr  recptr;
    2187    13726868 :         Page        page = BufferGetPage(buffer);
    2188    13726868 :         uint8       info = XLOG_HEAP_INSERT;
    2189    13726868 :         int         bufflags = 0;
    2190             : 
    2191             :         /*
    2192             :          * If this is a catalog, we need to transmit combo CIDs to properly
    2193             :          * decode, so log that as well.
    2194             :          */
    2195    13726868 :         if (RelationIsAccessibleInLogicalDecoding(relation))
    2196        6476 :             log_heap_new_cid(relation, heaptup);
    2197             : 
    2198             :         /*
    2199             :          * If this is the single and first tuple on page, we can reinit the
    2200             :          * page instead of restoring the whole thing.  Set flag, and hide
    2201             :          * buffer references from XLogInsert.
    2202             :          */
    2203    13902832 :         if (ItemPointerGetOffsetNumber(&(heaptup->t_self)) == FirstOffsetNumber &&
    2204      175964 :             PageGetMaxOffsetNumber(page) == FirstOffsetNumber)
    2205             :         {
    2206      174566 :             info |= XLOG_HEAP_INIT_PAGE;
    2207      174566 :             bufflags |= REGBUF_WILL_INIT;
    2208             :         }
    2209             : 
    2210    13726868 :         xlrec.offnum = ItemPointerGetOffsetNumber(&heaptup->t_self);
    2211    13726868 :         xlrec.flags = 0;
    2212    13726868 :         if (all_visible_cleared)
    2213       13352 :             xlrec.flags |= XLH_INSERT_ALL_VISIBLE_CLEARED;
    2214    13726868 :         if (options & HEAP_INSERT_SPECULATIVE)
    2215        4110 :             xlrec.flags |= XLH_INSERT_IS_SPECULATIVE;
    2216             :         Assert(ItemPointerGetBlockNumber(&heaptup->t_self) == BufferGetBlockNumber(buffer));
    2217             : 
    2218             :         /*
    2219             :          * For logical decoding, we need the tuple even if we're doing a full
    2220             :          * page write, so make sure it's included even if we take a full-page
    2221             :          * image. (XXX We could alternatively store a pointer into the FPW).
    2222             :          */
    2223    13726868 :         if (RelationIsLogicallyLogged(relation) &&
    2224      499750 :             !(options & HEAP_INSERT_NO_LOGICAL))
    2225             :         {
    2226      499696 :             xlrec.flags |= XLH_INSERT_CONTAINS_NEW_TUPLE;
    2227      499696 :             bufflags |= REGBUF_KEEP_DATA;
    2228             : 
    2229      499696 :             if (IsToastRelation(relation))
    2230        3572 :                 xlrec.flags |= XLH_INSERT_ON_TOAST_RELATION;
    2231             :         }
    2232             : 
    2233    13726868 :         XLogBeginInsert();
    2234    13726868 :         XLogRegisterData(&xlrec, SizeOfHeapInsert);
    2235             : 
    2236    13726868 :         xlhdr.t_infomask2 = heaptup->t_data->t_infomask2;
    2237    13726868 :         xlhdr.t_infomask = heaptup->t_data->t_infomask;
    2238    13726868 :         xlhdr.t_hoff = heaptup->t_data->t_hoff;
    2239             : 
    2240             :         /*
    2241             :          * note we mark xlhdr as belonging to buffer; if XLogInsert decides to
    2242             :          * write the whole page to the xlog, we don't need to store
    2243             :          * xl_heap_header in the xlog.
    2244             :          */
    2245    13726868 :         XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
    2246    13726868 :         XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
    2247             :         /* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
    2248    13726868 :         XLogRegisterBufData(0,
    2249    13726868 :                             (char *) heaptup->t_data + SizeofHeapTupleHeader,
    2250    13726868 :                             heaptup->t_len - SizeofHeapTupleHeader);
    2251             : 
    2252             :         /* filtering by origin on a row level is much more efficient */
    2253    13726868 :         XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
    2254             : 
    2255    13726868 :         recptr = XLogInsert(RM_HEAP_ID, info);
    2256             : 
    2257    13726868 :         PageSetLSN(page, recptr);
    2258             :     }
    2259             : 
    2260    15846636 :     END_CRIT_SECTION();
    2261             : 
    2262    15846636 :     UnlockReleaseBuffer(buffer);
    2263    15846636 :     if (vmbuffer != InvalidBuffer)
    2264       13934 :         ReleaseBuffer(vmbuffer);
    2265             : 
    2266             :     /*
    2267             :      * If tuple is cachable, mark it for invalidation from the caches in case
    2268             :      * we abort.  Note it is OK to do this after releasing the buffer, because
    2269             :      * the heaptup data structure is all in local memory, not in the shared
    2270             :      * buffer.
    2271             :      */
    2272    15846636 :     CacheInvalidateHeapTuple(relation, heaptup, NULL);
    2273             : 
    2274             :     /* Note: speculative insertions are counted too, even if aborted later */
    2275    15846636 :     pgstat_count_heap_insert(relation, 1);
    2276             : 
    2277             :     /*
    2278             :      * If heaptup is a private copy, release it.  Don't forget to copy t_self
    2279             :      * back to the caller's image, too.
    2280             :      */
    2281    15846636 :     if (heaptup != tup)
    2282             :     {
    2283       34268 :         tup->t_self = heaptup->t_self;
    2284       34268 :         heap_freetuple(heaptup);
    2285             :     }
    2286    15846636 : }
    2287             : 
    2288             : /*
    2289             :  * Subroutine for heap_insert(). Prepares a tuple for insertion. This sets the
    2290             :  * tuple header fields and toasts the tuple if necessary.  Returns a toasted
    2291             :  * version of the tuple if it was toasted, or the original tuple if not. Note
    2292             :  * that in any case, the header fields are also set in the original tuple.
    2293             :  */
    2294             : static HeapTuple
    2295    18691594 : heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid,
    2296             :                     CommandId cid, int options)
    2297             : {
    2298             :     /*
    2299             :      * To allow parallel inserts, we need to ensure that they are safe to be
    2300             :      * performed in workers. We have the infrastructure to allow parallel
    2301             :      * inserts in general except for the cases where inserts generate a new
    2302             :      * CommandId (eg. inserts into a table having a foreign key column).
    2303             :      */
    2304    18691594 :     if (IsParallelWorker())
    2305           0 :         ereport(ERROR,
    2306             :                 (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
    2307             :                  errmsg("cannot insert tuples in a parallel worker")));
    2308             : 
    2309    18691594 :     tup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
    2310    18691594 :     tup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
    2311    18691594 :     tup->t_data->t_infomask |= HEAP_XMAX_INVALID;
    2312    18691594 :     HeapTupleHeaderSetXmin(tup->t_data, xid);
    2313    18691594 :     if (options & HEAP_INSERT_FROZEN)
    2314      204222 :         HeapTupleHeaderSetXminFrozen(tup->t_data);
    2315             : 
    2316    18691594 :     HeapTupleHeaderSetCmin(tup->t_data, cid);
    2317    18691594 :     HeapTupleHeaderSetXmax(tup->t_data, 0); /* for cleanliness */
    2318    18691594 :     tup->t_tableOid = RelationGetRelid(relation);
    2319             : 
    2320             :     /*
    2321             :      * If the new tuple is too big for storage or contains already toasted
    2322             :      * out-of-line attributes from some other relation, invoke the toaster.
    2323             :      */
    2324    18691594 :     if (relation->rd_rel->relkind != RELKIND_RELATION &&
    2325       58212 :         relation->rd_rel->relkind != RELKIND_MATVIEW)
    2326             :     {
    2327             :         /* toast table entries should never be recursively toasted */
    2328             :         Assert(!HeapTupleHasExternal(tup));
    2329       58116 :         return tup;
    2330             :     }
    2331    18633478 :     else if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD)
    2332       34350 :         return heap_toast_insert_or_update(relation, tup, NULL, options);
    2333             :     else
    2334    18599128 :         return tup;
    2335             : }
    2336             : 
    2337             : /*
    2338             :  * Helper for heap_multi_insert() that computes the number of entire pages
    2339             :  * that inserting the remaining heaptuples requires. Used to determine how
    2340             :  * much the relation needs to be extended by.
    2341             :  */
    2342             : static int
    2343      684574 : heap_multi_insert_pages(HeapTuple *heaptuples, int done, int ntuples, Size saveFreeSpace)
    2344             : {
    2345      684574 :     size_t      page_avail = BLCKSZ - SizeOfPageHeaderData - saveFreeSpace;
    2346      684574 :     int         npages = 1;
    2347             : 
    2348     4718388 :     for (int i = done; i < ntuples; i++)
    2349             :     {
    2350     4033814 :         size_t      tup_sz = sizeof(ItemIdData) + MAXALIGN(heaptuples[i]->t_len);
    2351             : 
    2352     4033814 :         if (page_avail < tup_sz)
    2353             :         {
    2354       31214 :             npages++;
    2355       31214 :             page_avail = BLCKSZ - SizeOfPageHeaderData - saveFreeSpace;
    2356             :         }
    2357     4033814 :         page_avail -= tup_sz;
    2358             :     }
    2359             : 
    2360      684574 :     return npages;
    2361             : }
    2362             : 
    2363             : /*
    2364             :  *  heap_multi_insert   - insert multiple tuples into a heap
    2365             :  *
    2366             :  * This is like heap_insert(), but inserts multiple tuples in one operation.
    2367             :  * That's faster than calling heap_insert() in a loop, because when multiple
    2368             :  * tuples can be inserted on a single page, we can write just a single WAL
    2369             :  * record covering all of them, and only need to lock/unlock the page once.
    2370             :  *
    2371             :  * Note: this leaks memory into the current memory context. You can create a
    2372             :  * temporary context before calling this, if that's a problem.
    2373             :  */
    2374             : void
    2375      672182 : heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples,
    2376             :                   CommandId cid, int options, BulkInsertState bistate)
    2377             : {
    2378      672182 :     TransactionId xid = GetCurrentTransactionId();
    2379             :     HeapTuple  *heaptuples;
    2380             :     int         i;
    2381             :     int         ndone;
    2382             :     PGAlignedBlock scratch;
    2383             :     Page        page;
    2384      672182 :     Buffer      vmbuffer = InvalidBuffer;
    2385             :     bool        needwal;
    2386             :     Size        saveFreeSpace;
    2387      672182 :     bool        need_tuple_data = RelationIsLogicallyLogged(relation);
    2388      672182 :     bool        need_cids = RelationIsAccessibleInLogicalDecoding(relation);
    2389      672182 :     bool        starting_with_empty_page = false;
    2390      672182 :     int         npages = 0;
    2391      672182 :     int         npages_used = 0;
    2392             : 
    2393             :     /* currently not needed (thus unsupported) for heap_multi_insert() */
    2394             :     Assert(!(options & HEAP_INSERT_NO_LOGICAL));
    2395             : 
    2396      672182 :     needwal = RelationNeedsWAL(relation);
    2397      672182 :     saveFreeSpace = RelationGetTargetPageFreeSpace(relation,
    2398             :                                                    HEAP_DEFAULT_FILLFACTOR);
    2399             : 
    2400             :     /* Toast and set header data in all the slots */
    2401      672182 :     heaptuples = palloc(ntuples * sizeof(HeapTuple));
    2402     3517116 :     for (i = 0; i < ntuples; i++)
    2403             :     {
    2404             :         HeapTuple   tuple;
    2405             : 
    2406     2844934 :         tuple = ExecFetchSlotHeapTuple(slots[i], true, NULL);
    2407     2844934 :         slots[i]->tts_tableOid = RelationGetRelid(relation);
    2408     2844934 :         tuple->t_tableOid = slots[i]->tts_tableOid;
    2409     2844934 :         heaptuples[i] = heap_prepare_insert(relation, tuple, xid, cid,
    2410             :                                             options);
    2411             :     }
    2412             : 
    2413             :     /*
    2414             :      * We're about to do the actual inserts -- but check for conflict first,
    2415             :      * to minimize the possibility of having to roll back work we've just
    2416             :      * done.
    2417             :      *
    2418             :      * A check here does not definitively prevent a serialization anomaly;
    2419             :      * that check MUST be done at least past the point of acquiring an
    2420             :      * exclusive buffer content lock on every buffer that will be affected,
    2421             :      * and MAY be done after all inserts are reflected in the buffers and
    2422             :      * those locks are released; otherwise there is a race condition.  Since
    2423             :      * multiple buffers can be locked and unlocked in the loop below, and it
    2424             :      * would not be feasible to identify and lock all of those buffers before
    2425             :      * the loop, we must do a final check at the end.
    2426             :      *
    2427             :      * The check here could be omitted with no loss of correctness; it is
    2428             :      * present strictly as an optimization.
    2429             :      *
    2430             :      * For heap inserts, we only need to check for table-level SSI locks. Our
    2431             :      * new tuples can't possibly conflict with existing tuple locks, and heap
    2432             :      * page locks are only consolidated versions of tuple locks; they do not
    2433             :      * lock "gaps" as index page locks do.  So we don't need to specify a
    2434             :      * buffer when making the call, which makes for a faster check.
    2435             :      */
    2436      672182 :     CheckForSerializableConflictIn(relation, NULL, InvalidBlockNumber);
    2437             : 
    2438      672182 :     ndone = 0;
    2439     1373074 :     while (ndone < ntuples)
    2440             :     {
    2441             :         Buffer      buffer;
    2442      700892 :         bool        all_visible_cleared = false;
    2443      700892 :         bool        all_frozen_set = false;
    2444             :         int         nthispage;
    2445             : 
    2446      700892 :         CHECK_FOR_INTERRUPTS();
    2447             : 
    2448             :         /*
    2449             :          * Compute number of pages needed to fit the to-be-inserted tuples in
    2450             :          * the worst case.  This will be used to determine how much to extend
    2451             :          * the relation by in RelationGetBufferForTuple(), if needed.  If we
    2452             :          * filled a prior page from scratch, we can just update our last
    2453             :          * computation, but if we started with a partially filled page,
    2454             :          * recompute from scratch, the number of potentially required pages
    2455             :          * can vary due to tuples needing to fit onto the page, page headers
    2456             :          * etc.
    2457             :          */
    2458      700892 :         if (ndone == 0 || !starting_with_empty_page)
    2459             :         {
    2460      684574 :             npages = heap_multi_insert_pages(heaptuples, ndone, ntuples,
    2461             :                                              saveFreeSpace);
    2462      684574 :             npages_used = 0;
    2463             :         }
    2464             :         else
    2465       16318 :             npages_used++;
    2466             : 
    2467             :         /*
    2468             :          * Find buffer where at least the next tuple will fit.  If the page is
    2469             :          * all-visible, this will also pin the requisite visibility map page.
    2470             :          *
    2471             :          * Also pin visibility map page if COPY FREEZE inserts tuples into an
    2472             :          * empty page. See all_frozen_set below.
    2473             :          */
    2474      700892 :         buffer = RelationGetBufferForTuple(relation, heaptuples[ndone]->t_len,
    2475             :                                            InvalidBuffer, options, bistate,
    2476             :                                            &vmbuffer, NULL,
    2477             :                                            npages - npages_used);
    2478      700892 :         page = BufferGetPage(buffer);
    2479             : 
    2480      700892 :         starting_with_empty_page = PageGetMaxOffsetNumber(page) == 0;
    2481             : 
    2482      700892 :         if (starting_with_empty_page && (options & HEAP_INSERT_FROZEN))
    2483        3322 :             all_frozen_set = true;
    2484             : 
    2485             :         /* NO EREPORT(ERROR) from here till changes are logged */
    2486      700892 :         START_CRIT_SECTION();
    2487             : 
    2488             :         /*
    2489             :          * RelationGetBufferForTuple has ensured that the first tuple fits.
    2490             :          * Put that on the page, and then as many other tuples as fit.
    2491             :          */
    2492      700892 :         RelationPutHeapTuple(relation, buffer, heaptuples[ndone], false);
    2493             : 
    2494             :         /*
    2495             :          * For logical decoding we need combo CIDs to properly decode the
    2496             :          * catalog.
    2497             :          */
    2498      700892 :         if (needwal && need_cids)
    2499        9568 :             log_heap_new_cid(relation, heaptuples[ndone]);
    2500             : 
    2501     2844934 :         for (nthispage = 1; ndone + nthispage < ntuples; nthispage++)
    2502             :         {
    2503     2172752 :             HeapTuple   heaptup = heaptuples[ndone + nthispage];
    2504             : 
    2505     2172752 :             if (PageGetHeapFreeSpace(page) < MAXALIGN(heaptup->t_len) + saveFreeSpace)
    2506       28710 :                 break;
    2507             : 
    2508     2144042 :             RelationPutHeapTuple(relation, buffer, heaptup, false);
    2509             : 
    2510             :             /*
    2511             :              * For logical decoding we need combo CIDs to properly decode the
    2512             :              * catalog.
    2513             :              */
    2514     2144042 :             if (needwal && need_cids)
    2515        9154 :                 log_heap_new_cid(relation, heaptup);
    2516             :         }
    2517             : 
    2518             :         /*
    2519             :          * If the page is all visible, need to clear that, unless we're only
    2520             :          * going to add further frozen rows to it.
    2521             :          *
    2522             :          * If we're only adding already frozen rows to a previously empty
    2523             :          * page, mark it as all-visible.
    2524             :          */
    2525      700892 :         if (PageIsAllVisible(page) && !(options & HEAP_INSERT_FROZEN))
    2526             :         {
    2527        5452 :             all_visible_cleared = true;
    2528        5452 :             PageClearAllVisible(page);
    2529        5452 :             visibilitymap_clear(relation,
    2530             :                                 BufferGetBlockNumber(buffer),
    2531             :                                 vmbuffer, VISIBILITYMAP_VALID_BITS);
    2532             :         }
    2533      695440 :         else if (all_frozen_set)
    2534        3322 :             PageSetAllVisible(page);
    2535             : 
    2536             :         /*
    2537             :          * XXX Should we set PageSetPrunable on this page ? See heap_insert()
    2538             :          */
    2539             : 
    2540      700892 :         MarkBufferDirty(buffer);
    2541             : 
    2542             :         /* XLOG stuff */
    2543      700892 :         if (needwal)
    2544             :         {
    2545             :             XLogRecPtr  recptr;
    2546             :             xl_heap_multi_insert *xlrec;
    2547      693250 :             uint8       info = XLOG_HEAP2_MULTI_INSERT;
    2548             :             char       *tupledata;
    2549             :             int         totaldatalen;
    2550      693250 :             char       *scratchptr = scratch.data;
    2551             :             bool        init;
    2552      693250 :             int         bufflags = 0;
    2553             : 
    2554             :             /*
    2555             :              * If the page was previously empty, we can reinit the page
    2556             :              * instead of restoring the whole thing.
    2557             :              */
    2558      693250 :             init = starting_with_empty_page;
    2559             : 
    2560             :             /* allocate xl_heap_multi_insert struct from the scratch area */
    2561      693250 :             xlrec = (xl_heap_multi_insert *) scratchptr;
    2562      693250 :             scratchptr += SizeOfHeapMultiInsert;
    2563             : 
    2564             :             /*
    2565             :              * Allocate offsets array. Unless we're reinitializing the page,
    2566             :              * in that case the tuples are stored in order starting at
    2567             :              * FirstOffsetNumber and we don't need to store the offsets
    2568             :              * explicitly.
    2569             :              */
    2570      693250 :             if (!init)
    2571      668216 :                 scratchptr += nthispage * sizeof(OffsetNumber);
    2572             : 
    2573             :             /* the rest of the scratch space is used for tuple data */
    2574      693250 :             tupledata = scratchptr;
    2575             : 
    2576             :             /* check that the mutually exclusive flags are not both set */
    2577             :             Assert(!(all_visible_cleared && all_frozen_set));
    2578             : 
    2579      693250 :             xlrec->flags = 0;
    2580      693250 :             if (all_visible_cleared)
    2581        5452 :                 xlrec->flags = XLH_INSERT_ALL_VISIBLE_CLEARED;
    2582      693250 :             if (all_frozen_set)
    2583          26 :                 xlrec->flags = XLH_INSERT_ALL_FROZEN_SET;
    2584             : 
    2585      693250 :             xlrec->ntuples = nthispage;
    2586             : 
    2587             :             /*
    2588             :              * Write out an xl_multi_insert_tuple and the tuple data itself
    2589             :              * for each tuple.
    2590             :              */
    2591     3127382 :             for (i = 0; i < nthispage; i++)
    2592             :             {
    2593     2434132 :                 HeapTuple   heaptup = heaptuples[ndone + i];
    2594             :                 xl_multi_insert_tuple *tuphdr;
    2595             :                 int         datalen;
    2596             : 
    2597     2434132 :                 if (!init)
    2598     1401134 :                     xlrec->offsets[i] = ItemPointerGetOffsetNumber(&heaptup->t_self);
    2599             :                 /* xl_multi_insert_tuple needs two-byte alignment. */
    2600     2434132 :                 tuphdr = (xl_multi_insert_tuple *) SHORTALIGN(scratchptr);
    2601     2434132 :                 scratchptr = ((char *) tuphdr) + SizeOfMultiInsertTuple;
    2602             : 
    2603     2434132 :                 tuphdr->t_infomask2 = heaptup->t_data->t_infomask2;
    2604     2434132 :                 tuphdr->t_infomask = heaptup->t_data->t_infomask;
    2605     2434132 :                 tuphdr->t_hoff = heaptup->t_data->t_hoff;
    2606             : 
    2607             :                 /* write bitmap [+ padding] [+ oid] + data */
    2608     2434132 :                 datalen = heaptup->t_len - SizeofHeapTupleHeader;
    2609     2434132 :                 memcpy(scratchptr,
    2610     2434132 :                        (char *) heaptup->t_data + SizeofHeapTupleHeader,
    2611             :                        datalen);
    2612     2434132 :                 tuphdr->datalen = datalen;
    2613     2434132 :                 scratchptr += datalen;
    2614             :             }
    2615      693250 :             totaldatalen = scratchptr - tupledata;
    2616             :             Assert((scratchptr - scratch.data) < BLCKSZ);
    2617             : 
    2618      693250 :             if (need_tuple_data)
    2619         144 :                 xlrec->flags |= XLH_INSERT_CONTAINS_NEW_TUPLE;
    2620             : 
    2621             :             /*
    2622             :              * Signal that this is the last xl_heap_multi_insert record
    2623             :              * emitted by this call to heap_multi_insert(). Needed for logical
    2624             :              * decoding so it knows when to cleanup temporary data.
    2625             :              */
    2626      693250 :             if (ndone + nthispage == ntuples)
    2627      671358 :                 xlrec->flags |= XLH_INSERT_LAST_IN_MULTI;
    2628             : 
    2629      693250 :             if (init)
    2630             :             {
    2631       25034 :                 info |= XLOG_HEAP_INIT_PAGE;
    2632       25034 :                 bufflags |= REGBUF_WILL_INIT;
    2633             :             }
    2634             : 
    2635             :             /*
    2636             :              * If we're doing logical decoding, include the new tuple data
    2637             :              * even if we take a full-page image of the page.
    2638             :              */
    2639      693250 :             if (need_tuple_data)
    2640         144 :                 bufflags |= REGBUF_KEEP_DATA;
    2641             : 
    2642      693250 :             XLogBeginInsert();
    2643      693250 :             XLogRegisterData(xlrec, tupledata - scratch.data);
    2644      693250 :             XLogRegisterBuffer(0, buffer, REGBUF_STANDARD | bufflags);
    2645             : 
    2646      693250 :             XLogRegisterBufData(0, tupledata, totaldatalen);
    2647             : 
    2648             :             /* filtering by origin on a row level is much more efficient */
    2649      693250 :             XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
    2650             : 
    2651      693250 :             recptr = XLogInsert(RM_HEAP2_ID, info);
    2652             : 
    2653      693250 :             PageSetLSN(page, recptr);
    2654             :         }
    2655             : 
    2656      700892 :         END_CRIT_SECTION();
    2657             : 
    2658             :         /*
    2659             :          * If we've frozen everything on the page, update the visibilitymap.
    2660             :          * We're already holding pin on the vmbuffer.
    2661             :          */
    2662      700892 :         if (all_frozen_set)
    2663             :         {
    2664             :             Assert(PageIsAllVisible(page));
    2665             :             Assert(visibilitymap_pin_ok(BufferGetBlockNumber(buffer), vmbuffer));
    2666             : 
    2667             :             /*
    2668             :              * It's fine to use InvalidTransactionId here - this is only used
    2669             :              * when HEAP_INSERT_FROZEN is specified, which intentionally
    2670             :              * violates visibility rules.
    2671             :              */
    2672        3322 :             visibilitymap_set(relation, BufferGetBlockNumber(buffer), buffer,
    2673             :                               InvalidXLogRecPtr, vmbuffer,
    2674             :                               InvalidTransactionId,
    2675             :                               VISIBILITYMAP_ALL_VISIBLE | VISIBILITYMAP_ALL_FROZEN);
    2676             :         }
    2677             : 
    2678      700892 :         UnlockReleaseBuffer(buffer);
    2679      700892 :         ndone += nthispage;
    2680             : 
    2681             :         /*
    2682             :          * NB: Only release vmbuffer after inserting all tuples - it's fairly
    2683             :          * likely that we'll insert into subsequent heap pages that are likely
    2684             :          * to use the same vm page.
    2685             :          */
    2686             :     }
    2687             : 
    2688             :     /* We're done with inserting all tuples, so release the last vmbuffer. */
    2689      672182 :     if (vmbuffer != InvalidBuffer)
    2690        5656 :         ReleaseBuffer(vmbuffer);
    2691             : 
    2692             :     /*
    2693             :      * We're done with the actual inserts.  Check for conflicts again, to
    2694             :      * ensure that all rw-conflicts in to these inserts are detected.  Without
    2695             :      * this final check, a sequential scan of the heap may have locked the
    2696             :      * table after the "before" check, missing one opportunity to detect the
    2697             :      * conflict, and then scanned the table before the new tuples were there,
    2698             :      * missing the other chance to detect the conflict.
    2699             :      *
    2700             :      * For heap inserts, we only need to check for table-level SSI locks. Our
    2701             :      * new tuples can't possibly conflict with existing tuple locks, and heap
    2702             :      * page locks are only consolidated versions of tuple locks; they do not
    2703             :      * lock "gaps" as index page locks do.  So we don't need to specify a
    2704             :      * buffer when making the call.
    2705             :      */
    2706      672182 :     CheckForSerializableConflictIn(relation, NULL, InvalidBlockNumber);
    2707             : 
    2708             :     /*
    2709             :      * If tuples are cachable, mark them for invalidation from the caches in
    2710             :      * case we abort.  Note it is OK to do this after releasing the buffer,
    2711             :      * because the heaptuples data structure is all in local memory, not in
    2712             :      * the shared buffer.
    2713             :      */
    2714      672182 :     if (IsCatalogRelation(relation))
    2715             :     {
    2716     2319086 :         for (i = 0; i < ntuples; i++)
    2717     1649348 :             CacheInvalidateHeapTuple(relation, heaptuples[i], NULL);
    2718             :     }
    2719             : 
    2720             :     /* copy t_self fields back to the caller's slots */
    2721     3517116 :     for (i = 0; i < ntuples; i++)
    2722     2844934 :         slots[i]->tts_tid = heaptuples[i]->t_self;
    2723             : 
    2724      672182 :     pgstat_count_heap_insert(relation, ntuples);
    2725      672182 : }
    2726             : 
    2727             : /*
    2728             :  *  simple_heap_insert - insert a tuple
    2729             :  *
    2730             :  * Currently, this routine differs from heap_insert only in supplying
    2731             :  * a default command ID and not allowing access to the speedup options.
    2732             :  *
    2733             :  * This should be used rather than using heap_insert directly in most places
    2734             :  * where we are modifying system catalogs.
    2735             :  */
    2736             : void
    2737     1629328 : simple_heap_insert(Relation relation, HeapTuple tup)
    2738             : {
    2739     1629328 :     heap_insert(relation, tup, GetCurrentCommandId(true), 0, NULL);
    2740     1629328 : }
    2741             : 
    2742             : /*
    2743             :  * Given infomask/infomask2, compute the bits that must be saved in the
    2744             :  * "infobits" field of xl_heap_delete, xl_heap_update, xl_heap_lock,
    2745             :  * xl_heap_lock_updated WAL records.
    2746             :  *
    2747             :  * See fix_infomask_from_infobits.
    2748             :  */
    2749             : static uint8
    2750     3834228 : compute_infobits(uint16 infomask, uint16 infomask2)
    2751             : {
    2752             :     return
    2753     3834228 :         ((infomask & HEAP_XMAX_IS_MULTI) != 0 ? XLHL_XMAX_IS_MULTI : 0) |
    2754     3834228 :         ((infomask & HEAP_XMAX_LOCK_ONLY) != 0 ? XLHL_XMAX_LOCK_ONLY : 0) |
    2755     3834228 :         ((infomask & HEAP_XMAX_EXCL_LOCK) != 0 ? XLHL_XMAX_EXCL_LOCK : 0) |
    2756             :     /* note we ignore HEAP_XMAX_SHR_LOCK here */
    2757     7668456 :         ((infomask & HEAP_XMAX_KEYSHR_LOCK) != 0 ? XLHL_XMAX_KEYSHR_LOCK : 0) |
    2758             :         ((infomask2 & HEAP_KEYS_UPDATED) != 0 ?
    2759     3834228 :          XLHL_KEYS_UPDATED : 0);
    2760             : }
    2761             : 
    2762             : /*
    2763             :  * Given two versions of the same t_infomask for a tuple, compare them and
    2764             :  * return whether the relevant status for a tuple Xmax has changed.  This is
    2765             :  * used after a buffer lock has been released and reacquired: we want to ensure
    2766             :  * that the tuple state continues to be the same it was when we previously
    2767             :  * examined it.
    2768             :  *
    2769             :  * Note the Xmax field itself must be compared separately.
    2770             :  */
    2771             : static inline bool
    2772       10658 : xmax_infomask_changed(uint16 new_infomask, uint16 old_infomask)
    2773             : {
    2774       10658 :     const uint16 interesting =
    2775             :         HEAP_XMAX_IS_MULTI | HEAP_XMAX_LOCK_ONLY | HEAP_LOCK_MASK;
    2776             : 
    2777       10658 :     if ((new_infomask & interesting) != (old_infomask & interesting))
    2778          26 :         return true;
    2779             : 
    2780       10632 :     return false;
    2781             : }
    2782             : 
    2783             : /*
    2784             :  *  heap_delete - delete a tuple
    2785             :  *
    2786             :  * See table_tuple_delete() for an explanation of the parameters, except that
    2787             :  * this routine directly takes a tuple rather than a slot.
    2788             :  *
    2789             :  * In the failure cases, the routine fills *tmfd with the tuple's t_ctid,
    2790             :  * t_xmax (resolving a possible MultiXact, if necessary), and t_cmax (the last
    2791             :  * only for TM_SelfModified, since we cannot obtain cmax from a combo CID
    2792             :  * generated by another transaction).
    2793             :  */
    2794             : TM_Result
    2795     2943986 : heap_delete(Relation relation, ItemPointer tid,
    2796             :             CommandId cid, Snapshot crosscheck, bool wait,
    2797             :             TM_FailureData *tmfd, bool changingPart)
    2798             : {
    2799             :     TM_Result   result;
    2800     2943986 :     TransactionId xid = GetCurrentTransactionId();
    2801             :     ItemId      lp;
    2802             :     HeapTupleData tp;
    2803             :     Page        page;
    2804             :     BlockNumber block;
    2805             :     Buffer      buffer;
    2806     2943986 :     Buffer      vmbuffer = InvalidBuffer;
    2807             :     TransactionId new_xmax;
    2808             :     uint16      new_infomask,
    2809             :                 new_infomask2;
    2810     2943986 :     bool        have_tuple_lock = false;
    2811             :     bool        iscombo;
    2812     2943986 :     bool        all_visible_cleared = false;
    2813     2943986 :     HeapTuple   old_key_tuple = NULL;   /* replica identity of the tuple */
    2814     2943986 :     bool        old_key_copied = false;
    2815             : 
    2816             :     Assert(ItemPointerIsValid(tid));
    2817             : 
    2818             :     /*
    2819             :      * Forbid this during a parallel operation, lest it allocate a combo CID.
    2820             :      * Other workers might need that combo CID for visibility checks, and we
    2821             :      * have no provision for broadcasting it to them.
    2822             :      */
    2823     2943986 :     if (IsInParallelMode())
    2824           0 :         ereport(ERROR,
    2825             :                 (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
    2826             :                  errmsg("cannot delete tuples during a parallel operation")));
    2827             : 
    2828     2943986 :     block = ItemPointerGetBlockNumber(tid);
    2829     2943986 :     buffer = ReadBuffer(relation, block);
    2830     2943986 :     page = BufferGetPage(buffer);
    2831             : 
    2832             :     /*
    2833             :      * Before locking the buffer, pin the visibility map page if it appears to
    2834             :      * be necessary.  Since we haven't got the lock yet, someone else might be
    2835             :      * in the middle of changing this, so we'll need to recheck after we have
    2836             :      * the lock.
    2837             :      */
    2838     2943986 :     if (PageIsAllVisible(page))
    2839         446 :         visibilitymap_pin(relation, block, &vmbuffer);
    2840             : 
    2841     2943986 :     LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    2842             : 
    2843     2943986 :     lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tid));
    2844             :     Assert(ItemIdIsNormal(lp));
    2845             : 
    2846     2943986 :     tp.t_tableOid = RelationGetRelid(relation);
    2847     2943986 :     tp.t_data = (HeapTupleHeader) PageGetItem(page, lp);
    2848     2943986 :     tp.t_len = ItemIdGetLength(lp);
    2849     2943986 :     tp.t_self = *tid;
    2850             : 
    2851     2943988 : l1:
    2852             : 
    2853             :     /*
    2854             :      * If we didn't pin the visibility map page and the page has become all
    2855             :      * visible while we were busy locking the buffer, we'll have to unlock and
    2856             :      * re-lock, to avoid holding the buffer lock across an I/O.  That's a bit
    2857             :      * unfortunate, but hopefully shouldn't happen often.
    2858             :      */
    2859     2943988 :     if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))
    2860             :     {
    2861           0 :         LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    2862           0 :         visibilitymap_pin(relation, block, &vmbuffer);
    2863           0 :         LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    2864             :     }
    2865             : 
    2866     2943988 :     result = HeapTupleSatisfiesUpdate(&tp, cid, buffer);
    2867             : 
    2868     2943988 :     if (result == TM_Invisible)
    2869             :     {
    2870           0 :         UnlockReleaseBuffer(buffer);
    2871           0 :         ereport(ERROR,
    2872             :                 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
    2873             :                  errmsg("attempted to delete invisible tuple")));
    2874             :     }
    2875     2943988 :     else if (result == TM_BeingModified && wait)
    2876             :     {
    2877             :         TransactionId xwait;
    2878             :         uint16      infomask;
    2879             : 
    2880             :         /* must copy state data before unlocking buffer */
    2881       81110 :         xwait = HeapTupleHeaderGetRawXmax(tp.t_data);
    2882       81110 :         infomask = tp.t_data->t_infomask;
    2883             : 
    2884             :         /*
    2885             :          * Sleep until concurrent transaction ends -- except when there's a
    2886             :          * single locker and it's our own transaction.  Note we don't care
    2887             :          * which lock mode the locker has, because we need the strongest one.
    2888             :          *
    2889             :          * Before sleeping, we need to acquire tuple lock to establish our
    2890             :          * priority for the tuple (see heap_lock_tuple).  LockTuple will
    2891             :          * release us when we are next-in-line for the tuple.
    2892             :          *
    2893             :          * If we are forced to "start over" below, we keep the tuple lock;
    2894             :          * this arranges that we stay at the head of the line while rechecking
    2895             :          * tuple state.
    2896             :          */
    2897       81110 :         if (infomask & HEAP_XMAX_IS_MULTI)
    2898             :         {
    2899          16 :             bool        current_is_member = false;
    2900             : 
    2901          16 :             if (DoesMultiXactIdConflict((MultiXactId) xwait, infomask,
    2902             :                                         LockTupleExclusive, &current_is_member))
    2903             :             {
    2904          16 :                 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    2905             : 
    2906             :                 /*
    2907             :                  * Acquire the lock, if necessary (but skip it when we're
    2908             :                  * requesting a lock and already have one; avoids deadlock).
    2909             :                  */
    2910          16 :                 if (!current_is_member)
    2911          12 :                     heap_acquire_tuplock(relation, &(tp.t_self), LockTupleExclusive,
    2912             :                                          LockWaitBlock, &have_tuple_lock);
    2913             : 
    2914             :                 /* wait for multixact */
    2915          16 :                 MultiXactIdWait((MultiXactId) xwait, MultiXactStatusUpdate, infomask,
    2916             :                                 relation, &(tp.t_self), XLTW_Delete,
    2917             :                                 NULL);
    2918          16 :                 LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    2919             : 
    2920             :                 /*
    2921             :                  * If xwait had just locked the tuple then some other xact
    2922             :                  * could update this tuple before we get to this point.  Check
    2923             :                  * for xmax change, and start over if so.
    2924             :                  *
    2925             :                  * We also must start over if we didn't pin the VM page, and
    2926             :                  * the page has become all visible.
    2927             :                  */
    2928          32 :                 if ((vmbuffer == InvalidBuffer && PageIsAllVisible(page)) ||
    2929          32 :                     xmax_infomask_changed(tp.t_data->t_infomask, infomask) ||
    2930          16 :                     !TransactionIdEquals(HeapTupleHeaderGetRawXmax(tp.t_data),
    2931             :                                          xwait))
    2932           0 :                     goto l1;
    2933             :             }
    2934             : 
    2935             :             /*
    2936             :              * You might think the multixact is necessarily done here, but not
    2937             :              * so: it could have surviving members, namely our own xact or
    2938             :              * other subxacts of this backend.  It is legal for us to delete
    2939             :              * the tuple in either case, however (the latter case is
    2940             :              * essentially a situation of upgrading our former shared lock to
    2941             :              * exclusive).  We don't bother changing the on-disk hint bits
    2942             :              * since we are about to overwrite the xmax altogether.
    2943             :              */
    2944             :         }
    2945       81094 :         else if (!TransactionIdIsCurrentTransactionId(xwait))
    2946             :         {
    2947             :             /*
    2948             :              * Wait for regular transaction to end; but first, acquire tuple
    2949             :              * lock.
    2950             :              */
    2951          80 :             LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    2952          80 :             heap_acquire_tuplock(relation, &(tp.t_self), LockTupleExclusive,
    2953             :                                  LockWaitBlock, &have_tuple_lock);
    2954          80 :             XactLockTableWait(xwait, relation, &(tp.t_self), XLTW_Delete);
    2955          72 :             LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    2956             : 
    2957             :             /*
    2958             :              * xwait is done, but if xwait had just locked the tuple then some
    2959             :              * other xact could update this tuple before we get to this point.
    2960             :              * Check for xmax change, and start over if so.
    2961             :              *
    2962             :              * We also must start over if we didn't pin the VM page, and the
    2963             :              * page has become all visible.
    2964             :              */
    2965         144 :             if ((vmbuffer == InvalidBuffer && PageIsAllVisible(page)) ||
    2966         142 :                 xmax_infomask_changed(tp.t_data->t_infomask, infomask) ||
    2967          70 :                 !TransactionIdEquals(HeapTupleHeaderGetRawXmax(tp.t_data),
    2968             :                                      xwait))
    2969           2 :                 goto l1;
    2970             : 
    2971             :             /* Otherwise check if it committed or aborted */
    2972          70 :             UpdateXmaxHintBits(tp.t_data, buffer, xwait);
    2973             :         }
    2974             : 
    2975             :         /*
    2976             :          * We may overwrite if previous xmax aborted, or if it committed but
    2977             :          * only locked the tuple without updating it.
    2978             :          */
    2979      162172 :         if ((tp.t_data->t_infomask & HEAP_XMAX_INVALID) ||
    2980       81122 :             HEAP_XMAX_IS_LOCKED_ONLY(tp.t_data->t_infomask) ||
    2981          50 :             HeapTupleHeaderIsOnlyLocked(tp.t_data))
    2982       81058 :             result = TM_Ok;
    2983          42 :         else if (!ItemPointerEquals(&tp.t_self, &tp.t_data->t_ctid))
    2984          34 :             result = TM_Updated;
    2985             :         else
    2986           8 :             result = TM_Deleted;
    2987             :     }
    2988             : 
    2989             :     /* sanity check the result HeapTupleSatisfiesUpdate() and the logic above */
    2990             :     if (result != TM_Ok)
    2991             :     {
    2992             :         Assert(result == TM_SelfModified ||
    2993             :                result == TM_Updated ||
    2994             :                result == TM_Deleted ||
    2995             :                result == TM_BeingModified);
    2996             :         Assert(!(tp.t_data->t_infomask & HEAP_XMAX_INVALID));
    2997             :         Assert(result != TM_Updated ||
    2998             :                !ItemPointerEquals(&tp.t_self, &tp.t_data->t_ctid));
    2999             :     }
    3000             : 
    3001     2943978 :     if (crosscheck != InvalidSnapshot && result == TM_Ok)
    3002             :     {
    3003             :         /* Perform additional check for transaction-snapshot mode RI updates */
    3004           2 :         if (!HeapTupleSatisfiesVisibility(&tp, crosscheck, buffer))
    3005           2 :             result = TM_Updated;
    3006             :     }
    3007             : 
    3008     2943978 :     if (result != TM_Ok)
    3009             :     {
    3010         112 :         tmfd->ctid = tp.t_data->t_ctid;
    3011         112 :         tmfd->xmax = HeapTupleHeaderGetUpdateXid(tp.t_data);
    3012         112 :         if (result == TM_SelfModified)
    3013          42 :             tmfd->cmax = HeapTupleHeaderGetCmax(tp.t_data);
    3014             :         else
    3015          70 :             tmfd->cmax = InvalidCommandId;
    3016         112 :         UnlockReleaseBuffer(buffer);
    3017         112 :         if (have_tuple_lock)
    3018          42 :             UnlockTupleTuplock(relation, &(tp.t_self), LockTupleExclusive);
    3019         112 :         if (vmbuffer != InvalidBuffer)
    3020           0 :             ReleaseBuffer(vmbuffer);
    3021         112 :         return result;
    3022             :     }
    3023             : 
    3024             :     /*
    3025             :      * We're about to do the actual delete -- check for conflict first, to
    3026             :      * avoid possibly having to roll back work we've just done.
    3027             :      *
    3028             :      * This is safe without a recheck as long as there is no possibility of
    3029             :      * another process scanning the page between this check and the delete
    3030             :      * being visible to the scan (i.e., an exclusive buffer content lock is
    3031             :      * continuously held from this point until the tuple delete is visible).
    3032             :      */
    3033     2943866 :     CheckForSerializableConflictIn(relation, tid, BufferGetBlockNumber(buffer));
    3034             : 
    3035             :     /* replace cid with a combo CID if necessary */
    3036     2943838 :     HeapTupleHeaderAdjustCmax(tp.t_data, &cid, &iscombo);
    3037             : 
    3038             :     /*
    3039             :      * Compute replica identity tuple before entering the critical section so
    3040             :      * we don't PANIC upon a memory allocation failure.
    3041             :      */
    3042     2943838 :     old_key_tuple = ExtractReplicaIdentity(relation, &tp, true, &old_key_copied);
    3043             : 
    3044             :     /*
    3045             :      * If this is the first possibly-multixact-able operation in the current
    3046             :      * transaction, set my per-backend OldestMemberMXactId setting. We can be
    3047             :      * certain that the transaction will never become a member of any older
    3048             :      * MultiXactIds than that.  (We have to do this even if we end up just
    3049             :      * using our own TransactionId below, since some other backend could
    3050             :      * incorporate our XID into a MultiXact immediately afterwards.)
    3051             :      */
    3052     2943838 :     MultiXactIdSetOldestMember();
    3053             : 
    3054     2943838 :     compute_new_xmax_infomask(HeapTupleHeaderGetRawXmax(tp.t_data),
    3055     2943838 :                               tp.t_data->t_infomask, tp.t_data->t_infomask2,
    3056             :                               xid, LockTupleExclusive, true,
    3057             :                               &new_xmax, &new_infomask, &new_infomask2);
    3058             : 
    3059     2943838 :     START_CRIT_SECTION();
    3060             : 
    3061             :     /*
    3062             :      * If this transaction commits, the tuple will become DEAD sooner or
    3063             :      * later.  Set flag that this page is a candidate for pruning once our xid
    3064             :      * falls below the OldestXmin horizon.  If the transaction finally aborts,
    3065             :      * the subsequent page pruning will be a no-op and the hint will be
    3066             :      * cleared.
    3067             :      */
    3068     2943838 :     PageSetPrunable(page, xid);
    3069             : 
    3070     2943838 :     if (PageIsAllVisible(page))
    3071             :     {
    3072         446 :         all_visible_cleared = true;
    3073         446 :         PageClearAllVisible(page);
    3074         446 :         visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
    3075             :                             vmbuffer, VISIBILITYMAP_VALID_BITS);
    3076             :     }
    3077             : 
    3078             :     /* store transaction information of xact deleting the tuple */
    3079     2943838 :     tp.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED);
    3080     2943838 :     tp.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    3081     2943838 :     tp.t_data->t_infomask |= new_infomask;
    3082     2943838 :     tp.t_data->t_infomask2 |= new_infomask2;
    3083     2943838 :     HeapTupleHeaderClearHotUpdated(tp.t_data);
    3084     2943838 :     HeapTupleHeaderSetXmax(tp.t_data, new_xmax);
    3085     2943838 :     HeapTupleHeaderSetCmax(tp.t_data, cid, iscombo);
    3086             :     /* Make sure there is no forward chain link in t_ctid */
    3087     2943838 :     tp.t_data->t_ctid = tp.t_self;
    3088             : 
    3089             :     /* Signal that this is actually a move into another partition */
    3090     2943838 :     if (changingPart)
    3091         964 :         HeapTupleHeaderSetMovedPartitions(tp.t_data);
    3092             : 
    3093     2943838 :     MarkBufferDirty(buffer);
    3094             : 
    3095             :     /*
    3096             :      * XLOG stuff
    3097             :      *
    3098             :      * NB: heap_abort_speculative() uses the same xlog record and replay
    3099             :      * routines.
    3100             :      */
    3101     2943838 :     if (RelationNeedsWAL(relation))
    3102             :     {
    3103             :         xl_heap_delete xlrec;
    3104             :         xl_heap_header xlhdr;
    3105             :         XLogRecPtr  recptr;
    3106             : 
    3107             :         /*
    3108             :          * For logical decode we need combo CIDs to properly decode the
    3109             :          * catalog
    3110             :          */
    3111     2822620 :         if (RelationIsAccessibleInLogicalDecoding(relation))
    3112       12310 :             log_heap_new_cid(relation, &tp);
    3113             : 
    3114     2822620 :         xlrec.flags = 0;
    3115     2822620 :         if (all_visible_cleared)
    3116         446 :             xlrec.flags |= XLH_DELETE_ALL_VISIBLE_CLEARED;
    3117     2822620 :         if (changingPart)
    3118         964 :             xlrec.flags |= XLH_DELETE_IS_PARTITION_MOVE;
    3119     5645240 :         xlrec.infobits_set = compute_infobits(tp.t_data->t_infomask,
    3120     2822620 :                                               tp.t_data->t_infomask2);
    3121     2822620 :         xlrec.offnum = ItemPointerGetOffsetNumber(&tp.t_self);
    3122     2822620 :         xlrec.xmax = new_xmax;
    3123             : 
    3124     2822620 :         if (old_key_tuple != NULL)
    3125             :         {
    3126       94028 :             if (relation->rd_rel->relreplident == REPLICA_IDENTITY_FULL)
    3127         256 :                 xlrec.flags |= XLH_DELETE_CONTAINS_OLD_TUPLE;
    3128             :             else
    3129       93772 :                 xlrec.flags |= XLH_DELETE_CONTAINS_OLD_KEY;
    3130             :         }
    3131             : 
    3132     2822620 :         XLogBeginInsert();
    3133     2822620 :         XLogRegisterData(&xlrec, SizeOfHeapDelete);
    3134             : 
    3135     2822620 :         XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
    3136             : 
    3137             :         /*
    3138             :          * Log replica identity of the deleted tuple if there is one
    3139             :          */
    3140     2822620 :         if (old_key_tuple != NULL)
    3141             :         {
    3142       94028 :             xlhdr.t_infomask2 = old_key_tuple->t_data->t_infomask2;
    3143       94028 :             xlhdr.t_infomask = old_key_tuple->t_data->t_infomask;
    3144       94028 :             xlhdr.t_hoff = old_key_tuple->t_data->t_hoff;
    3145             : 
    3146       94028 :             XLogRegisterData(&xlhdr, SizeOfHeapHeader);
    3147       94028 :             XLogRegisterData((char *) old_key_tuple->t_data
    3148             :                              + SizeofHeapTupleHeader,
    3149       94028 :                              old_key_tuple->t_len
    3150             :                              - SizeofHeapTupleHeader);
    3151             :         }
    3152             : 
    3153             :         /* filtering by origin on a row level is much more efficient */
    3154     2822620 :         XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
    3155             : 
    3156     2822620 :         recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
    3157             : 
    3158     2822620 :         PageSetLSN(page, recptr);
    3159             :     }
    3160             : 
    3161     2943838 :     END_CRIT_SECTION();
    3162             : 
    3163     2943838 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    3164             : 
    3165     2943838 :     if (vmbuffer != InvalidBuffer)
    3166         446 :         ReleaseBuffer(vmbuffer);
    3167             : 
    3168             :     /*
    3169             :      * If the tuple has toasted out-of-line attributes, we need to delete
    3170             :      * those items too.  We have to do this before releasing the buffer
    3171             :      * because we need to look at the contents of the tuple, but it's OK to
    3172             :      * release the content lock on the buffer first.
    3173             :      */
    3174     2943838 :     if (relation->rd_rel->relkind != RELKIND_RELATION &&
    3175        4806 :         relation->rd_rel->relkind != RELKIND_MATVIEW)
    3176             :     {
    3177             :         /* toast table entries should never be recursively toasted */
    3178             :         Assert(!HeapTupleHasExternal(&tp));
    3179             :     }
    3180     2939052 :     else if (HeapTupleHasExternal(&tp))
    3181         586 :         heap_toast_delete(relation, &tp, false);
    3182             : 
    3183             :     /*
    3184             :      * Mark tuple for invalidation from system caches at next command
    3185             :      * boundary. We have to do this before releasing the buffer because we
    3186             :      * need to look at the contents of the tuple.
    3187             :      */
    3188     2943838 :     CacheInvalidateHeapTuple(relation, &tp, NULL);
    3189             : 
    3190             :     /* Now we can release the buffer */
    3191     2943838 :     ReleaseBuffer(buffer);
    3192             : 
    3193             :     /*
    3194             :      * Release the lmgr tuple lock, if we had it.
    3195             :      */
    3196     2943838 :     if (have_tuple_lock)
    3197          40 :         UnlockTupleTuplock(relation, &(tp.t_self), LockTupleExclusive);
    3198             : 
    3199     2943838 :     pgstat_count_heap_delete(relation);
    3200             : 
    3201     2943838 :     if (old_key_tuple != NULL && old_key_copied)
    3202       93774 :         heap_freetuple(old_key_tuple);
    3203             : 
    3204     2943838 :     return TM_Ok;
    3205             : }
    3206             : 
    3207             : /*
    3208             :  *  simple_heap_delete - delete a tuple
    3209             :  *
    3210             :  * This routine may be used to delete a tuple when concurrent updates of
    3211             :  * the target tuple are not expected (for example, because we have a lock
    3212             :  * on the relation associated with the tuple).  Any failure is reported
    3213             :  * via ereport().
    3214             :  */
    3215             : void
    3216     1217490 : simple_heap_delete(Relation relation, ItemPointer tid)
    3217             : {
    3218             :     TM_Result   result;
    3219             :     TM_FailureData tmfd;
    3220             : 
    3221     1217490 :     result = heap_delete(relation, tid,
    3222             :                          GetCurrentCommandId(true), InvalidSnapshot,
    3223             :                          true /* wait for commit */ ,
    3224             :                          &tmfd, false /* changingPart */ );
    3225     1217490 :     switch (result)
    3226             :     {
    3227           0 :         case TM_SelfModified:
    3228             :             /* Tuple was already updated in current command? */
    3229           0 :             elog(ERROR, "tuple already updated by self");
    3230             :             break;
    3231             : 
    3232     1217490 :         case TM_Ok:
    3233             :             /* done successfully */
    3234     1217490 :             break;
    3235             : 
    3236           0 :         case TM_Updated:
    3237           0 :             elog(ERROR, "tuple concurrently updated");
    3238             :             break;
    3239             : 
    3240           0 :         case TM_Deleted:
    3241           0 :             elog(ERROR, "tuple concurrently deleted");
    3242             :             break;
    3243             : 
    3244           0 :         default:
    3245           0 :             elog(ERROR, "unrecognized heap_delete status: %u", result);
    3246             :             break;
    3247             :     }
    3248     1217490 : }
    3249             : 
    3250             : /*
    3251             :  *  heap_update - replace a tuple
    3252             :  *
    3253             :  * See table_tuple_update() for an explanation of the parameters, except that
    3254             :  * this routine directly takes a tuple rather than a slot.
    3255             :  *
    3256             :  * In the failure cases, the routine fills *tmfd with the tuple's t_ctid,
    3257             :  * t_xmax (resolving a possible MultiXact, if necessary), and t_cmax (the last
    3258             :  * only for TM_SelfModified, since we cannot obtain cmax from a combo CID
    3259             :  * generated by another transaction).
    3260             :  */
    3261             : TM_Result
    3262      595790 : heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
    3263             :             CommandId cid, Snapshot crosscheck, bool wait,
    3264             :             TM_FailureData *tmfd, LockTupleMode *lockmode,
    3265             :             TU_UpdateIndexes *update_indexes)
    3266             : {
    3267             :     TM_Result   result;
    3268      595790 :     TransactionId xid = GetCurrentTransactionId();
    3269             :     Bitmapset  *hot_attrs;
    3270             :     Bitmapset  *sum_attrs;
    3271             :     Bitmapset  *key_attrs;
    3272             :     Bitmapset  *id_attrs;
    3273             :     Bitmapset  *interesting_attrs;
    3274             :     Bitmapset  *modified_attrs;
    3275             :     ItemId      lp;
    3276             :     HeapTupleData oldtup;
    3277             :     HeapTuple   heaptup;
    3278      595790 :     HeapTuple   old_key_tuple = NULL;
    3279      595790 :     bool        old_key_copied = false;
    3280             :     Page        page;
    3281             :     BlockNumber block;
    3282             :     MultiXactStatus mxact_status;
    3283             :     Buffer      buffer,
    3284             :                 newbuf,
    3285      595790 :                 vmbuffer = InvalidBuffer,
    3286      595790 :                 vmbuffer_new = InvalidBuffer;
    3287             :     bool        need_toast;
    3288             :     Size        newtupsize,
    3289             :                 pagefree;
    3290      595790 :     bool        have_tuple_lock = false;
    3291             :     bool        iscombo;
    3292      595790 :     bool        use_hot_update = false;
    3293      595790 :     bool        summarized_update = false;
    3294             :     bool        key_intact;
    3295      595790 :     bool        all_visible_cleared = false;
    3296      595790 :     bool        all_visible_cleared_new = false;
    3297             :     bool        checked_lockers;
    3298             :     bool        locker_remains;
    3299      595790 :     bool        id_has_external = false;
    3300             :     TransactionId xmax_new_tuple,
    3301             :                 xmax_old_tuple;
    3302             :     uint16      infomask_old_tuple,
    3303             :                 infomask2_old_tuple,
    3304             :                 infomask_new_tuple,
    3305             :                 infomask2_new_tuple;
    3306             : 
    3307             :     Assert(ItemPointerIsValid(otid));
    3308             : 
    3309             :     /* Cheap, simplistic check that the tuple matches the rel's rowtype. */
    3310             :     Assert(HeapTupleHeaderGetNatts(newtup->t_data) <=
    3311             :            RelationGetNumberOfAttributes(relation));
    3312             : 
    3313             :     /*
    3314             :      * Forbid this during a parallel operation, lest it allocate a combo CID.
    3315             :      * Other workers might need that combo CID for visibility checks, and we
    3316             :      * have no provision for broadcasting it to them.
    3317             :      */
    3318      595790 :     if (IsInParallelMode())
    3319           0 :         ereport(ERROR,
    3320             :                 (errcode(ERRCODE_INVALID_TRANSACTION_STATE),
    3321             :                  errmsg("cannot update tuples during a parallel operation")));
    3322             : 
    3323             : #ifdef USE_ASSERT_CHECKING
    3324             :     check_lock_if_inplace_updateable_rel(relation, otid, newtup);
    3325             : #endif
    3326             : 
    3327             :     /*
    3328             :      * Fetch the list of attributes to be checked for various operations.
    3329             :      *
    3330             :      * For HOT considerations, this is wasted effort if we fail to update or
    3331             :      * have to put the new tuple on a different page.  But we must compute the
    3332             :      * list before obtaining buffer lock --- in the worst case, if we are
    3333             :      * doing an update on one of the relevant system catalogs, we could
    3334             :      * deadlock if we try to fetch the list later.  In any case, the relcache
    3335             :      * caches the data so this is usually pretty cheap.
    3336             :      *
    3337             :      * We also need columns used by the replica identity and columns that are
    3338             :      * considered the "key" of rows in the table.
    3339             :      *
    3340             :      * Note that we get copies of each bitmap, so we need not worry about
    3341             :      * relcache flush happening midway through.
    3342             :      */
    3343      595790 :     hot_attrs = RelationGetIndexAttrBitmap(relation,
    3344             :                                            INDEX_ATTR_BITMAP_HOT_BLOCKING);
    3345      595790 :     sum_attrs = RelationGetIndexAttrBitmap(relation,
    3346             :                                            INDEX_ATTR_BITMAP_SUMMARIZED);
    3347      595790 :     key_attrs = RelationGetIndexAttrBitmap(relation, INDEX_ATTR_BITMAP_KEY);
    3348      595790 :     id_attrs = RelationGetIndexAttrBitmap(relation,
    3349             :                                           INDEX_ATTR_BITMAP_IDENTITY_KEY);
    3350      595790 :     interesting_attrs = NULL;
    3351      595790 :     interesting_attrs = bms_add_members(interesting_attrs, hot_attrs);
    3352      595790 :     interesting_attrs = bms_add_members(interesting_attrs, sum_attrs);
    3353      595790 :     interesting_attrs = bms_add_members(interesting_attrs, key_attrs);
    3354      595790 :     interesting_attrs = bms_add_members(interesting_attrs, id_attrs);
    3355             : 
    3356      595790 :     block = ItemPointerGetBlockNumber(otid);
    3357      595790 :     INJECTION_POINT("heap_update-before-pin");
    3358      595790 :     buffer = ReadBuffer(relation, block);
    3359      595790 :     page = BufferGetPage(buffer);
    3360             : 
    3361             :     /*
    3362             :      * Before locking the buffer, pin the visibility map page if it appears to
    3363             :      * be necessary.  Since we haven't got the lock yet, someone else might be
    3364             :      * in the middle of changing this, so we'll need to recheck after we have
    3365             :      * the lock.
    3366             :      */
    3367      595790 :     if (PageIsAllVisible(page))
    3368        3234 :         visibilitymap_pin(relation, block, &vmbuffer);
    3369             : 
    3370      595790 :     LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    3371             : 
    3372      595790 :     lp = PageGetItemId(page, ItemPointerGetOffsetNumber(otid));
    3373             : 
    3374             :     /*
    3375             :      * Usually, a buffer pin and/or snapshot blocks pruning of otid, ensuring
    3376             :      * we see LP_NORMAL here.  When the otid origin is a syscache, we may have
    3377             :      * neither a pin nor a snapshot.  Hence, we may see other LP_ states, each
    3378             :      * of which indicates concurrent pruning.
    3379             :      *
    3380             :      * Failing with TM_Updated would be most accurate.  However, unlike other
    3381             :      * TM_Updated scenarios, we don't know the successor ctid in LP_UNUSED and
    3382             :      * LP_DEAD cases.  While the distinction between TM_Updated and TM_Deleted
    3383             :      * does matter to SQL statements UPDATE and MERGE, those SQL statements
    3384             :      * hold a snapshot that ensures LP_NORMAL.  Hence, the choice between
    3385             :      * TM_Updated and TM_Deleted affects only the wording of error messages.
    3386             :      * Settle on TM_Deleted, for two reasons.  First, it avoids complicating
    3387             :      * the specification of when tmfd->ctid is valid.  Second, it creates
    3388             :      * error log evidence that we took this branch.
    3389             :      *
    3390             :      * Since it's possible to see LP_UNUSED at otid, it's also possible to see
    3391             :      * LP_NORMAL for a tuple that replaced LP_UNUSED.  If it's a tuple for an
    3392             :      * unrelated row, we'll fail with "duplicate key value violates unique".
    3393             :      * XXX if otid is the live, newer version of the newtup row, we'll discard
    3394             :      * changes originating in versions of this catalog row after the version
    3395             :      * the caller got from syscache.  See syscache-update-pruned.spec.
    3396             :      */
    3397      595790 :     if (!ItemIdIsNormal(lp))
    3398             :     {
    3399             :         Assert(RelationSupportsSysCache(RelationGetRelid(relation)));
    3400             : 
    3401           2 :         UnlockReleaseBuffer(buffer);
    3402             :         Assert(!have_tuple_lock);
    3403           2 :         if (vmbuffer != InvalidBuffer)
    3404           2 :             ReleaseBuffer(vmbuffer);
    3405           2 :         tmfd->ctid = *otid;
    3406           2 :         tmfd->xmax = InvalidTransactionId;
    3407           2 :         tmfd->cmax = InvalidCommandId;
    3408           2 :         *update_indexes = TU_None;
    3409             : 
    3410           2 :         bms_free(hot_attrs);
    3411           2 :         bms_free(sum_attrs);
    3412           2 :         bms_free(key_attrs);
    3413           2 :         bms_free(id_attrs);
    3414             :         /* modified_attrs not yet initialized */
    3415           2 :         bms_free(interesting_attrs);
    3416           2 :         return TM_Deleted;
    3417             :     }
    3418             : 
    3419             :     /*
    3420             :      * Fill in enough data in oldtup for HeapDetermineColumnsInfo to work
    3421             :      * properly.
    3422             :      */
    3423      595788 :     oldtup.t_tableOid = RelationGetRelid(relation);
    3424      595788 :     oldtup.t_data = (HeapTupleHeader) PageGetItem(page, lp);
    3425      595788 :     oldtup.t_len = ItemIdGetLength(lp);
    3426      595788 :     oldtup.t_self = *otid;
    3427             : 
    3428             :     /* the new tuple is ready, except for this: */
    3429      595788 :     newtup->t_tableOid = RelationGetRelid(relation);
    3430             : 
    3431             :     /*
    3432             :      * Determine columns modified by the update.  Additionally, identify
    3433             :      * whether any of the unmodified replica identity key attributes in the
    3434             :      * old tuple is externally stored or not.  This is required because for
    3435             :      * such attributes the flattened value won't be WAL logged as part of the
    3436             :      * new tuple so we must include it as part of the old_key_tuple.  See
    3437             :      * ExtractReplicaIdentity.
    3438             :      */
    3439      595788 :     modified_attrs = HeapDetermineColumnsInfo(relation, interesting_attrs,
    3440             :                                               id_attrs, &oldtup,
    3441             :                                               newtup, &id_has_external);
    3442             : 
    3443             :     /*
    3444             :      * If we're not updating any "key" column, we can grab a weaker lock type.
    3445             :      * This allows for more concurrency when we are running simultaneously
    3446             :      * with foreign key checks.
    3447             :      *
    3448             :      * Note that if a column gets detoasted while executing the update, but
    3449             :      * the value ends up being the same, this test will fail and we will use
    3450             :      * the stronger lock.  This is acceptable; the important case to optimize
    3451             :      * is updates that don't manipulate key columns, not those that
    3452             :      * serendipitously arrive at the same key values.
    3453             :      */
    3454      595788 :     if (!bms_overlap(modified_attrs, key_attrs))
    3455             :     {
    3456      587540 :         *lockmode = LockTupleNoKeyExclusive;
    3457      587540 :         mxact_status = MultiXactStatusNoKeyUpdate;
    3458      587540 :         key_intact = true;
    3459             : 
    3460             :         /*
    3461             :          * If this is the first possibly-multixact-able operation in the
    3462             :          * current transaction, set my per-backend OldestMemberMXactId
    3463             :          * setting. We can be certain that the transaction will never become a
    3464             :          * member of any older MultiXactIds than that.  (We have to do this
    3465             :          * even if we end up just using our own TransactionId below, since
    3466             :          * some other backend could incorporate our XID into a MultiXact
    3467             :          * immediately afterwards.)
    3468             :          */
    3469      587540 :         MultiXactIdSetOldestMember();
    3470             :     }
    3471             :     else
    3472             :     {
    3473        8248 :         *lockmode = LockTupleExclusive;
    3474        8248 :         mxact_status = MultiXactStatusUpdate;
    3475        8248 :         key_intact = false;
    3476             :     }
    3477             : 
    3478             :     /*
    3479             :      * Note: beyond this point, use oldtup not otid to refer to old tuple.
    3480             :      * otid may very well point at newtup->t_self, which we will overwrite
    3481             :      * with the new tuple's location, so there's great risk of confusion if we
    3482             :      * use otid anymore.
    3483             :      */
    3484             : 
    3485      595788 : l2:
    3486      595790 :     checked_lockers = false;
    3487      595790 :     locker_remains = false;
    3488      595790 :     result = HeapTupleSatisfiesUpdate(&oldtup, cid, buffer);
    3489             : 
    3490             :     /* see below about the "no wait" case */
    3491             :     Assert(result != TM_BeingModified || wait);
    3492             : 
    3493      595790 :     if (result == TM_Invisible)
    3494             :     {
    3495           0 :         UnlockReleaseBuffer(buffer);
    3496           0 :         ereport(ERROR,
    3497             :                 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
    3498             :                  errmsg("attempted to update invisible tuple")));
    3499             :     }
    3500      595790 :     else if (result == TM_BeingModified && wait)
    3501             :     {
    3502             :         TransactionId xwait;
    3503             :         uint16      infomask;
    3504       71882 :         bool        can_continue = false;
    3505             : 
    3506             :         /*
    3507             :          * XXX note that we don't consider the "no wait" case here.  This
    3508             :          * isn't a problem currently because no caller uses that case, but it
    3509             :          * should be fixed if such a caller is introduced.  It wasn't a
    3510             :          * problem previously because this code would always wait, but now
    3511             :          * that some tuple locks do not conflict with one of the lock modes we
    3512             :          * use, it is possible that this case is interesting to handle
    3513             :          * specially.
    3514             :          *
    3515             :          * This may cause failures with third-party code that calls
    3516             :          * heap_update directly.
    3517             :          */
    3518             : 
    3519             :         /* must copy state data before unlocking buffer */
    3520       71882 :         xwait = HeapTupleHeaderGetRawXmax(oldtup.t_data);
    3521       71882 :         infomask = oldtup.t_data->t_infomask;
    3522             : 
    3523             :         /*
    3524             :          * Now we have to do something about the existing locker.  If it's a
    3525             :          * multi, sleep on it; we might be awakened before it is completely
    3526             :          * gone (or even not sleep at all in some cases); we need to preserve
    3527             :          * it as locker, unless it is gone completely.
    3528             :          *
    3529             :          * If it's not a multi, we need to check for sleeping conditions
    3530             :          * before actually going to sleep.  If the update doesn't conflict
    3531             :          * with the locks, we just continue without sleeping (but making sure
    3532             :          * it is preserved).
    3533             :          *
    3534             :          * Before sleeping, we need to acquire tuple lock to establish our
    3535             :          * priority for the tuple (see heap_lock_tuple).  LockTuple will
    3536             :          * release us when we are next-in-line for the tuple.  Note we must
    3537             :          * not acquire the tuple lock until we're sure we're going to sleep;
    3538             :          * otherwise we're open for race conditions with other transactions
    3539             :          * holding the tuple lock which sleep on us.
    3540             :          *
    3541             :          * If we are forced to "start over" below, we keep the tuple lock;
    3542             :          * this arranges that we stay at the head of the line while rechecking
    3543             :          * tuple state.
    3544             :          */
    3545       71882 :         if (infomask & HEAP_XMAX_IS_MULTI)
    3546             :         {
    3547             :             TransactionId update_xact;
    3548             :             int         remain;
    3549         120 :             bool        current_is_member = false;
    3550             : 
    3551         120 :             if (DoesMultiXactIdConflict((MultiXactId) xwait, infomask,
    3552             :                                         *lockmode, &current_is_member))
    3553             :             {
    3554          16 :                 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    3555             : 
    3556             :                 /*
    3557             :                  * Acquire the lock, if necessary (but skip it when we're
    3558             :                  * requesting a lock and already have one; avoids deadlock).
    3559             :                  */
    3560          16 :                 if (!current_is_member)
    3561           0 :                     heap_acquire_tuplock(relation, &(oldtup.t_self), *lockmode,
    3562             :                                          LockWaitBlock, &have_tuple_lock);
    3563             : 
    3564             :                 /* wait for multixact */
    3565          16 :                 MultiXactIdWait((MultiXactId) xwait, mxact_status, infomask,
    3566             :                                 relation, &oldtup.t_self, XLTW_Update,
    3567             :                                 &remain);
    3568          16 :                 checked_lockers = true;
    3569          16 :                 locker_remains = remain != 0;
    3570          16 :                 LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    3571             : 
    3572             :                 /*
    3573             :                  * If xwait had just locked the tuple then some other xact
    3574             :                  * could update this tuple before we get to this point.  Check
    3575             :                  * for xmax change, and start over if so.
    3576             :                  */
    3577          16 :                 if (xmax_infomask_changed(oldtup.t_data->t_infomask,
    3578          16 :                                           infomask) ||
    3579          16 :                     !TransactionIdEquals(HeapTupleHeaderGetRawXmax(oldtup.t_data),
    3580             :                                          xwait))
    3581           0 :                     goto l2;
    3582             :             }
    3583             : 
    3584             :             /*
    3585             :              * Note that the multixact may not be done by now.  It could have
    3586             :              * surviving members; our own xact or other subxacts of this
    3587             :              * backend, and also any other concurrent transaction that locked
    3588             :              * the tuple with LockTupleKeyShare if we only got
    3589             :              * LockTupleNoKeyExclusive.  If this is the case, we have to be
    3590             :              * careful to mark the updated tuple with the surviving members in
    3591             :              * Xmax.
    3592             :              *
    3593             :              * Note that there could have been another update in the
    3594             :              * MultiXact. In that case, we need to check whether it committed
    3595             :              * or aborted. If it aborted we are safe to update it again;
    3596             :              * otherwise there is an update conflict, and we have to return
    3597             :              * TableTuple{Deleted, Updated} below.
    3598             :              *
    3599             :              * In the LockTupleExclusive case, we still need to preserve the
    3600             :              * surviving members: those would include the tuple locks we had
    3601             :              * before this one, which are important to keep in case this
    3602             :              * subxact aborts.
    3603             :              */
    3604         120 :             if (!HEAP_XMAX_IS_LOCKED_ONLY(oldtup.t_data->t_infomask))
    3605          16 :                 update_xact = HeapTupleGetUpdateXid(oldtup.t_data);
    3606             :             else
    3607         104 :                 update_xact = InvalidTransactionId;
    3608             : 
    3609             :             /*
    3610             :              * There was no UPDATE in the MultiXact; or it aborted. No
    3611             :              * TransactionIdIsInProgress() call needed here, since we called
    3612             :              * MultiXactIdWait() above.
    3613             :              */
    3614         136 :             if (!TransactionIdIsValid(update_xact) ||
    3615          16 :                 TransactionIdDidAbort(update_xact))
    3616         106 :                 can_continue = true;
    3617             :         }
    3618       71762 :         else if (TransactionIdIsCurrentTransactionId(xwait))
    3619             :         {
    3620             :             /*
    3621             :              * The only locker is ourselves; we can avoid grabbing the tuple
    3622             :              * lock here, but must preserve our locking information.
    3623             :              */
    3624       71578 :             checked_lockers = true;
    3625       71578 :             locker_remains = true;
    3626       71578 :             can_continue = true;
    3627             :         }
    3628         184 :         else if (HEAP_XMAX_IS_KEYSHR_LOCKED(infomask) && key_intact)
    3629             :         {
    3630             :             /*
    3631             :              * If it's just a key-share locker, and we're not changing the key
    3632             :              * columns, we don't need to wait for it to end; but we need to
    3633             :              * preserve it as locker.
    3634             :              */
    3635          58 :             checked_lockers = true;
    3636          58 :             locker_remains = true;
    3637          58 :             can_continue = true;
    3638             :         }
    3639             :         else
    3640             :         {
    3641             :             /*
    3642             :              * Wait for regular transaction to end; but first, acquire tuple
    3643             :              * lock.
    3644             :              */
    3645         126 :             LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    3646         126 :             heap_acquire_tuplock(relation, &(oldtup.t_self), *lockmode,
    3647             :                                  LockWaitBlock, &have_tuple_lock);
    3648         126 :             XactLockTableWait(xwait, relation, &oldtup.t_self,
    3649             :                               XLTW_Update);
    3650         126 :             checked_lockers = true;
    3651         126 :             LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    3652             : 
    3653             :             /*
    3654             :              * xwait is done, but if xwait had just locked the tuple then some
    3655             :              * other xact could update this tuple before we get to this point.
    3656             :              * Check for xmax change, and start over if so.
    3657             :              */
    3658         250 :             if (xmax_infomask_changed(oldtup.t_data->t_infomask, infomask) ||
    3659         124 :                 !TransactionIdEquals(xwait,
    3660             :                                      HeapTupleHeaderGetRawXmax(oldtup.t_data)))
    3661           2 :                 goto l2;
    3662             : 
    3663             :             /* Otherwise check if it committed or aborted */
    3664         124 :             UpdateXmaxHintBits(oldtup.t_data, buffer, xwait);
    3665         124 :             if (oldtup.t_data->t_infomask & HEAP_XMAX_INVALID)
    3666          30 :                 can_continue = true;
    3667             :         }
    3668             : 
    3669       71880 :         if (can_continue)
    3670       71772 :             result = TM_Ok;
    3671         108 :         else if (!ItemPointerEquals(&oldtup.t_self, &oldtup.t_data->t_ctid))
    3672          98 :             result = TM_Updated;
    3673             :         else
    3674          10 :             result = TM_Deleted;
    3675             :     }
    3676             : 
    3677             :     /* Sanity check the result HeapTupleSatisfiesUpdate() and the logic above */
    3678             :     if (result != TM_Ok)
    3679             :     {
    3680             :         Assert(result == TM_SelfModified ||
    3681             :                result == TM_Updated ||
    3682             :                result == TM_Deleted ||
    3683             :                result == TM_BeingModified);
    3684             :         Assert(!(oldtup.t_data->t_infomask & HEAP_XMAX_INVALID));
    3685             :         Assert(result != TM_Updated ||
    3686             :                !ItemPointerEquals(&oldtup.t_self, &oldtup.t_data->t_ctid));
    3687             :     }
    3688             : 
    3689      595788 :     if (crosscheck != InvalidSnapshot && result == TM_Ok)
    3690             :     {
    3691             :         /* Perform additional check for transaction-snapshot mode RI updates */
    3692           2 :         if (!HeapTupleSatisfiesVisibility(&oldtup, crosscheck, buffer))
    3693           2 :             result = TM_Updated;
    3694             :     }
    3695             : 
    3696      595788 :     if (result != TM_Ok)
    3697             :     {
    3698         304 :         tmfd->ctid = oldtup.t_data->t_ctid;
    3699         304 :         tmfd->xmax = HeapTupleHeaderGetUpdateXid(oldtup.t_data);
    3700         304 :         if (result == TM_SelfModified)
    3701         104 :             tmfd->cmax = HeapTupleHeaderGetCmax(oldtup.t_data);
    3702             :         else
    3703         200 :             tmfd->cmax = InvalidCommandId;
    3704         304 :         UnlockReleaseBuffer(buffer);
    3705         304 :         if (have_tuple_lock)
    3706          94 :             UnlockTupleTuplock(relation, &(oldtup.t_self), *lockmode);
    3707         304 :         if (vmbuffer != InvalidBuffer)
    3708           0 :             ReleaseBuffer(vmbuffer);
    3709         304 :         *update_indexes = TU_None;
    3710             : 
    3711         304 :         bms_free(hot_attrs);
    3712         304 :         bms_free(sum_attrs);
    3713         304 :         bms_free(key_attrs);
    3714         304 :         bms_free(id_attrs);
    3715         304 :         bms_free(modified_attrs);
    3716         304 :         bms_free(interesting_attrs);
    3717         304 :         return result;
    3718             :     }
    3719             : 
    3720             :     /*
    3721             :      * If we didn't pin the visibility map page and the page has become all
    3722             :      * visible while we were busy locking the buffer, or during some
    3723             :      * subsequent window during which we had it unlocked, we'll have to unlock
    3724             :      * and re-lock, to avoid holding the buffer lock across an I/O.  That's a
    3725             :      * bit unfortunate, especially since we'll now have to recheck whether the
    3726             :      * tuple has been locked or updated under us, but hopefully it won't
    3727             :      * happen very often.
    3728             :      */
    3729      595484 :     if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))
    3730             :     {
    3731           0 :         LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    3732           0 :         visibilitymap_pin(relation, block, &vmbuffer);
    3733           0 :         LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    3734           0 :         goto l2;
    3735             :     }
    3736             : 
    3737             :     /* Fill in transaction status data */
    3738             : 
    3739             :     /*
    3740             :      * If the tuple we're updating is locked, we need to preserve the locking
    3741             :      * info in the old tuple's Xmax.  Prepare a new Xmax value for this.
    3742             :      */
    3743      595484 :     compute_new_xmax_infomask(HeapTupleHeaderGetRawXmax(oldtup.t_data),
    3744      595484 :                               oldtup.t_data->t_infomask,
    3745      595484 :                               oldtup.t_data->t_infomask2,
    3746             :                               xid, *lockmode, true,
    3747             :                               &xmax_old_tuple, &infomask_old_tuple,
    3748             :                               &infomask2_old_tuple);
    3749             : 
    3750             :     /*
    3751             :      * And also prepare an Xmax value for the new copy of the tuple.  If there
    3752             :      * was no xmax previously, or there was one but all lockers are now gone,
    3753             :      * then use InvalidTransactionId; otherwise, get the xmax from the old
    3754             :      * tuple.  (In rare cases that might also be InvalidTransactionId and yet
    3755             :      * not have the HEAP_XMAX_INVALID bit set; that's fine.)
    3756             :      */
    3757      667226 :     if ((oldtup.t_data->t_infomask & HEAP_XMAX_INVALID) ||
    3758      143484 :         HEAP_LOCKED_UPGRADED(oldtup.t_data->t_infomask) ||
    3759       71638 :         (checked_lockers && !locker_remains))
    3760      523742 :         xmax_new_tuple = InvalidTransactionId;
    3761             :     else
    3762       71742 :         xmax_new_tuple = HeapTupleHeaderGetRawXmax(oldtup.t_data);
    3763             : 
    3764      595484 :     if (!TransactionIdIsValid(xmax_new_tuple))
    3765             :     {
    3766      523742 :         infomask_new_tuple = HEAP_XMAX_INVALID;
    3767      523742 :         infomask2_new_tuple = 0;
    3768             :     }
    3769             :     else
    3770             :     {
    3771             :         /*
    3772             :          * If we found a valid Xmax for the new tuple, then the infomask bits
    3773             :          * to use on the new tuple depend on what was there on the old one.
    3774             :          * Note that since we're doing an update, the only possibility is that
    3775             :          * the lockers had FOR KEY SHARE lock.
    3776             :          */
    3777       71742 :         if (oldtup.t_data->t_infomask & HEAP_XMAX_IS_MULTI)
    3778             :         {
    3779         106 :             GetMultiXactIdHintBits(xmax_new_tuple, &infomask_new_tuple,
    3780             :                                    &infomask2_new_tuple);
    3781             :         }
    3782             :         else
    3783             :         {
    3784       71636 :             infomask_new_tuple = HEAP_XMAX_KEYSHR_LOCK | HEAP_XMAX_LOCK_ONLY;
    3785       71636 :             infomask2_new_tuple = 0;
    3786             :         }
    3787             :     }
    3788             : 
    3789             :     /*
    3790             :      * Prepare the new tuple with the appropriate initial values of Xmin and
    3791             :      * Xmax, as well as initial infomask bits as computed above.
    3792             :      */
    3793      595484 :     newtup->t_data->t_infomask &= ~(HEAP_XACT_MASK);
    3794      595484 :     newtup->t_data->t_infomask2 &= ~(HEAP2_XACT_MASK);
    3795      595484 :     HeapTupleHeaderSetXmin(newtup->t_data, xid);
    3796      595484 :     HeapTupleHeaderSetCmin(newtup->t_data, cid);
    3797      595484 :     newtup->t_data->t_infomask |= HEAP_UPDATED | infomask_new_tuple;
    3798      595484 :     newtup->t_data->t_infomask2 |= infomask2_new_tuple;
    3799      595484 :     HeapTupleHeaderSetXmax(newtup->t_data, xmax_new_tuple);
    3800             : 
    3801             :     /*
    3802             :      * Replace cid with a combo CID if necessary.  Note that we already put
    3803             :      * the plain cid into the new tuple.
    3804             :      */
    3805      595484 :     HeapTupleHeaderAdjustCmax(oldtup.t_data, &cid, &iscombo);
    3806             : 
    3807             :     /*
    3808             :      * If the toaster needs to be activated, OR if the new tuple will not fit
    3809             :      * on the same page as the old, then we need to release the content lock
    3810             :      * (but not the pin!) on the old tuple's buffer while we are off doing
    3811             :      * TOAST and/or table-file-extension work.  We must mark the old tuple to
    3812             :      * show that it's locked, else other processes may try to update it
    3813             :      * themselves.
    3814             :      *
    3815             :      * We need to invoke the toaster if there are already any out-of-line
    3816             :      * toasted values present, or if the new tuple is over-threshold.
    3817             :      */
    3818      595484 :     if (relation->rd_rel->relkind != RELKIND_RELATION &&
    3819           0 :         relation->rd_rel->relkind != RELKIND_MATVIEW)
    3820             :     {
    3821             :         /* toast table entries should never be recursively toasted */
    3822             :         Assert(!HeapTupleHasExternal(&oldtup));
    3823             :         Assert(!HeapTupleHasExternal(newtup));
    3824           0 :         need_toast = false;
    3825             :     }
    3826             :     else
    3827     1785736 :         need_toast = (HeapTupleHasExternal(&oldtup) ||
    3828     1190252 :                       HeapTupleHasExternal(newtup) ||
    3829      594720 :                       newtup->t_len > TOAST_TUPLE_THRESHOLD);
    3830             : 
    3831      595484 :     pagefree = PageGetHeapFreeSpace(page);
    3832             : 
    3833      595484 :     newtupsize = MAXALIGN(newtup->t_len);
    3834             : 
    3835      595484 :     if (need_toast || newtupsize > pagefree)
    3836      293418 :     {
    3837             :         TransactionId xmax_lock_old_tuple;
    3838             :         uint16      infomask_lock_old_tuple,
    3839             :                     infomask2_lock_old_tuple;
    3840      293418 :         bool        cleared_all_frozen = false;
    3841             : 
    3842             :         /*
    3843             :          * To prevent concurrent sessions from updating the tuple, we have to
    3844             :          * temporarily mark it locked, while we release the page-level lock.
    3845             :          *
    3846             :          * To satisfy the rule that any xid potentially appearing in a buffer
    3847             :          * written out to disk, we unfortunately have to WAL log this
    3848             :          * temporary modification.  We can reuse xl_heap_lock for this
    3849             :          * purpose.  If we crash/error before following through with the
    3850             :          * actual update, xmax will be of an aborted transaction, allowing
    3851             :          * other sessions to proceed.
    3852             :          */
    3853             : 
    3854             :         /*
    3855             :          * Compute xmax / infomask appropriate for locking the tuple. This has
    3856             :          * to be done separately from the combo that's going to be used for
    3857             :          * updating, because the potentially created multixact would otherwise
    3858             :          * be wrong.
    3859             :          */
    3860      293418 :         compute_new_xmax_infomask(HeapTupleHeaderGetRawXmax(oldtup.t_data),
    3861      293418 :                                   oldtup.t_data->t_infomask,
    3862      293418 :                                   oldtup.t_data->t_infomask2,
    3863             :                                   xid, *lockmode, false,
    3864             :                                   &xmax_lock_old_tuple, &infomask_lock_old_tuple,
    3865             :                                   &infomask2_lock_old_tuple);
    3866             : 
    3867             :         Assert(HEAP_XMAX_IS_LOCKED_ONLY(infomask_lock_old_tuple));
    3868             : 
    3869      293418 :         START_CRIT_SECTION();
    3870             : 
    3871             :         /* Clear obsolete visibility flags ... */
    3872      293418 :         oldtup.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED);
    3873      293418 :         oldtup.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    3874      293418 :         HeapTupleClearHotUpdated(&oldtup);
    3875             :         /* ... and store info about transaction updating this tuple */
    3876             :         Assert(TransactionIdIsValid(xmax_lock_old_tuple));
    3877      293418 :         HeapTupleHeaderSetXmax(oldtup.t_data, xmax_lock_old_tuple);
    3878      293418 :         oldtup.t_data->t_infomask |= infomask_lock_old_tuple;
    3879      293418 :         oldtup.t_data->t_infomask2 |= infomask2_lock_old_tuple;
    3880      293418 :         HeapTupleHeaderSetCmax(oldtup.t_data, cid, iscombo);
    3881             : 
    3882             :         /* temporarily make it look not-updated, but locked */
    3883      293418 :         oldtup.t_data->t_ctid = oldtup.t_self;
    3884             : 
    3885             :         /*
    3886             :          * Clear all-frozen bit on visibility map if needed. We could
    3887             :          * immediately reset ALL_VISIBLE, but given that the WAL logging
    3888             :          * overhead would be unchanged, that doesn't seem necessarily
    3889             :          * worthwhile.
    3890             :          */
    3891      295126 :         if (PageIsAllVisible(page) &&
    3892        1708 :             visibilitymap_clear(relation, block, vmbuffer,
    3893             :                                 VISIBILITYMAP_ALL_FROZEN))
    3894        1402 :             cleared_all_frozen = true;
    3895             : 
    3896      293418 :         MarkBufferDirty(buffer);
    3897             : 
    3898      293418 :         if (RelationNeedsWAL(relation))
    3899             :         {
    3900             :             xl_heap_lock xlrec;
    3901             :             XLogRecPtr  recptr;
    3902             : 
    3903      273162 :             XLogBeginInsert();
    3904      273162 :             XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
    3905             : 
    3906      273162 :             xlrec.offnum = ItemPointerGetOffsetNumber(&oldtup.t_self);
    3907      273162 :             xlrec.xmax = xmax_lock_old_tuple;
    3908      546324 :             xlrec.infobits_set = compute_infobits(oldtup.t_data->t_infomask,
    3909      273162 :                                                   oldtup.t_data->t_infomask2);
    3910      273162 :             xlrec.flags =
    3911      273162 :                 cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
    3912      273162 :             XLogRegisterData(&xlrec, SizeOfHeapLock);
    3913      273162 :             recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
    3914      273162 :             PageSetLSN(page, recptr);
    3915             :         }
    3916             : 
    3917      293418 :         END_CRIT_SECTION();
    3918             : 
    3919      293418 :         LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    3920             : 
    3921             :         /*
    3922             :          * Let the toaster do its thing, if needed.
    3923             :          *
    3924             :          * Note: below this point, heaptup is the data we actually intend to
    3925             :          * store into the relation; newtup is the caller's original untoasted
    3926             :          * data.
    3927             :          */
    3928      293418 :         if (need_toast)
    3929             :         {
    3930             :             /* Note we always use WAL and FSM during updates */
    3931        3078 :             heaptup = heap_toast_insert_or_update(relation, newtup, &oldtup, 0);
    3932        3078 :             newtupsize = MAXALIGN(heaptup->t_len);
    3933             :         }
    3934             :         else
    3935      290340 :             heaptup = newtup;
    3936             : 
    3937             :         /*
    3938             :          * Now, do we need a new page for the tuple, or not?  This is a bit
    3939             :          * tricky since someone else could have added tuples to the page while
    3940             :          * we weren't looking.  We have to recheck the available space after
    3941             :          * reacquiring the buffer lock.  But don't bother to do that if the
    3942             :          * former amount of free space is still not enough; it's unlikely
    3943             :          * there's more free now than before.
    3944             :          *
    3945             :          * What's more, if we need to get a new page, we will need to acquire
    3946             :          * buffer locks on both old and new pages.  To avoid deadlock against
    3947             :          * some other backend trying to get the same two locks in the other
    3948             :          * order, we must be consistent about the order we get the locks in.
    3949             :          * We use the rule "lock the lower-numbered page of the relation
    3950             :          * first".  To implement this, we must do RelationGetBufferForTuple
    3951             :          * while not holding the lock on the old page, and we must rely on it
    3952             :          * to get the locks on both pages in the correct order.
    3953             :          *
    3954             :          * Another consideration is that we need visibility map page pin(s) if
    3955             :          * we will have to clear the all-visible flag on either page.  If we
    3956             :          * call RelationGetBufferForTuple, we rely on it to acquire any such
    3957             :          * pins; but if we don't, we have to handle that here.  Hence we need
    3958             :          * a loop.
    3959             :          */
    3960             :         for (;;)
    3961             :         {
    3962      293418 :             if (newtupsize > pagefree)
    3963             :             {
    3964             :                 /* It doesn't fit, must use RelationGetBufferForTuple. */
    3965      292490 :                 newbuf = RelationGetBufferForTuple(relation, heaptup->t_len,
    3966             :                                                    buffer, 0, NULL,
    3967             :                                                    &vmbuffer_new, &vmbuffer,
    3968             :                                                    0);
    3969             :                 /* We're all done. */
    3970      292490 :                 break;
    3971             :             }
    3972             :             /* Acquire VM page pin if needed and we don't have it. */
    3973         928 :             if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))
    3974           0 :                 visibilitymap_pin(relation, block, &vmbuffer);
    3975             :             /* Re-acquire the lock on the old tuple's page. */
    3976         928 :             LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    3977             :             /* Re-check using the up-to-date free space */
    3978         928 :             pagefree = PageGetHeapFreeSpace(page);
    3979         928 :             if (newtupsize > pagefree ||
    3980         928 :                 (vmbuffer == InvalidBuffer && PageIsAllVisible(page)))
    3981             :             {
    3982             :                 /*
    3983             :                  * Rats, it doesn't fit anymore, or somebody just now set the
    3984             :                  * all-visible flag.  We must now unlock and loop to avoid
    3985             :                  * deadlock.  Fortunately, this path should seldom be taken.
    3986             :                  */
    3987           0 :                 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    3988             :             }
    3989             :             else
    3990             :             {
    3991             :                 /* We're all done. */
    3992         928 :                 newbuf = buffer;
    3993         928 :                 break;
    3994             :             }
    3995             :         }
    3996             :     }
    3997             :     else
    3998             :     {
    3999             :         /* No TOAST work needed, and it'll fit on same page */
    4000      302066 :         newbuf = buffer;
    4001      302066 :         heaptup = newtup;
    4002             :     }
    4003             : 
    4004             :     /*
    4005             :      * We're about to do the actual update -- check for conflict first, to
    4006             :      * avoid possibly having to roll back work we've just done.
    4007             :      *
    4008             :      * This is safe without a recheck as long as there is no possibility of
    4009             :      * another process scanning the pages between this check and the update
    4010             :      * being visible to the scan (i.e., exclusive buffer content lock(s) are
    4011             :      * continuously held from this point until the tuple update is visible).
    4012             :      *
    4013             :      * For the new tuple the only check needed is at the relation level, but
    4014             :      * since both tuples are in the same relation and the check for oldtup
    4015             :      * will include checking the relation level, there is no benefit to a
    4016             :      * separate check for the new tuple.
    4017             :      */
    4018      595484 :     CheckForSerializableConflictIn(relation, &oldtup.t_self,
    4019             :                                    BufferGetBlockNumber(buffer));
    4020             : 
    4021             :     /*
    4022             :      * At this point newbuf and buffer are both pinned and locked, and newbuf
    4023             :      * has enough space for the new tuple.  If they are the same buffer, only
    4024             :      * one pin is held.
    4025             :      */
    4026             : 
    4027      595460 :     if (newbuf == buffer)
    4028             :     {
    4029             :         /*
    4030             :          * Since the new tuple is going into the same page, we might be able
    4031             :          * to do a HOT update.  Check if any of the index columns have been
    4032             :          * changed.
    4033             :          */
    4034      302970 :         if (!bms_overlap(modified_attrs, hot_attrs))
    4035             :         {
    4036      279132 :             use_hot_update = true;
    4037             : 
    4038             :             /*
    4039             :              * If none of the columns that are used in hot-blocking indexes
    4040             :              * were updated, we can apply HOT, but we do still need to check
    4041             :              * if we need to update the summarizing indexes, and update those
    4042             :              * indexes if the columns were updated, or we may fail to detect
    4043             :              * e.g. value bound changes in BRIN minmax indexes.
    4044             :              */
    4045      279132 :             if (bms_overlap(modified_attrs, sum_attrs))
    4046        3282 :                 summarized_update = true;
    4047             :         }
    4048             :     }
    4049             :     else
    4050             :     {
    4051             :         /* Set a hint that the old page could use prune/defrag */
    4052      292490 :         PageSetFull(page);
    4053             :     }
    4054             : 
    4055             :     /*
    4056             :      * Compute replica identity tuple before entering the critical section so
    4057             :      * we don't PANIC upon a memory allocation failure.
    4058             :      * ExtractReplicaIdentity() will return NULL if nothing needs to be
    4059             :      * logged.  Pass old key required as true only if the replica identity key
    4060             :      * columns are modified or it has external data.
    4061             :      */
    4062      595460 :     old_key_tuple = ExtractReplicaIdentity(relation, &oldtup,
    4063      595460 :                                            bms_overlap(modified_attrs, id_attrs) ||
    4064             :                                            id_has_external,
    4065             :                                            &old_key_copied);
    4066             : 
    4067             :     /* NO EREPORT(ERROR) from here till changes are logged */
    4068      595460 :     START_CRIT_SECTION();
    4069             : 
    4070             :     /*
    4071             :      * If this transaction commits, the old tuple will become DEAD sooner or
    4072             :      * later.  Set flag that this page is a candidate for pruning once our xid
    4073             :      * falls below the OldestXmin horizon.  If the transaction finally aborts,
    4074             :      * the subsequent page pruning will be a no-op and the hint will be
    4075             :      * cleared.
    4076             :      *
    4077             :      * XXX Should we set hint on newbuf as well?  If the transaction aborts,
    4078             :      * there would be a prunable tuple in the newbuf; but for now we choose
    4079             :      * not to optimize for aborts.  Note that heap_xlog_update must be kept in
    4080             :      * sync if this decision changes.
    4081             :      */
    4082      595460 :     PageSetPrunable(page, xid);
    4083             : 
    4084      595460 :     if (use_hot_update)
    4085             :     {
    4086             :         /* Mark the old tuple as HOT-updated */
    4087      279132 :         HeapTupleSetHotUpdated(&oldtup);
    4088             :         /* And mark the new tuple as heap-only */
    4089      279132 :         HeapTupleSetHeapOnly(heaptup);
    4090             :         /* Mark the caller's copy too, in case different from heaptup */
    4091      279132 :         HeapTupleSetHeapOnly(newtup);
    4092             :     }
    4093             :     else
    4094             :     {
    4095             :         /* Make sure tuples are correctly marked as not-HOT */
    4096      316328 :         HeapTupleClearHotUpdated(&oldtup);
    4097      316328 :         HeapTupleClearHeapOnly(heaptup);
    4098      316328 :         HeapTupleClearHeapOnly(newtup);
    4099             :     }
    4100             : 
    4101      595460 :     RelationPutHeapTuple(relation, newbuf, heaptup, false); /* insert new tuple */
    4102             : 
    4103             : 
    4104             :     /* Clear obsolete visibility flags, possibly set by ourselves above... */
    4105      595460 :     oldtup.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED);
    4106      595460 :     oldtup.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    4107             :     /* ... and store info about transaction updating this tuple */
    4108             :     Assert(TransactionIdIsValid(xmax_old_tuple));
    4109      595460 :     HeapTupleHeaderSetXmax(oldtup.t_data, xmax_old_tuple);
    4110      595460 :     oldtup.t_data->t_infomask |= infomask_old_tuple;
    4111      595460 :     oldtup.t_data->t_infomask2 |= infomask2_old_tuple;
    4112      595460 :     HeapTupleHeaderSetCmax(oldtup.t_data, cid, iscombo);
    4113             : 
    4114             :     /* record address of new tuple in t_ctid of old one */
    4115      595460 :     oldtup.t_data->t_ctid = heaptup->t_self;
    4116             : 
    4117             :     /* clear PD_ALL_VISIBLE flags, reset all visibilitymap bits */
    4118      595460 :     if (PageIsAllVisible(BufferGetPage(buffer)))
    4119             :     {
    4120        3232 :         all_visible_cleared = true;
    4121        3232 :         PageClearAllVisible(BufferGetPage(buffer));
    4122        3232 :         visibilitymap_clear(relation, BufferGetBlockNumber(buffer),
    4123             :                             vmbuffer, VISIBILITYMAP_VALID_BITS);
    4124             :     }
    4125      595460 :     if (newbuf != buffer && PageIsAllVisible(BufferGetPage(newbuf)))
    4126             :     {
    4127        1486 :         all_visible_cleared_new = true;
    4128        1486 :         PageClearAllVisible(BufferGetPage(newbuf));
    4129        1486 :         visibilitymap_clear(relation, BufferGetBlockNumber(newbuf),
    4130             :                             vmbuffer_new, VISIBILITYMAP_VALID_BITS);
    4131             :     }
    4132             : 
    4133      595460 :     if (newbuf != buffer)
    4134      292490 :         MarkBufferDirty(newbuf);
    4135      595460 :     MarkBufferDirty(buffer);
    4136             : 
    4137             :     /* XLOG stuff */
    4138      595460 :     if (RelationNeedsWAL(relation))
    4139             :     {
    4140             :         XLogRecPtr  recptr;
    4141             : 
    4142             :         /*
    4143             :          * For logical decoding we need combo CIDs to properly decode the
    4144             :          * catalog.
    4145             :          */
    4146      572742 :         if (RelationIsAccessibleInLogicalDecoding(relation))
    4147             :         {
    4148        5346 :             log_heap_new_cid(relation, &oldtup);
    4149        5346 :             log_heap_new_cid(relation, heaptup);
    4150             :         }
    4151             : 
    4152      572742 :         recptr = log_heap_update(relation, buffer,
    4153             :                                  newbuf, &oldtup, heaptup,
    4154             :                                  old_key_tuple,
    4155             :                                  all_visible_cleared,
    4156             :                                  all_visible_cleared_new);
    4157      572742 :         if (newbuf != buffer)
    4158             :         {
    4159      272246 :             PageSetLSN(BufferGetPage(newbuf), recptr);
    4160             :         }
    4161      572742 :         PageSetLSN(BufferGetPage(buffer), recptr);
    4162             :     }
    4163             : 
    4164      595460 :     END_CRIT_SECTION();
    4165             : 
    4166      595460 :     if (newbuf != buffer)
    4167      292490 :         LockBuffer(newbuf, BUFFER_LOCK_UNLOCK);
    4168      595460 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    4169             : 
    4170             :     /*
    4171             :      * Mark old tuple for invalidation from system caches at next command
    4172             :      * boundary, and mark the new tuple for invalidation in case we abort. We
    4173             :      * have to do this before releasing the buffer because oldtup is in the
    4174             :      * buffer.  (heaptup is all in local memory, but it's necessary to process
    4175             :      * both tuple versions in one call to inval.c so we can avoid redundant
    4176             :      * sinval messages.)
    4177             :      */
    4178      595460 :     CacheInvalidateHeapTuple(relation, &oldtup, heaptup);
    4179             : 
    4180             :     /* Now we can release the buffer(s) */
    4181      595460 :     if (newbuf != buffer)
    4182      292490 :         ReleaseBuffer(newbuf);
    4183      595460 :     ReleaseBuffer(buffer);
    4184      595460 :     if (BufferIsValid(vmbuffer_new))
    4185        1486 :         ReleaseBuffer(vmbuffer_new);
    4186      595460 :     if (BufferIsValid(vmbuffer))
    4187        3232 :         ReleaseBuffer(vmbuffer);
    4188             : 
    4189             :     /*
    4190             :      * Release the lmgr tuple lock, if we had it.
    4191             :      */
    4192      595460 :     if (have_tuple_lock)
    4193          30 :         UnlockTupleTuplock(relation, &(oldtup.t_self), *lockmode);
    4194             : 
    4195      595460 :     pgstat_count_heap_update(relation, use_hot_update, newbuf != buffer);
    4196             : 
    4197             :     /*
    4198             :      * If heaptup is a private copy, release it.  Don't forget to copy t_self
    4199             :      * back to the caller's image, too.
    4200             :      */
    4201      595460 :     if (heaptup != newtup)
    4202             :     {
    4203        2984 :         newtup->t_self = heaptup->t_self;
    4204        2984 :         heap_freetuple(heaptup);
    4205             :     }
    4206             : 
    4207             :     /*
    4208             :      * If it is a HOT update, the update may still need to update summarized
    4209             :      * indexes, lest we fail to update those summaries and get incorrect
    4210             :      * results (for example, minmax bounds of the block may change with this
    4211             :      * update).
    4212             :      */
    4213      595460 :     if (use_hot_update)
    4214             :     {
    4215      279132 :         if (summarized_update)
    4216        3282 :             *update_indexes = TU_Summarizing;
    4217             :         else
    4218      275850 :             *update_indexes = TU_None;
    4219             :     }
    4220             :     else
    4221      316328 :         *update_indexes = TU_All;
    4222             : 
    4223      595460 :     if (old_key_tuple != NULL && old_key_copied)
    4224         166 :         heap_freetuple(old_key_tuple);
    4225             : 
    4226      595460 :     bms_free(hot_attrs);
    4227      595460 :     bms_free(sum_attrs);
    4228      595460 :     bms_free(key_attrs);
    4229      595460 :     bms_free(id_attrs);
    4230      595460 :     bms_free(modified_attrs);
    4231      595460 :     bms_free(interesting_attrs);
    4232             : 
    4233      595460 :     return TM_Ok;
    4234             : }
    4235             : 
    4236             : #ifdef USE_ASSERT_CHECKING
    4237             : /*
    4238             :  * Confirm adequate lock held during heap_update(), per rules from
    4239             :  * README.tuplock section "Locking to write inplace-updated tables".
    4240             :  */
    4241             : static void
    4242             : check_lock_if_inplace_updateable_rel(Relation relation,
    4243             :                                      ItemPointer otid,
    4244             :                                      HeapTuple newtup)
    4245             : {
    4246             :     /* LOCKTAG_TUPLE acceptable for any catalog */
    4247             :     switch (RelationGetRelid(relation))
    4248             :     {
    4249             :         case RelationRelationId:
    4250             :         case DatabaseRelationId:
    4251             :             {
    4252             :                 LOCKTAG     tuptag;
    4253             : 
    4254             :                 SET_LOCKTAG_TUPLE(tuptag,
    4255             :                                   relation->rd_lockInfo.lockRelId.dbId,
    4256             :                                   relation->rd_lockInfo.lockRelId.relId,
    4257             :                                   ItemPointerGetBlockNumber(otid),
    4258             :                                   ItemPointerGetOffsetNumber(otid));
    4259             :                 if (LockHeldByMe(&tuptag, InplaceUpdateTupleLock, false))
    4260             :                     return;
    4261             :             }
    4262             :             break;
    4263             :         default:
    4264             :             Assert(!IsInplaceUpdateRelation(relation));
    4265             :             return;
    4266             :     }
    4267             : 
    4268             :     switch (RelationGetRelid(relation))
    4269             :     {
    4270             :         case RelationRelationId:
    4271             :             {
    4272             :                 /* LOCKTAG_TUPLE or LOCKTAG_RELATION ok */
    4273             :                 Form_pg_class classForm = (Form_pg_class) GETSTRUCT(newtup);
    4274             :                 Oid         relid = classForm->oid;
    4275             :                 Oid         dbid;
    4276             :                 LOCKTAG     tag;
    4277             : 
    4278             :                 if (IsSharedRelation(relid))
    4279             :                     dbid = InvalidOid;
    4280             :                 else
    4281             :                     dbid = MyDatabaseId;
    4282             : 
    4283             :                 if (classForm->relkind == RELKIND_INDEX)
    4284             :                 {
    4285             :                     Relation    irel = index_open(relid, AccessShareLock);
    4286             : 
    4287             :                     SET_LOCKTAG_RELATION(tag, dbid, irel->rd_index->indrelid);
    4288             :                     index_close(irel, AccessShareLock);
    4289             :                 }
    4290             :                 else
    4291             :                     SET_LOCKTAG_RELATION(tag, dbid, relid);
    4292             : 
    4293             :                 if (!LockHeldByMe(&tag, ShareUpdateExclusiveLock, false) &&
    4294             :                     !LockHeldByMe(&tag, ShareRowExclusiveLock, true))
    4295             :                     elog(WARNING,
    4296             :                          "missing lock for relation \"%s\" (OID %u, relkind %c) @ TID (%u,%u)",
    4297             :                          NameStr(classForm->relname),
    4298             :                          relid,
    4299             :                          classForm->relkind,
    4300             :                          ItemPointerGetBlockNumber(otid),
    4301             :                          ItemPointerGetOffsetNumber(otid));
    4302             :             }
    4303             :             break;
    4304             :         case DatabaseRelationId:
    4305             :             {
    4306             :                 /* LOCKTAG_TUPLE required */
    4307             :                 Form_pg_database dbForm = (Form_pg_database) GETSTRUCT(newtup);
    4308             : 
    4309             :                 elog(WARNING,
    4310             :                      "missing lock on database \"%s\" (OID %u) @ TID (%u,%u)",
    4311             :                      NameStr(dbForm->datname),
    4312             :                      dbForm->oid,
    4313             :                      ItemPointerGetBlockNumber(otid),
    4314             :                      ItemPointerGetOffsetNumber(otid));
    4315             :             }
    4316             :             break;
    4317             :     }
    4318             : }
    4319             : 
    4320             : /*
    4321             :  * Confirm adequate relation lock held, per rules from README.tuplock section
    4322             :  * "Locking to write inplace-updated tables".
    4323             :  */
    4324             : static void
    4325             : check_inplace_rel_lock(HeapTuple oldtup)
    4326             : {
    4327             :     Form_pg_class classForm = (Form_pg_class) GETSTRUCT(oldtup);
    4328             :     Oid         relid = classForm->oid;
    4329             :     Oid         dbid;
    4330             :     LOCKTAG     tag;
    4331             : 
    4332             :     if (IsSharedRelation(relid))
    4333             :         dbid = InvalidOid;
    4334             :     else
    4335             :         dbid = MyDatabaseId;
    4336             : 
    4337             :     if (classForm->relkind == RELKIND_INDEX)
    4338             :     {
    4339             :         Relation    irel = index_open(relid, AccessShareLock);
    4340             : 
    4341             :         SET_LOCKTAG_RELATION(tag, dbid, irel->rd_index->indrelid);
    4342             :         index_close(irel, AccessShareLock);
    4343             :     }
    4344             :     else
    4345             :         SET_LOCKTAG_RELATION(tag, dbid, relid);
    4346             : 
    4347             :     if (!LockHeldByMe(&tag, ShareUpdateExclusiveLock, true))
    4348             :         elog(WARNING,
    4349             :              "missing lock for relation \"%s\" (OID %u, relkind %c) @ TID (%u,%u)",
    4350             :              NameStr(classForm->relname),
    4351             :              relid,
    4352             :              classForm->relkind,
    4353             :              ItemPointerGetBlockNumber(&oldtup->t_self),
    4354             :              ItemPointerGetOffsetNumber(&oldtup->t_self));
    4355             : }
    4356             : #endif
    4357             : 
    4358             : /*
    4359             :  * Check if the specified attribute's values are the same.  Subroutine for
    4360             :  * HeapDetermineColumnsInfo.
    4361             :  */
    4362             : static bool
    4363     1417750 : heap_attr_equals(TupleDesc tupdesc, int attrnum, Datum value1, Datum value2,
    4364             :                  bool isnull1, bool isnull2)
    4365             : {
    4366             :     /*
    4367             :      * If one value is NULL and other is not, then they are certainly not
    4368             :      * equal
    4369             :      */
    4370     1417750 :     if (isnull1 != isnull2)
    4371          90 :         return false;
    4372             : 
    4373             :     /*
    4374             :      * If both are NULL, they can be considered equal.
    4375             :      */
    4376     1417660 :     if (isnull1)
    4377        9982 :         return true;
    4378             : 
    4379             :     /*
    4380             :      * We do simple binary comparison of the two datums.  This may be overly
    4381             :      * strict because there can be multiple binary representations for the
    4382             :      * same logical value.  But we should be OK as long as there are no false
    4383             :      * positives.  Using a type-specific equality operator is messy because
    4384             :      * there could be multiple notions of equality in different operator
    4385             :      * classes; furthermore, we cannot safely invoke user-defined functions
    4386             :      * while holding exclusive buffer lock.
    4387             :      */
    4388     1407678 :     if (attrnum <= 0)
    4389             :     {
    4390             :         /* The only allowed system columns are OIDs, so do this */
    4391           0 :         return (DatumGetObjectId(value1) == DatumGetObjectId(value2));
    4392             :     }
    4393             :     else
    4394             :     {
    4395             :         CompactAttribute *att;
    4396             : 
    4397             :         Assert(attrnum <= tupdesc->natts);
    4398     1407678 :         att = TupleDescCompactAttr(tupdesc, attrnum - 1);
    4399     1407678 :         return datumIsEqual(value1, value2, att->attbyval, att->attlen);
    4400             :     }
    4401             : }
    4402             : 
    4403             : /*
    4404             :  * Check which columns are being updated.
    4405             :  *
    4406             :  * Given an updated tuple, determine (and return into the output bitmapset),
    4407             :  * from those listed as interesting, the set of columns that changed.
    4408             :  *
    4409             :  * has_external indicates if any of the unmodified attributes (from those
    4410             :  * listed as interesting) of the old tuple is a member of external_cols and is
    4411             :  * stored externally.
    4412             :  */
    4413             : static Bitmapset *
    4414      595788 : HeapDetermineColumnsInfo(Relation relation,
    4415             :                          Bitmapset *interesting_cols,
    4416             :                          Bitmapset *external_cols,
    4417             :                          HeapTuple oldtup, HeapTuple newtup,
    4418             :                          bool *has_external)
    4419             : {
    4420             :     int         attidx;
    4421      595788 :     Bitmapset  *modified = NULL;
    4422      595788 :     TupleDesc   tupdesc = RelationGetDescr(relation);
    4423             : 
    4424      595788 :     attidx = -1;
    4425     2013538 :     while ((attidx = bms_next_member(interesting_cols, attidx)) >= 0)
    4426             :     {
    4427             :         /* attidx is zero-based, attrnum is the normal attribute number */
    4428     1417750 :         AttrNumber  attrnum = attidx + FirstLowInvalidHeapAttributeNumber;
    4429             :         Datum       value1,
    4430             :                     value2;
    4431             :         bool        isnull1,
    4432             :                     isnull2;
    4433             : 
    4434             :         /*
    4435             :          * If it's a whole-tuple reference, say "not equal".  It's not really
    4436             :          * worth supporting this case, since it could only succeed after a
    4437             :          * no-op update, which is hardly a case worth optimizing for.
    4438             :          */
    4439     1417750 :         if (attrnum == 0)
    4440             :         {
    4441           0 :             modified = bms_add_member(modified, attidx);
    4442     1356462 :             continue;
    4443             :         }
    4444             : 
    4445             :         /*
    4446             :          * Likewise, automatically say "not equal" for any system attribute
    4447             :          * other than tableOID; we cannot expect these to be consistent in a
    4448             :          * HOT chain, or even to be set correctly yet in the new tuple.
    4449             :          */
    4450     1417750 :         if (attrnum < 0)
    4451             :         {
    4452           0 :             if (attrnum != TableOidAttributeNumber)
    4453             :             {
    4454           0 :                 modified = bms_add_member(modified, attidx);
    4455           0 :                 continue;
    4456             :             }
    4457             :         }
    4458             : 
    4459             :         /*
    4460             :          * Extract the corresponding values.  XXX this is pretty inefficient
    4461             :          * if there are many indexed columns.  Should we do a single
    4462             :          * heap_deform_tuple call on each tuple, instead?   But that doesn't
    4463             :          * work for system columns ...
    4464             :          */
    4465     1417750 :         value1 = heap_getattr(oldtup, attrnum, tupdesc, &isnull1);
    4466     1417750 :         value2 = heap_getattr(newtup, attrnum, tupdesc, &isnull2);
    4467             : 
    4468     1417750 :         if (!heap_attr_equals(tupdesc, attrnum, value1,
    4469             :                               value2, isnull1, isnull2))
    4470             :         {
    4471       53212 :             modified = bms_add_member(modified, attidx);
    4472       53212 :             continue;
    4473             :         }
    4474             : 
    4475             :         /*
    4476             :          * No need to check attributes that can't be stored externally. Note
    4477             :          * that system attributes can't be stored externally.
    4478             :          */
    4479     1364538 :         if (attrnum < 0 || isnull1 ||
    4480     1354556 :             TupleDescCompactAttr(tupdesc, attrnum - 1)->attlen != -1)
    4481     1303250 :             continue;
    4482             : 
    4483             :         /*
    4484             :          * Check if the old tuple's attribute is stored externally and is a
    4485             :          * member of external_cols.
    4486             :          */
    4487       61298 :         if (VARATT_IS_EXTERNAL((struct varlena *) DatumGetPointer(value1)) &&
    4488          10 :             bms_is_member(attidx, external_cols))
    4489           4 :             *has_external = true;
    4490             :     }
    4491             : 
    4492      595788 :     return modified;
    4493             : }
    4494             : 
    4495             : /*
    4496             :  *  simple_heap_update - replace a tuple
    4497             :  *
    4498             :  * This routine may be used to update a tuple when concurrent updates of
    4499             :  * the target tuple are not expected (for example, because we have a lock
    4500             :  * on the relation associated with the tuple).  Any failure is reported
    4501             :  * via ereport().
    4502             :  */
    4503             : void
    4504      210306 : simple_heap_update(Relation relation, ItemPointer otid, HeapTuple tup,
    4505             :                    TU_UpdateIndexes *update_indexes)
    4506             : {
    4507             :     TM_Result   result;
    4508             :     TM_FailureData tmfd;
    4509             :     LockTupleMode lockmode;
    4510             : 
    4511      210306 :     result = heap_update(relation, otid, tup,
    4512             :                          GetCurrentCommandId(true), InvalidSnapshot,
    4513             :                          true /* wait for commit */ ,
    4514             :                          &tmfd, &lockmode, update_indexes);
    4515      210306 :     switch (result)
    4516             :     {
    4517           0 :         case TM_SelfModified:
    4518             :             /* Tuple was already updated in current command? */
    4519           0 :             elog(ERROR, "tuple already updated by self");
    4520             :             break;
    4521             : 
    4522      210304 :         case TM_Ok:
    4523             :             /* done successfully */
    4524      210304 :             break;
    4525             : 
    4526           0 :         case TM_Updated:
    4527           0 :             elog(ERROR, "tuple concurrently updated");
    4528             :             break;
    4529             : 
    4530           2 :         case TM_Deleted:
    4531           2 :             elog(ERROR, "tuple concurrently deleted");
    4532             :             break;
    4533             : 
    4534           0 :         default:
    4535           0 :             elog(ERROR, "unrecognized heap_update status: %u", result);
    4536             :             break;
    4537             :     }
    4538      210304 : }
    4539             : 
    4540             : 
    4541             : /*
    4542             :  * Return the MultiXactStatus corresponding to the given tuple lock mode.
    4543             :  */
    4544             : static MultiXactStatus
    4545        2414 : get_mxact_status_for_lock(LockTupleMode mode, bool is_update)
    4546             : {
    4547             :     int         retval;
    4548             : 
    4549        2414 :     if (is_update)
    4550         192 :         retval = tupleLockExtraInfo[mode].updstatus;
    4551             :     else
    4552        2222 :         retval = tupleLockExtraInfo[mode].lockstatus;
    4553             : 
    4554        2414 :     if (retval == -1)
    4555           0 :         elog(ERROR, "invalid lock tuple mode %d/%s", mode,
    4556             :              is_update ? "true" : "false");
    4557             : 
    4558        2414 :     return (MultiXactStatus) retval;
    4559             : }
    4560             : 
    4561             : /*
    4562             :  *  heap_lock_tuple - lock a tuple in shared or exclusive mode
    4563             :  *
    4564             :  * Note that this acquires a buffer pin, which the caller must release.
    4565             :  *
    4566             :  * Input parameters:
    4567             :  *  relation: relation containing tuple (caller must hold suitable lock)
    4568             :  *  tid: TID of tuple to lock
    4569             :  *  cid: current command ID (used for visibility test, and stored into
    4570             :  *      tuple's cmax if lock is successful)
    4571             :  *  mode: indicates if shared or exclusive tuple lock is desired
    4572             :  *  wait_policy: what to do if tuple lock is not available
    4573             :  *  follow_updates: if true, follow the update chain to also lock descendant
    4574             :  *      tuples.
    4575             :  *
    4576             :  * Output parameters:
    4577             :  *  *tuple: all fields filled in
    4578             :  *  *buffer: set to buffer holding tuple (pinned but not locked at exit)
    4579             :  *  *tmfd: filled in failure cases (see below)
    4580             :  *
    4581             :  * Function results are the same as the ones for table_tuple_lock().
    4582             :  *
    4583             :  * In the failure cases other than TM_Invisible, the routine fills
    4584             :  * *tmfd with the tuple's t_ctid, t_xmax (resolving a possible MultiXact,
    4585             :  * if necessary), and t_cmax (the last only for TM_SelfModified,
    4586             :  * since we cannot obtain cmax from a combo CID generated by another
    4587             :  * transaction).
    4588             :  * See comments for struct TM_FailureData for additional info.
    4589             :  *
    4590             :  * See README.tuplock for a thorough explanation of this mechanism.
    4591             :  */
    4592             : TM_Result
    4593      169690 : heap_lock_tuple(Relation relation, HeapTuple tuple,
    4594             :                 CommandId cid, LockTupleMode mode, LockWaitPolicy wait_policy,
    4595             :                 bool follow_updates,
    4596             :                 Buffer *buffer, TM_FailureData *tmfd)
    4597             : {
    4598             :     TM_Result   result;
    4599      169690 :     ItemPointer tid = &(tuple->t_self);
    4600             :     ItemId      lp;
    4601             :     Page        page;
    4602      169690 :     Buffer      vmbuffer = InvalidBuffer;
    4603             :     BlockNumber block;
    4604             :     TransactionId xid,
    4605             :                 xmax;
    4606             :     uint16      old_infomask,
    4607             :                 new_infomask,
    4608             :                 new_infomask2;
    4609      169690 :     bool        first_time = true;
    4610      169690 :     bool        skip_tuple_lock = false;
    4611      169690 :     bool        have_tuple_lock = false;
    4612      169690 :     bool        cleared_all_frozen = false;
    4613             : 
    4614      169690 :     *buffer = ReadBuffer(relation, ItemPointerGetBlockNumber(tid));
    4615      169690 :     block = ItemPointerGetBlockNumber(tid);
    4616             : 
    4617             :     /*
    4618             :      * Before locking the buffer, pin the visibility map page if it appears to
    4619             :      * be necessary.  Since we haven't got the lock yet, someone else might be
    4620             :      * in the middle of changing this, so we'll need to recheck after we have
    4621             :      * the lock.
    4622             :      */
    4623      169690 :     if (PageIsAllVisible(BufferGetPage(*buffer)))
    4624        3318 :         visibilitymap_pin(relation, block, &vmbuffer);
    4625             : 
    4626      169690 :     LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4627             : 
    4628      169690 :     page = BufferGetPage(*buffer);
    4629      169690 :     lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tid));
    4630             :     Assert(ItemIdIsNormal(lp));
    4631             : 
    4632      169690 :     tuple->t_data = (HeapTupleHeader) PageGetItem(page, lp);
    4633      169690 :     tuple->t_len = ItemIdGetLength(lp);
    4634      169690 :     tuple->t_tableOid = RelationGetRelid(relation);
    4635             : 
    4636      169716 : l3:
    4637      169716 :     result = HeapTupleSatisfiesUpdate(tuple, cid, *buffer);
    4638             : 
    4639      169716 :     if (result == TM_Invisible)
    4640             :     {
    4641             :         /*
    4642             :          * This is possible, but only when locking a tuple for ON CONFLICT
    4643             :          * UPDATE.  We return this value here rather than throwing an error in
    4644             :          * order to give that case the opportunity to throw a more specific
    4645             :          * error.
    4646             :          */
    4647          24 :         result = TM_Invisible;
    4648          24 :         goto out_locked;
    4649             :     }
    4650      169692 :     else if (result == TM_BeingModified ||
    4651      154004 :              result == TM_Updated ||
    4652             :              result == TM_Deleted)
    4653             :     {
    4654             :         TransactionId xwait;
    4655             :         uint16      infomask;
    4656             :         uint16      infomask2;
    4657             :         bool        require_sleep;
    4658             :         ItemPointerData t_ctid;
    4659             : 
    4660             :         /* must copy state data before unlocking buffer */
    4661       15690 :         xwait = HeapTupleHeaderGetRawXmax(tuple->t_data);
    4662       15690 :         infomask = tuple->t_data->t_infomask;
    4663       15690 :         infomask2 = tuple->t_data->t_infomask2;
    4664       15690 :         ItemPointerCopy(&tuple->t_data->t_ctid, &t_ctid);
    4665             : 
    4666       15690 :         LockBuffer(*buffer, BUFFER_LOCK_UNLOCK);
    4667             : 
    4668             :         /*
    4669             :          * If any subtransaction of the current top transaction already holds
    4670             :          * a lock as strong as or stronger than what we're requesting, we
    4671             :          * effectively hold the desired lock already.  We *must* succeed
    4672             :          * without trying to take the tuple lock, else we will deadlock
    4673             :          * against anyone wanting to acquire a stronger lock.
    4674             :          *
    4675             :          * Note we only do this the first time we loop on the HTSU result;
    4676             :          * there is no point in testing in subsequent passes, because
    4677             :          * evidently our own transaction cannot have acquired a new lock after
    4678             :          * the first time we checked.
    4679             :          */
    4680       15690 :         if (first_time)
    4681             :         {
    4682       15672 :             first_time = false;
    4683             : 
    4684       15672 :             if (infomask & HEAP_XMAX_IS_MULTI)
    4685             :             {
    4686             :                 int         i;
    4687             :                 int         nmembers;
    4688             :                 MultiXactMember *members;
    4689             : 
    4690             :                 /*
    4691             :                  * We don't need to allow old multixacts here; if that had
    4692             :                  * been the case, HeapTupleSatisfiesUpdate would have returned
    4693             :                  * MayBeUpdated and we wouldn't be here.
    4694             :                  */
    4695             :                 nmembers =
    4696         176 :                     GetMultiXactIdMembers(xwait, &members, false,
    4697         176 :                                           HEAP_XMAX_IS_LOCKED_ONLY(infomask));
    4698             : 
    4699         526 :                 for (i = 0; i < nmembers; i++)
    4700             :                 {
    4701             :                     /* only consider members of our own transaction */
    4702         378 :                     if (!TransactionIdIsCurrentTransactionId(members[i].xid))
    4703         280 :                         continue;
    4704             : 
    4705          98 :                     if (TUPLOCK_from_mxstatus(members[i].status) >= mode)
    4706             :                     {
    4707          28 :                         pfree(members);
    4708          28 :                         result = TM_Ok;
    4709          28 :                         goto out_unlocked;
    4710             :                     }
    4711             :                     else
    4712             :                     {
    4713             :                         /*
    4714             :                          * Disable acquisition of the heavyweight tuple lock.
    4715             :                          * Otherwise, when promoting a weaker lock, we might
    4716             :                          * deadlock with another locker that has acquired the
    4717             :                          * heavyweight tuple lock and is waiting for our
    4718             :                          * transaction to finish.
    4719             :                          *
    4720             :                          * Note that in this case we still need to wait for
    4721             :                          * the multixact if required, to avoid acquiring
    4722             :                          * conflicting locks.
    4723             :                          */
    4724          70 :                         skip_tuple_lock = true;
    4725             :                     }
    4726             :                 }
    4727             : 
    4728         148 :                 if (members)
    4729         148 :                     pfree(members);
    4730             :             }
    4731       15496 :             else if (TransactionIdIsCurrentTransactionId(xwait))
    4732             :             {
    4733       13042 :                 switch (mode)
    4734             :                 {
    4735         324 :                     case LockTupleKeyShare:
    4736             :                         Assert(HEAP_XMAX_IS_KEYSHR_LOCKED(infomask) ||
    4737             :                                HEAP_XMAX_IS_SHR_LOCKED(infomask) ||
    4738             :                                HEAP_XMAX_IS_EXCL_LOCKED(infomask));
    4739         324 :                         result = TM_Ok;
    4740         324 :                         goto out_unlocked;
    4741          12 :                     case LockTupleShare:
    4742          24 :                         if (HEAP_XMAX_IS_SHR_LOCKED(infomask) ||
    4743          12 :                             HEAP_XMAX_IS_EXCL_LOCKED(infomask))
    4744             :                         {
    4745           0 :                             result = TM_Ok;
    4746           0 :                             goto out_unlocked;
    4747             :                         }
    4748          12 :                         break;
    4749         122 :                     case LockTupleNoKeyExclusive:
    4750         122 :                         if (HEAP_XMAX_IS_EXCL_LOCKED(infomask))
    4751             :                         {
    4752         100 :                             result = TM_Ok;
    4753         100 :                             goto out_unlocked;
    4754             :                         }
    4755          22 :                         break;
    4756       12584 :                     case LockTupleExclusive:
    4757       12584 :                         if (HEAP_XMAX_IS_EXCL_LOCKED(infomask) &&
    4758        2504 :                             infomask2 & HEAP_KEYS_UPDATED)
    4759             :                         {
    4760        2462 :                             result = TM_Ok;
    4761        2462 :                             goto out_unlocked;
    4762             :                         }
    4763       10122 :                         break;
    4764             :                 }
    4765        2472 :             }
    4766             :         }
    4767             : 
    4768             :         /*
    4769             :          * Initially assume that we will have to wait for the locking
    4770             :          * transaction(s) to finish.  We check various cases below in which
    4771             :          * this can be turned off.
    4772             :          */
    4773       12776 :         require_sleep = true;
    4774       12776 :         if (mode == LockTupleKeyShare)
    4775             :         {
    4776             :             /*
    4777             :              * If we're requesting KeyShare, and there's no update present, we
    4778             :              * don't need to wait.  Even if there is an update, we can still
    4779             :              * continue if the key hasn't been modified.
    4780             :              *
    4781             :              * However, if there are updates, we need to walk the update chain
    4782             :              * to mark future versions of the row as locked, too.  That way,
    4783             :              * if somebody deletes that future version, we're protected
    4784             :              * against the key going away.  This locking of future versions
    4785             :              * could block momentarily, if a concurrent transaction is
    4786             :              * deleting a key; or it could return a value to the effect that
    4787             :              * the transaction deleting the key has already committed.  So we
    4788             :              * do this before re-locking the buffer; otherwise this would be
    4789             :              * prone to deadlocks.
    4790             :              *
    4791             :              * Note that the TID we're locking was grabbed before we unlocked
    4792             :              * the buffer.  For it to change while we're not looking, the
    4793             :              * other properties we're testing for below after re-locking the
    4794             :              * buffer would also change, in which case we would restart this
    4795             :              * loop above.
    4796             :              */
    4797        1174 :             if (!(infomask2 & HEAP_KEYS_UPDATED))
    4798             :             {
    4799             :                 bool        updated;
    4800             : 
    4801        1112 :                 updated = !HEAP_XMAX_IS_LOCKED_ONLY(infomask);
    4802             : 
    4803             :                 /*
    4804             :                  * If there are updates, follow the update chain; bail out if
    4805             :                  * that cannot be done.
    4806             :                  */
    4807        1112 :                 if (follow_updates && updated)
    4808             :                 {
    4809             :                     TM_Result   res;
    4810             : 
    4811         100 :                     res = heap_lock_updated_tuple(relation, tuple, &t_ctid,
    4812             :                                                   GetCurrentTransactionId(),
    4813             :                                                   mode);
    4814         100 :                     if (res != TM_Ok)
    4815             :                     {
    4816          12 :                         result = res;
    4817             :                         /* recovery code expects to have buffer lock held */
    4818          12 :                         LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4819         360 :                         goto failed;
    4820             :                     }
    4821             :                 }
    4822             : 
    4823        1100 :                 LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4824             : 
    4825             :                 /*
    4826             :                  * Make sure it's still an appropriate lock, else start over.
    4827             :                  * Also, if it wasn't updated before we released the lock, but
    4828             :                  * is updated now, we start over too; the reason is that we
    4829             :                  * now need to follow the update chain to lock the new
    4830             :                  * versions.
    4831             :                  */
    4832        1100 :                 if (!HeapTupleHeaderIsOnlyLocked(tuple->t_data) &&
    4833          86 :                     ((tuple->t_data->t_infomask2 & HEAP_KEYS_UPDATED) ||
    4834          86 :                      !updated))
    4835          26 :                     goto l3;
    4836             : 
    4837             :                 /* Things look okay, so we can skip sleeping */
    4838        1100 :                 require_sleep = false;
    4839             : 
    4840             :                 /*
    4841             :                  * Note we allow Xmax to change here; other updaters/lockers
    4842             :                  * could have modified it before we grabbed the buffer lock.
    4843             :                  * However, this is not a problem, because with the recheck we
    4844             :                  * just did we ensure that they still don't conflict with the
    4845             :                  * lock we want.
    4846             :                  */
    4847             :             }
    4848             :         }
    4849       11602 :         else if (mode == LockTupleShare)
    4850             :         {
    4851             :             /*
    4852             :              * If we're requesting Share, we can similarly avoid sleeping if
    4853             :              * there's no update and no exclusive lock present.
    4854             :              */
    4855         882 :             if (HEAP_XMAX_IS_LOCKED_ONLY(infomask) &&
    4856         882 :                 !HEAP_XMAX_IS_EXCL_LOCKED(infomask))
    4857             :             {
    4858         870 :                 LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4859             : 
    4860             :                 /*
    4861             :                  * Make sure it's still an appropriate lock, else start over.
    4862             :                  * See above about allowing xmax to change.
    4863             :                  */
    4864        1740 :                 if (!HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_data->t_infomask) ||
    4865         870 :                     HEAP_XMAX_IS_EXCL_LOCKED(tuple->t_data->t_infomask))
    4866           0 :                     goto l3;
    4867         870 :                 require_sleep = false;
    4868             :             }
    4869             :         }
    4870       10720 :         else if (mode == LockTupleNoKeyExclusive)
    4871             :         {
    4872             :             /*
    4873             :              * If we're requesting NoKeyExclusive, we might also be able to
    4874             :              * avoid sleeping; just ensure that there no conflicting lock
    4875             :              * already acquired.
    4876             :              */
    4877         310 :             if (infomask & HEAP_XMAX_IS_MULTI)
    4878             :             {
    4879          52 :                 if (!DoesMultiXactIdConflict((MultiXactId) xwait, infomask,
    4880             :                                              mode, NULL))
    4881             :                 {
    4882             :                     /*
    4883             :                      * No conflict, but if the xmax changed under us in the
    4884             :                      * meantime, start over.
    4885             :                      */
    4886          26 :                     LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4887          52 :                     if (xmax_infomask_changed(tuple->t_data->t_infomask, infomask) ||
    4888          26 :                         !TransactionIdEquals(HeapTupleHeaderGetRawXmax(tuple->t_data),
    4889             :                                              xwait))
    4890           0 :                         goto l3;
    4891             : 
    4892             :                     /* otherwise, we're good */
    4893          26 :                     require_sleep = false;
    4894             :                 }
    4895             :             }
    4896         258 :             else if (HEAP_XMAX_IS_KEYSHR_LOCKED(infomask))
    4897             :             {
    4898          34 :                 LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4899             : 
    4900             :                 /* if the xmax changed in the meantime, start over */
    4901          68 :                 if (xmax_infomask_changed(tuple->t_data->t_infomask, infomask) ||
    4902          34 :                     !TransactionIdEquals(HeapTupleHeaderGetRawXmax(tuple->t_data),
    4903             :                                          xwait))
    4904           0 :                     goto l3;
    4905             :                 /* otherwise, we're good */
    4906          34 :                 require_sleep = false;
    4907             :             }
    4908             :         }
    4909             : 
    4910             :         /*
    4911             :          * As a check independent from those above, we can also avoid sleeping
    4912             :          * if the current transaction is the sole locker of the tuple.  Note
    4913             :          * that the strength of the lock already held is irrelevant; this is
    4914             :          * not about recording the lock in Xmax (which will be done regardless
    4915             :          * of this optimization, below).  Also, note that the cases where we
    4916             :          * hold a lock stronger than we are requesting are already handled
    4917             :          * above by not doing anything.
    4918             :          *
    4919             :          * Note we only deal with the non-multixact case here; MultiXactIdWait
    4920             :          * is well equipped to deal with this situation on its own.
    4921             :          */
    4922       23416 :         if (require_sleep && !(infomask & HEAP_XMAX_IS_MULTI) &&
    4923       10652 :             TransactionIdIsCurrentTransactionId(xwait))
    4924             :         {
    4925             :             /* ... but if the xmax changed in the meantime, start over */
    4926       10122 :             LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4927       20244 :             if (xmax_infomask_changed(tuple->t_data->t_infomask, infomask) ||
    4928       10122 :                 !TransactionIdEquals(HeapTupleHeaderGetRawXmax(tuple->t_data),
    4929             :                                      xwait))
    4930           0 :                 goto l3;
    4931             :             Assert(HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_data->t_infomask));
    4932       10122 :             require_sleep = false;
    4933             :         }
    4934             : 
    4935             :         /*
    4936             :          * Time to sleep on the other transaction/multixact, if necessary.
    4937             :          *
    4938             :          * If the other transaction is an update/delete that's already
    4939             :          * committed, then sleeping cannot possibly do any good: if we're
    4940             :          * required to sleep, get out to raise an error instead.
    4941             :          *
    4942             :          * By here, we either have already acquired the buffer exclusive lock,
    4943             :          * or we must wait for the locking transaction or multixact; so below
    4944             :          * we ensure that we grab buffer lock after the sleep.
    4945             :          */
    4946       12764 :         if (require_sleep && (result == TM_Updated || result == TM_Deleted))
    4947             :         {
    4948         272 :             LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4949         272 :             goto failed;
    4950             :         }
    4951       12492 :         else if (require_sleep)
    4952             :         {
    4953             :             /*
    4954             :              * Acquire tuple lock to establish our priority for the tuple, or
    4955             :              * die trying.  LockTuple will release us when we are next-in-line
    4956             :              * for the tuple.  We must do this even if we are share-locking,
    4957             :              * but not if we already have a weaker lock on the tuple.
    4958             :              *
    4959             :              * If we are forced to "start over" below, we keep the tuple lock;
    4960             :              * this arranges that we stay at the head of the line while
    4961             :              * rechecking tuple state.
    4962             :              */
    4963         340 :             if (!skip_tuple_lock &&
    4964         308 :                 !heap_acquire_tuplock(relation, tid, mode, wait_policy,
    4965             :                                       &have_tuple_lock))
    4966             :             {
    4967             :                 /*
    4968             :                  * This can only happen if wait_policy is Skip and the lock
    4969             :                  * couldn't be obtained.
    4970             :                  */
    4971           2 :                 result = TM_WouldBlock;
    4972             :                 /* recovery code expects to have buffer lock held */
    4973           2 :                 LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    4974           2 :                 goto failed;
    4975             :             }
    4976             : 
    4977         336 :             if (infomask & HEAP_XMAX_IS_MULTI)
    4978             :             {
    4979          80 :                 MultiXactStatus status = get_mxact_status_for_lock(mode, false);
    4980             : 
    4981             :                 /* We only ever lock tuples, never update them */
    4982          80 :                 if (status >= MultiXactStatusNoKeyUpdate)
    4983           0 :                     elog(ERROR, "invalid lock mode in heap_lock_tuple");
    4984             : 
    4985             :                 /* wait for multixact to end, or die trying  */
    4986          80 :                 switch (wait_policy)
    4987             :                 {
    4988          72 :                     case LockWaitBlock:
    4989          72 :                         MultiXactIdWait((MultiXactId) xwait, status, infomask,
    4990             :                                         relation, &tuple->t_self, XLTW_Lock, NULL);
    4991          72 :                         break;
    4992           4 :                     case LockWaitSkip:
    4993           4 :                         if (!ConditionalMultiXactIdWait((MultiXactId) xwait,
    4994             :                                                         status, infomask, relation,
    4995             :                                                         NULL, false))
    4996             :                         {
    4997           4 :                             result = TM_WouldBlock;
    4998             :                             /* recovery code expects to have buffer lock held */
    4999           4 :                             LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    5000           4 :                             goto failed;
    5001             :                         }
    5002           0 :                         break;
    5003           4 :                     case LockWaitError:
    5004           4 :                         if (!ConditionalMultiXactIdWait((MultiXactId) xwait,
    5005             :                                                         status, infomask, relation,
    5006             :                                                         NULL, log_lock_failure))
    5007           4 :                             ereport(ERROR,
    5008             :                                     (errcode(ERRCODE_LOCK_NOT_AVAILABLE),
    5009             :                                      errmsg("could not obtain lock on row in relation \"%s\"",
    5010             :                                             RelationGetRelationName(relation))));
    5011             : 
    5012           0 :                         break;
    5013             :                 }
    5014             : 
    5015             :                 /*
    5016             :                  * Of course, the multixact might not be done here: if we're
    5017             :                  * requesting a light lock mode, other transactions with light
    5018             :                  * locks could still be alive, as well as locks owned by our
    5019             :                  * own xact or other subxacts of this backend.  We need to
    5020             :                  * preserve the surviving MultiXact members.  Note that it
    5021             :                  * isn't absolutely necessary in the latter case, but doing so
    5022             :                  * is simpler.
    5023             :                  */
    5024          72 :             }
    5025             :             else
    5026             :             {
    5027             :                 /* wait for regular transaction to end, or die trying */
    5028         256 :                 switch (wait_policy)
    5029             :                 {
    5030         178 :                     case LockWaitBlock:
    5031         178 :                         XactLockTableWait(xwait, relation, &tuple->t_self,
    5032             :                                           XLTW_Lock);
    5033         178 :                         break;
    5034          66 :                     case LockWaitSkip:
    5035          66 :                         if (!ConditionalXactLockTableWait(xwait, false))
    5036             :                         {
    5037          66 :                             result = TM_WouldBlock;
    5038             :                             /* recovery code expects to have buffer lock held */
    5039          66 :                             LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    5040          66 :                             goto failed;
    5041             :                         }
    5042           0 :                         break;
    5043          12 :                     case LockWaitError:
    5044          12 :                         if (!ConditionalXactLockTableWait(xwait, log_lock_failure))
    5045          12 :                             ereport(ERROR,
    5046             :                                     (errcode(ERRCODE_LOCK_NOT_AVAILABLE),
    5047             :                                      errmsg("could not obtain lock on row in relation \"%s\"",
    5048             :                                             RelationGetRelationName(relation))));
    5049           0 :                         break;
    5050             :                 }
    5051         250 :             }
    5052             : 
    5053             :             /* if there are updates, follow the update chain */
    5054         250 :             if (follow_updates && !HEAP_XMAX_IS_LOCKED_ONLY(infomask))
    5055             :             {
    5056             :                 TM_Result   res;
    5057             : 
    5058          80 :                 res = heap_lock_updated_tuple(relation, tuple, &t_ctid,
    5059             :                                               GetCurrentTransactionId(),
    5060             :                                               mode);
    5061          80 :                 if (res != TM_Ok)
    5062             :                 {
    5063           4 :                     result = res;
    5064             :                     /* recovery code expects to have buffer lock held */
    5065           4 :                     LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    5066           4 :                     goto failed;
    5067             :                 }
    5068             :             }
    5069             : 
    5070         246 :             LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    5071             : 
    5072             :             /*
    5073             :              * xwait is done, but if xwait had just locked the tuple then some
    5074             :              * other xact could update this tuple before we get to this point.
    5075             :              * Check for xmax change, and start over if so.
    5076             :              */
    5077         470 :             if (xmax_infomask_changed(tuple->t_data->t_infomask, infomask) ||
    5078         224 :                 !TransactionIdEquals(HeapTupleHeaderGetRawXmax(tuple->t_data),
    5079             :                                      xwait))
    5080          26 :                 goto l3;
    5081             : 
    5082         220 :             if (!(infomask & HEAP_XMAX_IS_MULTI))
    5083             :             {
    5084             :                 /*
    5085             :                  * Otherwise check if it committed or aborted.  Note we cannot
    5086             :                  * be here if the tuple was only locked by somebody who didn't
    5087             :                  * conflict with us; that would have been handled above.  So
    5088             :                  * that transaction must necessarily be gone by now.  But
    5089             :                  * don't check for this in the multixact case, because some
    5090             :                  * locker transactions might still be running.
    5091             :                  */
    5092         156 :                 UpdateXmaxHintBits(tuple->t_data, *buffer, xwait);
    5093             :             }
    5094             :         }
    5095             : 
    5096             :         /* By here, we're certain that we hold buffer exclusive lock again */
    5097             : 
    5098             :         /*
    5099             :          * We may lock if previous xmax aborted, or if it committed but only
    5100             :          * locked the tuple without updating it; or if we didn't have to wait
    5101             :          * at all for whatever reason.
    5102             :          */
    5103       12372 :         if (!require_sleep ||
    5104         380 :             (tuple->t_data->t_infomask & HEAP_XMAX_INVALID) ||
    5105         288 :             HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_data->t_infomask) ||
    5106         128 :             HeapTupleHeaderIsOnlyLocked(tuple->t_data))
    5107       12256 :             result = TM_Ok;
    5108         116 :         else if (!ItemPointerEquals(&tuple->t_self, &tuple->t_data->t_ctid))
    5109          92 :             result = TM_Updated;
    5110             :         else
    5111          24 :             result = TM_Deleted;
    5112             :     }
    5113             : 
    5114      154002 : failed:
    5115      166734 :     if (result != TM_Ok)
    5116             :     {
    5117             :         Assert(result == TM_SelfModified || result == TM_Updated ||
    5118             :                result == TM_Deleted || result == TM_WouldBlock);
    5119             : 
    5120             :         /*
    5121             :          * When locking a tuple under LockWaitSkip semantics and we fail with
    5122             :          * TM_WouldBlock above, it's possible for concurrent transactions to
    5123             :          * release the lock and set HEAP_XMAX_INVALID in the meantime.  So
    5124             :          * this assert is slightly different from the equivalent one in
    5125             :          * heap_delete and heap_update.
    5126             :          */
    5127             :         Assert((result == TM_WouldBlock) ||
    5128             :                !(tuple->t_data->t_infomask & HEAP_XMAX_INVALID));
    5129             :         Assert(result != TM_Updated ||
    5130             :                !ItemPointerEquals(&tuple->t_self, &tuple->t_data->t_ctid));
    5131         488 :         tmfd->ctid = tuple->t_data->t_ctid;
    5132         488 :         tmfd->xmax = HeapTupleHeaderGetUpdateXid(tuple->t_data);
    5133         488 :         if (result == TM_SelfModified)
    5134          12 :             tmfd->cmax = HeapTupleHeaderGetCmax(tuple->t_data);
    5135             :         else
    5136         476 :             tmfd->cmax = InvalidCommandId;
    5137         488 :         goto out_locked;
    5138             :     }
    5139             : 
    5140             :     /*
    5141             :      * If we didn't pin the visibility map page and the page has become all
    5142             :      * visible while we were busy locking the buffer, or during some
    5143             :      * subsequent window during which we had it unlocked, we'll have to unlock
    5144             :      * and re-lock, to avoid holding the buffer lock across I/O.  That's a bit
    5145             :      * unfortunate, especially since we'll now have to recheck whether the
    5146             :      * tuple has been locked or updated under us, but hopefully it won't
    5147             :      * happen very often.
    5148             :      */
    5149      166246 :     if (vmbuffer == InvalidBuffer && PageIsAllVisible(page))
    5150             :     {
    5151           0 :         LockBuffer(*buffer, BUFFER_LOCK_UNLOCK);
    5152           0 :         visibilitymap_pin(relation, block, &vmbuffer);
    5153           0 :         LockBuffer(*buffer, BUFFER_LOCK_EXCLUSIVE);
    5154           0 :         goto l3;
    5155             :     }
    5156             : 
    5157      166246 :     xmax = HeapTupleHeaderGetRawXmax(tuple->t_data);
    5158      166246 :     old_infomask = tuple->t_data->t_infomask;
    5159             : 
    5160             :     /*
    5161             :      * If this is the first possibly-multixact-able operation in the current
    5162             :      * transaction, set my per-backend OldestMemberMXactId setting. We can be
    5163             :      * certain that the transaction will never become a member of any older
    5164             :      * MultiXactIds than that.  (We have to do this even if we end up just
    5165             :      * using our own TransactionId below, since some other backend could
    5166             :      * incorporate our XID into a MultiXact immediately afterwards.)
    5167             :      */
    5168      166246 :     MultiXactIdSetOldestMember();
    5169             : 
    5170             :     /*
    5171             :      * Compute the new xmax and infomask to store into the tuple.  Note we do
    5172             :      * not modify the tuple just yet, because that would leave it in the wrong
    5173             :      * state if multixact.c elogs.
    5174             :      */
    5175      166246 :     compute_new_xmax_infomask(xmax, old_infomask, tuple->t_data->t_infomask2,
    5176             :                               GetCurrentTransactionId(), mode, false,
    5177             :                               &xid, &new_infomask, &new_infomask2);
    5178             : 
    5179      166246 :     START_CRIT_SECTION();
    5180             : 
    5181             :     /*
    5182             :      * Store transaction information of xact locking the tuple.
    5183             :      *
    5184             :      * Note: Cmax is meaningless in this context, so don't set it; this avoids
    5185             :      * possibly generating a useless combo CID.  Moreover, if we're locking a
    5186             :      * previously updated tuple, it's important to preserve the Cmax.
    5187             :      *
    5188             :      * Also reset the HOT UPDATE bit, but only if there's no update; otherwise
    5189             :      * we would break the HOT chain.
    5190             :      */
    5191      166246 :     tuple->t_data->t_infomask &= ~HEAP_XMAX_BITS;
    5192      166246 :     tuple->t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    5193      166246 :     tuple->t_data->t_infomask |= new_infomask;
    5194      166246 :     tuple->t_data->t_infomask2 |= new_infomask2;
    5195      166246 :     if (HEAP_XMAX_IS_LOCKED_ONLY(new_infomask))
    5196      166168 :         HeapTupleHeaderClearHotUpdated(tuple->t_data);
    5197      166246 :     HeapTupleHeaderSetXmax(tuple->t_data, xid);
    5198             : 
    5199             :     /*
    5200             :      * Make sure there is no forward chain link in t_ctid.  Note that in the
    5201             :      * cases where the tuple has been updated, we must not overwrite t_ctid,
    5202             :      * because it was set by the updater.  Moreover, if the tuple has been
    5203             :      * updated, we need to follow the update chain to lock the new versions of
    5204             :      * the tuple as well.
    5205             :      */
    5206      166246 :     if (HEAP_XMAX_IS_LOCKED_ONLY(new_infomask))
    5207      166168 :         tuple->t_data->t_ctid = *tid;
    5208             : 
    5209             :     /* Clear only the all-frozen bit on visibility map if needed */
    5210      169564 :     if (PageIsAllVisible(page) &&
    5211        3318 :         visibilitymap_clear(relation, block, vmbuffer,
    5212             :                             VISIBILITYMAP_ALL_FROZEN))
    5213          30 :         cleared_all_frozen = true;
    5214             : 
    5215             : 
    5216      166246 :     MarkBufferDirty(*buffer);
    5217             : 
    5218             :     /*
    5219             :      * XLOG stuff.  You might think that we don't need an XLOG record because
    5220             :      * there is no state change worth restoring after a crash.  You would be
    5221             :      * wrong however: we have just written either a TransactionId or a
    5222             :      * MultiXactId that may never have been seen on disk before, and we need
    5223             :      * to make sure that there are XLOG entries covering those ID numbers.
    5224             :      * Else the same IDs might be re-used after a crash, which would be
    5225             :      * disastrous if this page made it to disk before the crash.  Essentially
    5226             :      * we have to enforce the WAL log-before-data rule even in this case.
    5227             :      * (Also, in a PITR log-shipping or 2PC environment, we have to have XLOG
    5228             :      * entries for everything anyway.)
    5229             :      */
    5230      166246 :     if (RelationNeedsWAL(relation))
    5231             :     {
    5232             :         xl_heap_lock xlrec;
    5233             :         XLogRecPtr  recptr;
    5234             : 
    5235      165558 :         XLogBeginInsert();
    5236      165558 :         XLogRegisterBuffer(0, *buffer, REGBUF_STANDARD);
    5237             : 
    5238      165558 :         xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self);
    5239      165558 :         xlrec.xmax = xid;
    5240      331116 :         xlrec.infobits_set = compute_infobits(new_infomask,
    5241      165558 :                                               tuple->t_data->t_infomask2);
    5242      165558 :         xlrec.flags = cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
    5243      165558 :         XLogRegisterData(&xlrec, SizeOfHeapLock);
    5244             : 
    5245             :         /* we don't decode row locks atm, so no need to log the origin */
    5246             : 
    5247      165558 :         recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_LOCK);
    5248             : 
    5249      165558 :         PageSetLSN(page, recptr);
    5250             :     }
    5251             : 
    5252      166246 :     END_CRIT_SECTION();
    5253             : 
    5254      166246 :     result = TM_Ok;
    5255             : 
    5256      166758 : out_locked:
    5257      166758 :     LockBuffer(*buffer, BUFFER_LOCK_UNLOCK);
    5258             : 
    5259      169672 : out_unlocked:
    5260      169672 :     if (BufferIsValid(vmbuffer))
    5261        3318 :         ReleaseBuffer(vmbuffer);
    5262             : 
    5263             :     /*
    5264             :      * Don't update the visibility map here. Locking a tuple doesn't change
    5265             :      * visibility info.
    5266             :      */
    5267             : 
    5268             :     /*
    5269             :      * Now that we have successfully marked the tuple as locked, we can
    5270             :      * release the lmgr tuple lock, if we had it.
    5271             :      */
    5272      169672 :     if (have_tuple_lock)
    5273         278 :         UnlockTupleTuplock(relation, tid, mode);
    5274             : 
    5275      169672 :     return result;
    5276             : }
    5277             : 
    5278             : /*
    5279             :  * Acquire heavyweight lock on the given tuple, in preparation for acquiring
    5280             :  * its normal, Xmax-based tuple lock.
    5281             :  *
    5282             :  * have_tuple_lock is an input and output parameter: on input, it indicates
    5283             :  * whether the lock has previously been acquired (and this function does
    5284             :  * nothing in that case).  If this function returns success, have_tuple_lock
    5285             :  * has been flipped to true.
    5286             :  *
    5287             :  * Returns false if it was unable to obtain the lock; this can only happen if
    5288             :  * wait_policy is Skip.
    5289             :  */
    5290             : static bool
    5291         526 : heap_acquire_tuplock(Relation relation, ItemPointer tid, LockTupleMode mode,
    5292             :                      LockWaitPolicy wait_policy, bool *have_tuple_lock)
    5293             : {
    5294         526 :     if (*have_tuple_lock)
    5295          18 :         return true;
    5296             : 
    5297         508 :     switch (wait_policy)
    5298             :     {
    5299         426 :         case LockWaitBlock:
    5300         426 :             LockTupleTuplock(relation, tid, mode);
    5301         426 :             break;
    5302             : 
    5303          68 :         case LockWaitSkip:
    5304          68 :             if (!ConditionalLockTupleTuplock(relation, tid, mode, false))
    5305           2 :                 return false;
    5306          66 :             break;
    5307             : 
    5308          14 :         case LockWaitError:
    5309          14 :             if (!ConditionalLockTupleTuplock(relation, tid, mode, log_lock_failure))
    5310           2 :                 ereport(ERROR,
    5311             :                         (errcode(ERRCODE_LOCK_NOT_AVAILABLE),
    5312             :                          errmsg("could not obtain lock on row in relation \"%s\"",
    5313             :                                 RelationGetRelationName(relation))));
    5314          12 :             break;
    5315             :     }
    5316         504 :     *have_tuple_lock = true;
    5317             : 
    5318         504 :     return true;
    5319             : }
    5320             : 
    5321             : /*
    5322             :  * Given an original set of Xmax and infomask, and a transaction (identified by
    5323             :  * add_to_xmax) acquiring a new lock of some mode, compute the new Xmax and
    5324             :  * corresponding infomasks to use on the tuple.
    5325             :  *
    5326             :  * Note that this might have side effects such as creating a new MultiXactId.
    5327             :  *
    5328             :  * Most callers will have called HeapTupleSatisfiesUpdate before this function;
    5329             :  * that will have set the HEAP_XMAX_INVALID bit if the xmax was a MultiXactId
    5330             :  * but it was not running anymore. There is a race condition, which is that the
    5331             :  * MultiXactId may have finished since then, but that uncommon case is handled
    5332             :  * either here, or within MultiXactIdExpand.
    5333             :  *
    5334             :  * There is a similar race condition possible when the old xmax was a regular
    5335             :  * TransactionId.  We test TransactionIdIsInProgress again just to narrow the
    5336             :  * window, but it's still possible to end up creating an unnecessary
    5337             :  * MultiXactId.  Fortunately this is harmless.
    5338             :  */
    5339             : static void
    5340     4206914 : compute_new_xmax_infomask(TransactionId xmax, uint16 old_infomask,
    5341             :                           uint16 old_infomask2, TransactionId add_to_xmax,
    5342             :                           LockTupleMode mode, bool is_update,
    5343             :                           TransactionId *result_xmax, uint16 *result_infomask,
    5344             :                           uint16 *result_infomask2)
    5345             : {
    5346             :     TransactionId new_xmax;
    5347             :     uint16      new_infomask,
    5348             :                 new_infomask2;
    5349             : 
    5350             :     Assert(TransactionIdIsCurrentTransactionId(add_to_xmax));
    5351             : 
    5352     4206914 : l5:
    5353     4206914 :     new_infomask = 0;
    5354     4206914 :     new_infomask2 = 0;
    5355     4206914 :     if (old_infomask & HEAP_XMAX_INVALID)
    5356             :     {
    5357             :         /*
    5358             :          * No previous locker; we just insert our own TransactionId.
    5359             :          *
    5360             :          * Note that it's critical that this case be the first one checked,
    5361             :          * because there are several blocks below that come back to this one
    5362             :          * to implement certain optimizations; old_infomask might contain
    5363             :          * other dirty bits in those cases, but we don't really care.
    5364             :          */
    5365     3996842 :         if (is_update)
    5366             :         {
    5367     3539130 :             new_xmax = add_to_xmax;
    5368     3539130 :             if (mode == LockTupleExclusive)
    5369     3016156 :                 new_infomask2 |= HEAP_KEYS_UPDATED;
    5370             :         }
    5371             :         else
    5372             :         {
    5373      457712 :             new_infomask |= HEAP_XMAX_LOCK_ONLY;
    5374      457712 :             switch (mode)
    5375             :             {
    5376        5022 :                 case LockTupleKeyShare:
    5377        5022 :                     new_xmax = add_to_xmax;
    5378        5022 :                     new_infomask |= HEAP_XMAX_KEYSHR_LOCK;
    5379        5022 :                     break;
    5380        1442 :                 case LockTupleShare:
    5381        1442 :                     new_xmax = add_to_xmax;
    5382        1442 :                     new_infomask |= HEAP_XMAX_SHR_LOCK;
    5383        1442 :                     break;
    5384      260034 :                 case LockTupleNoKeyExclusive:
    5385      260034 :                     new_xmax = add_to_xmax;
    5386      260034 :                     new_infomask |= HEAP_XMAX_EXCL_LOCK;
    5387      260034 :                     break;
    5388      191214 :                 case LockTupleExclusive:
    5389      191214 :                     new_xmax = add_to_xmax;
    5390      191214 :                     new_infomask |= HEAP_XMAX_EXCL_LOCK;
    5391      191214 :                     new_infomask2 |= HEAP_KEYS_UPDATED;
    5392      191214 :                     break;
    5393           0 :                 default:
    5394           0 :                     new_xmax = InvalidTransactionId;    /* silence compiler */
    5395           0 :                     elog(ERROR, "invalid lock mode");
    5396             :             }
    5397             :         }
    5398             :     }
    5399      210072 :     else if (old_infomask & HEAP_XMAX_IS_MULTI)
    5400             :     {
    5401             :         MultiXactStatus new_status;
    5402             : 
    5403             :         /*
    5404             :          * Currently we don't allow XMAX_COMMITTED to be set for multis, so
    5405             :          * cross-check.
    5406             :          */
    5407             :         Assert(!(old_infomask & HEAP_XMAX_COMMITTED));
    5408             : 
    5409             :         /*
    5410             :          * A multixact together with LOCK_ONLY set but neither lock bit set
    5411             :          * (i.e. a pg_upgraded share locked tuple) cannot possibly be running
    5412             :          * anymore.  This check is critical for databases upgraded by
    5413             :          * pg_upgrade; both MultiXactIdIsRunning and MultiXactIdExpand assume
    5414             :          * that such multis are never passed.
    5415             :          */
    5416         250 :         if (HEAP_LOCKED_UPGRADED(old_infomask))
    5417             :         {
    5418           0 :             old_infomask &= ~HEAP_XMAX_IS_MULTI;
    5419           0 :             old_infomask |= HEAP_XMAX_INVALID;
    5420           0 :             goto l5;
    5421             :         }
    5422             : 
    5423             :         /*
    5424             :          * If the XMAX is already a MultiXactId, then we need to expand it to
    5425             :          * include add_to_xmax; but if all the members were lockers and are
    5426             :          * all gone, we can do away with the IS_MULTI bit and just set
    5427             :          * add_to_xmax as the only locker/updater.  If all lockers are gone
    5428             :          * and we have an updater that aborted, we can also do without a
    5429             :          * multi.
    5430             :          *
    5431             :          * The cost of doing GetMultiXactIdMembers would be paid by
    5432             :          * MultiXactIdExpand if we weren't to do this, so this check is not
    5433             :          * incurring extra work anyhow.
    5434             :          */
    5435         250 :         if (!MultiXactIdIsRunning(xmax, HEAP_XMAX_IS_LOCKED_ONLY(old_infomask)))
    5436             :         {
    5437          48 :             if (HEAP_XMAX_IS_LOCKED_ONLY(old_infomask) ||
    5438          16 :                 !TransactionIdDidCommit(MultiXactIdGetUpdateXid(xmax,
    5439             :                                                                 old_infomask)))
    5440             :             {
    5441             :                 /*
    5442             :                  * Reset these bits and restart; otherwise fall through to
    5443             :                  * create a new multi below.
    5444             :                  */
    5445          48 :                 old_infomask &= ~HEAP_XMAX_IS_MULTI;
    5446          48 :                 old_infomask |= HEAP_XMAX_INVALID;
    5447          48 :                 goto l5;
    5448             :             }
    5449             :         }
    5450             : 
    5451         202 :         new_status = get_mxact_status_for_lock(mode, is_update);
    5452             : 
    5453         202 :         new_xmax = MultiXactIdExpand((MultiXactId) xmax, add_to_xmax,
    5454             :                                      new_status);
    5455         202 :         GetMultiXactIdHintBits(new_xmax, &new_infomask, &new_infomask2);
    5456             :     }
    5457      209822 :     else if (old_infomask & HEAP_XMAX_COMMITTED)
    5458             :     {
    5459             :         /*
    5460             :          * It's a committed update, so we need to preserve him as updater of
    5461             :          * the tuple.
    5462             :          */
    5463             :         MultiXactStatus status;
    5464             :         MultiXactStatus new_status;
    5465             : 
    5466          26 :         if (old_infomask2 & HEAP_KEYS_UPDATED)
    5467           0 :             status = MultiXactStatusUpdate;
    5468             :         else
    5469          26 :             status = MultiXactStatusNoKeyUpdate;
    5470             : 
    5471          26 :         new_status = get_mxact_status_for_lock(mode, is_update);
    5472             : 
    5473             :         /*
    5474             :          * since it's not running, it's obviously impossible for the old
    5475             :          * updater to be identical to the current one, so we need not check
    5476             :          * for that case as we do in the block above.
    5477             :          */
    5478          26 :         new_xmax = MultiXactIdCreate(xmax, status, add_to_xmax, new_status);
    5479          26 :         GetMultiXactIdHintBits(new_xmax, &new_infomask, &new_infomask2);
    5480             :     }
    5481      209796 :     else if (TransactionIdIsInProgress(xmax))
    5482             :     {
    5483             :         /*
    5484             :          * If the XMAX is a valid, in-progress TransactionId, then we need to
    5485             :          * create a new MultiXactId that includes both the old locker or
    5486             :          * updater and our own TransactionId.
    5487             :          */
    5488             :         MultiXactStatus new_status;
    5489             :         MultiXactStatus old_status;
    5490             :         LockTupleMode old_mode;
    5491             : 
    5492      209778 :         if (HEAP_XMAX_IS_LOCKED_ONLY(old_infomask))
    5493             :         {
    5494      209726 :             if (HEAP_XMAX_IS_KEYSHR_LOCKED(old_infomask))
    5495       11244 :                 old_status = MultiXactStatusForKeyShare;
    5496      198482 :             else if (HEAP_XMAX_IS_SHR_LOCKED(old_infomask))
    5497         862 :                 old_status = MultiXactStatusForShare;
    5498      197620 :             else if (HEAP_XMAX_IS_EXCL_LOCKED(old_infomask))
    5499             :             {
    5500      197620 :                 if (old_infomask2 & HEAP_KEYS_UPDATED)
    5501      185490 :                     old_status = MultiXactStatusForUpdate;
    5502             :                 else
    5503       12130 :                     old_status = MultiXactStatusForNoKeyUpdate;
    5504             :             }
    5505             :             else
    5506             :             {
    5507             :                 /*
    5508             :                  * LOCK_ONLY can be present alone only when a page has been
    5509             :                  * upgraded by pg_upgrade.  But in that case,
    5510             :                  * TransactionIdIsInProgress() should have returned false.  We
    5511             :                  * assume it's no longer locked in this case.
    5512             :                  */
    5513           0 :                 elog(WARNING, "LOCK_ONLY found for Xid in progress %u", xmax);
    5514           0 :                 old_infomask |= HEAP_XMAX_INVALID;
    5515           0 :                 old_infomask &= ~HEAP_XMAX_LOCK_ONLY;
    5516           0 :                 goto l5;
    5517             :             }
    5518             :         }
    5519             :         else
    5520             :         {
    5521             :             /* it's an update, but which kind? */
    5522          52 :             if (old_infomask2 & HEAP_KEYS_UPDATED)
    5523           0 :                 old_status = MultiXactStatusUpdate;
    5524             :             else
    5525          52 :                 old_status = MultiXactStatusNoKeyUpdate;
    5526             :         }
    5527             : 
    5528      209778 :         old_mode = TUPLOCK_from_mxstatus(old_status);
    5529             : 
    5530             :         /*
    5531             :          * If the lock to be acquired is for the same TransactionId as the
    5532             :          * existing lock, there's an optimization possible: consider only the
    5533             :          * strongest of both locks as the only one present, and restart.
    5534             :          */
    5535      209778 :         if (xmax == add_to_xmax)
    5536             :         {
    5537             :             /*
    5538             :              * Note that it's not possible for the original tuple to be
    5539             :              * updated: we wouldn't be here because the tuple would have been
    5540             :              * invisible and we wouldn't try to update it.  As a subtlety,
    5541             :              * this code can also run when traversing an update chain to lock
    5542             :              * future versions of a tuple.  But we wouldn't be here either,
    5543             :              * because the add_to_xmax would be different from the original
    5544             :              * updater.
    5545             :              */
    5546             :             Assert(HEAP_XMAX_IS_LOCKED_ONLY(old_infomask));
    5547             : 
    5548             :             /* acquire the strongest of both */
    5549      207738 :             if (mode < old_mode)
    5550      104340 :                 mode = old_mode;
    5551             :             /* mustn't touch is_update */
    5552             : 
    5553      207738 :             old_infomask |= HEAP_XMAX_INVALID;
    5554      207738 :             goto l5;
    5555             :         }
    5556             : 
    5557             :         /* otherwise, just fall back to creating a new multixact */
    5558        2040 :         new_status = get_mxact_status_for_lock(mode, is_update);
    5559        2040 :         new_xmax = MultiXactIdCreate(xmax, old_status,
    5560             :                                      add_to_xmax, new_status);
    5561        2040 :         GetMultiXactIdHintBits(new_xmax, &new_infomask, &new_infomask2);
    5562             :     }
    5563          28 :     else if (!HEAP_XMAX_IS_LOCKED_ONLY(old_infomask) &&
    5564          10 :              TransactionIdDidCommit(xmax))
    5565           2 :     {
    5566             :         /*
    5567             :          * It's a committed update, so we gotta preserve him as updater of the
    5568             :          * tuple.
    5569             :          */
    5570             :         MultiXactStatus status;
    5571             :         MultiXactStatus new_status;
    5572             : 
    5573           2 :         if (old_infomask2 & HEAP_KEYS_UPDATED)
    5574           0 :             status = MultiXactStatusUpdate;
    5575             :         else
    5576           2 :             status = MultiXactStatusNoKeyUpdate;
    5577             : 
    5578           2 :         new_status = get_mxact_status_for_lock(mode, is_update);
    5579             : 
    5580             :         /*
    5581             :          * since it's not running, it's obviously impossible for the old
    5582             :          * updater to be identical to the current one, so we need not check
    5583             :          * for that case as we do in the block above.
    5584             :          */
    5585           2 :         new_xmax = MultiXactIdCreate(xmax, status, add_to_xmax, new_status);
    5586           2 :         GetMultiXactIdHintBits(new_xmax, &new_infomask, &new_infomask2);
    5587             :     }
    5588             :     else
    5589             :     {
    5590             :         /*
    5591             :          * Can get here iff the locking/updating transaction was running when
    5592             :          * the infomask was extracted from the tuple, but finished before
    5593             :          * TransactionIdIsInProgress got to run.  Deal with it as if there was
    5594             :          * no locker at all in the first place.
    5595             :          */
    5596          16 :         old_infomask |= HEAP_XMAX_INVALID;
    5597          16 :         goto l5;
    5598             :     }
    5599             : 
    5600     3999112 :     *result_infomask = new_infomask;
    5601     3999112 :     *result_infomask2 = new_infomask2;
    5602     3999112 :     *result_xmax = new_xmax;
    5603     3999112 : }
    5604             : 
    5605             : /*
    5606             :  * Subroutine for heap_lock_updated_tuple_rec.
    5607             :  *
    5608             :  * Given a hypothetical multixact status held by the transaction identified
    5609             :  * with the given xid, does the current transaction need to wait, fail, or can
    5610             :  * it continue if it wanted to acquire a lock of the given mode?  "needwait"
    5611             :  * is set to true if waiting is necessary; if it can continue, then TM_Ok is
    5612             :  * returned.  If the lock is already held by the current transaction, return
    5613             :  * TM_SelfModified.  In case of a conflict with another transaction, a
    5614             :  * different HeapTupleSatisfiesUpdate return code is returned.
    5615             :  *
    5616             :  * The held status is said to be hypothetical because it might correspond to a
    5617             :  * lock held by a single Xid, i.e. not a real MultiXactId; we express it this
    5618             :  * way for simplicity of API.
    5619             :  */
    5620             : static TM_Result
    5621          64 : test_lockmode_for_conflict(MultiXactStatus status, TransactionId xid,
    5622             :                            LockTupleMode mode, HeapTuple tup,
    5623             :                            bool *needwait)
    5624             : {
    5625             :     MultiXactStatus wantedstatus;
    5626             : 
    5627          64 :     *needwait = false;
    5628          64 :     wantedstatus = get_mxact_status_for_lock(mode, false);
    5629             : 
    5630             :     /*
    5631             :      * Note: we *must* check TransactionIdIsInProgress before
    5632             :      * TransactionIdDidAbort/Commit; see comment at top of heapam_visibility.c
    5633             :      * for an explanation.
    5634             :      */
    5635          64 :     if (TransactionIdIsCurrentTransactionId(xid))
    5636             :     {
    5637             :         /*
    5638             :          * The tuple has already been locked by our own transaction.  This is
    5639             :          * very rare but can happen if multiple transactions are trying to
    5640             :          * lock an ancient version of the same tuple.
    5641             :          */
    5642           0 :         return TM_SelfModified;
    5643             :     }
    5644          64 :     else if (TransactionIdIsInProgress(xid))
    5645             :     {
    5646             :         /*
    5647             :          * If the locking transaction is running, what we do depends on
    5648             :          * whether the lock modes conflict: if they do, then we must wait for
    5649             :          * it to finish; otherwise we can fall through to lock this tuple
    5650             :          * version without waiting.
    5651             :          */
    5652          32 :         if (DoLockModesConflict(LOCKMODE_from_mxstatus(status),
    5653          32 :                                 LOCKMODE_from_mxstatus(wantedstatus)))
    5654             :         {
    5655          16 :             *needwait = true;
    5656             :         }
    5657             : 
    5658             :         /*
    5659             :          * If we set needwait above, then this value doesn't matter;
    5660             :          * otherwise, this value signals to caller that it's okay to proceed.
    5661             :          */
    5662          32 :         return TM_Ok;
    5663             :     }
    5664          32 :     else if (TransactionIdDidAbort(xid))
    5665           6 :         return TM_Ok;
    5666          26 :     else if (TransactionIdDidCommit(xid))
    5667             :     {
    5668             :         /*
    5669             :          * The other transaction committed.  If it was only a locker, then the
    5670             :          * lock is completely gone now and we can return success; but if it
    5671             :          * was an update, then what we do depends on whether the two lock
    5672             :          * modes conflict.  If they conflict, then we must report error to
    5673             :          * caller. But if they don't, we can fall through to allow the current
    5674             :          * transaction to lock the tuple.
    5675             :          *
    5676             :          * Note: the reason we worry about ISUPDATE here is because as soon as
    5677             :          * a transaction ends, all its locks are gone and meaningless, and
    5678             :          * thus we can ignore them; whereas its updates persist.  In the
    5679             :          * TransactionIdIsInProgress case, above, we don't need to check
    5680             :          * because we know the lock is still "alive" and thus a conflict needs
    5681             :          * always be checked.
    5682             :          */
    5683          26 :         if (!ISUPDATE_from_mxstatus(status))
    5684           8 :             return TM_Ok;
    5685             : 
    5686          18 :         if (DoLockModesConflict(LOCKMODE_from_mxstatus(status),
    5687          18 :                                 LOCKMODE_from_mxstatus(wantedstatus)))
    5688             :         {
    5689             :             /* bummer */
    5690          16 :             if (!ItemPointerEquals(&tup->t_self, &tup->t_data->t_ctid))
    5691          12 :                 return TM_Updated;
    5692             :             else
    5693           4 :                 return TM_Deleted;
    5694             :         }
    5695             : 
    5696           2 :         return TM_Ok;
    5697             :     }
    5698             : 
    5699             :     /* Not in progress, not aborted, not committed -- must have crashed */
    5700           0 :     return TM_Ok;
    5701             : }
    5702             : 
    5703             : 
    5704             : /*
    5705             :  * Recursive part of heap_lock_updated_tuple
    5706             :  *
    5707             :  * Fetch the tuple pointed to by tid in rel, and mark it as locked by the given
    5708             :  * xid with the given mode; if this tuple is updated, recurse to lock the new
    5709             :  * version as well.
    5710             :  */
    5711             : static TM_Result
    5712         162 : heap_lock_updated_tuple_rec(Relation rel, ItemPointer tid, TransactionId xid,
    5713             :                             LockTupleMode mode)
    5714             : {
    5715             :     TM_Result   result;
    5716             :     ItemPointerData tupid;
    5717             :     HeapTupleData mytup;
    5718             :     Buffer      buf;
    5719             :     uint16      new_infomask,
    5720             :                 new_infomask2,
    5721             :                 old_infomask,
    5722             :                 old_infomask2;
    5723             :     TransactionId xmax,
    5724             :                 new_xmax;
    5725         162 :     TransactionId priorXmax = InvalidTransactionId;
    5726         162 :     bool        cleared_all_frozen = false;
    5727             :     bool        pinned_desired_page;
    5728         162 :     Buffer      vmbuffer = InvalidBuffer;
    5729             :     BlockNumber block;
    5730             : 
    5731         162 :     ItemPointerCopy(tid, &tupid);
    5732             : 
    5733             :     for (;;)
    5734             :     {
    5735         168 :         new_infomask = 0;
    5736         168 :         new_xmax = InvalidTransactionId;
    5737         168 :         block = ItemPointerGetBlockNumber(&tupid);
    5738         168 :         ItemPointerCopy(&tupid, &(mytup.t_self));
    5739             : 
    5740         168 :         if (!heap_fetch(rel, SnapshotAny, &mytup, &buf, false))
    5741             :         {
    5742             :             /*
    5743             :              * if we fail to find the updated version of the tuple, it's
    5744             :              * because it was vacuumed/pruned away after its creator
    5745             :              * transaction aborted.  So behave as if we got to the end of the
    5746             :              * chain, and there's no further tuple to lock: return success to
    5747             :              * caller.
    5748             :              */
    5749           0 :             result = TM_Ok;
    5750           0 :             goto out_unlocked;
    5751             :         }
    5752             : 
    5753         168 : l4:
    5754         184 :         CHECK_FOR_INTERRUPTS();
    5755             : 
    5756             :         /*
    5757             :          * Before locking the buffer, pin the visibility map page if it
    5758             :          * appears to be necessary.  Since we haven't got the lock yet,
    5759             :          * someone else might be in the middle of changing this, so we'll need
    5760             :          * to recheck after we have the lock.
    5761             :          */
    5762         184 :         if (PageIsAllVisible(BufferGetPage(buf)))
    5763             :         {
    5764           0 :             visibilitymap_pin(rel, block, &vmbuffer);
    5765           0 :             pinned_desired_page = true;
    5766             :         }
    5767             :         else
    5768         184 :             pinned_desired_page = false;
    5769             : 
    5770         184 :         LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
    5771             : 
    5772             :         /*
    5773             :          * If we didn't pin the visibility map page and the page has become
    5774             :          * all visible while we were busy locking the buffer, we'll have to
    5775             :          * unlock and re-lock, to avoid holding the buffer lock across I/O.
    5776             :          * That's a bit unfortunate, but hopefully shouldn't happen often.
    5777             :          *
    5778             :          * Note: in some paths through this function, we will reach here
    5779             :          * holding a pin on a vm page that may or may not be the one matching
    5780             :          * this page.  If this page isn't all-visible, we won't use the vm
    5781             :          * page, but we hold onto such a pin till the end of the function.
    5782             :          */
    5783         184 :         if (!pinned_desired_page && PageIsAllVisible(BufferGetPage(buf)))
    5784             :         {
    5785           0 :             LockBuffer(buf, BUFFER_LOCK_UNLOCK);
    5786           0 :             visibilitymap_pin(rel, block, &vmbuffer);
    5787           0 :             LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE);
    5788             :         }
    5789             : 
    5790             :         /*
    5791             :          * Check the tuple XMIN against prior XMAX, if any.  If we reached the
    5792             :          * end of the chain, we're done, so return success.
    5793             :          */
    5794         190 :         if (TransactionIdIsValid(priorXmax) &&
    5795           6 :             !TransactionIdEquals(HeapTupleHeaderGetXmin(mytup.t_data),
    5796             :                                  priorXmax))
    5797             :         {
    5798           0 :             result = TM_Ok;
    5799           0 :             goto out_locked;
    5800             :         }
    5801             : 
    5802             :         /*
    5803             :          * Also check Xmin: if this tuple was created by an aborted
    5804             :          * (sub)transaction, then we already locked the last live one in the
    5805             :          * chain, thus we're done, so return success.
    5806             :          */
    5807         184 :         if (TransactionIdDidAbort(HeapTupleHeaderGetXmin(mytup.t_data)))
    5808             :         {
    5809          26 :             result = TM_Ok;
    5810          26 :             goto out_locked;
    5811             :         }
    5812             : 
    5813         158 :         old_infomask = mytup.t_data->t_infomask;
    5814         158 :         old_infomask2 = mytup.t_data->t_infomask2;
    5815         158 :         xmax = HeapTupleHeaderGetRawXmax(mytup.t_data);
    5816             : 
    5817             :         /*
    5818             :          * If this tuple version has been updated or locked by some concurrent
    5819             :          * transaction(s), what we do depends on whether our lock mode
    5820             :          * conflicts with what those other transactions hold, and also on the
    5821             :          * status of them.
    5822             :          */
    5823         158 :         if (!(old_infomask & HEAP_XMAX_INVALID))
    5824             :         {
    5825             :             TransactionId rawxmax;
    5826             :             bool        needwait;
    5827             : 
    5828          60 :             rawxmax = HeapTupleHeaderGetRawXmax(mytup.t_data);
    5829          60 :             if (old_infomask & HEAP_XMAX_IS_MULTI)
    5830             :             {
    5831             :                 int         nmembers;
    5832             :                 int         i;
    5833             :                 MultiXactMember *members;
    5834             : 
    5835             :                 /*
    5836             :                  * We don't need a test for pg_upgrade'd tuples: this is only
    5837             :                  * applied to tuples after the first in an update chain.  Said
    5838             :                  * first tuple in the chain may well be locked-in-9.2-and-
    5839             :                  * pg_upgraded, but that one was already locked by our caller,
    5840             :                  * not us; and any subsequent ones cannot be because our
    5841             :                  * caller must necessarily have obtained a snapshot later than
    5842             :                  * the pg_upgrade itself.
    5843             :                  */
    5844             :                 Assert(!HEAP_LOCKED_UPGRADED(mytup.t_data->t_infomask));
    5845             : 
    5846           2 :                 nmembers = GetMultiXactIdMembers(rawxmax, &members, false,
    5847           2 :                                                  HEAP_XMAX_IS_LOCKED_ONLY(old_infomask));
    5848           8 :                 for (i = 0; i < nmembers; i++)
    5849             :                 {
    5850           6 :                     result = test_lockmode_for_conflict(members[i].status,
    5851           6 :                                                         members[i].xid,
    5852             :                                                         mode,
    5853             :                                                         &mytup,
    5854             :                                                         &needwait);
    5855             : 
    5856             :                     /*
    5857             :                      * If the tuple was already locked by ourselves in a
    5858             :                      * previous iteration of this (say heap_lock_tuple was
    5859             :                      * forced to restart the locking loop because of a change
    5860             :                      * in xmax), then we hold the lock already on this tuple
    5861             :                      * version and we don't need to do anything; and this is
    5862             :                      * not an error condition either.  We just need to skip
    5863             :                      * this tuple and continue locking the next version in the
    5864             :                      * update chain.
    5865             :                      */
    5866           6 :                     if (result == TM_SelfModified)
    5867             :                     {
    5868           0 :                         pfree(members);
    5869           0 :                         goto next;
    5870             :                     }
    5871             : 
    5872           6 :                     if (needwait)
    5873             :                     {
    5874           0 :                         LockBuffer(buf, BUFFER_LOCK_UNLOCK);
    5875           0 :                         XactLockTableWait(members[i].xid, rel,
    5876             :                                           &mytup.t_self,
    5877             :                                           XLTW_LockUpdated);
    5878           0 :                         pfree(members);
    5879           0 :                         goto l4;
    5880             :                     }
    5881           6 :                     if (result != TM_Ok)
    5882             :                     {
    5883           0 :                         pfree(members);
    5884           0 :                         goto out_locked;
    5885             :                     }
    5886             :                 }
    5887           2 :                 if (members)
    5888           2 :                     pfree(members);
    5889             :             }
    5890             :             else
    5891             :             {
    5892             :                 MultiXactStatus status;
    5893             : 
    5894             :                 /*
    5895             :                  * For a non-multi Xmax, we first need to compute the
    5896             :                  * corresponding MultiXactStatus by using the infomask bits.
    5897             :                  */
    5898          58 :                 if (HEAP_XMAX_IS_LOCKED_ONLY(old_infomask))
    5899             :                 {
    5900          20 :                     if (HEAP_XMAX_IS_KEYSHR_LOCKED(old_infomask))
    5901          20 :                         status = MultiXactStatusForKeyShare;
    5902           0 :                     else if (HEAP_XMAX_IS_SHR_LOCKED(old_infomask))
    5903           0 :                         status = MultiXactStatusForShare;
    5904           0 :                     else if (HEAP_XMAX_IS_EXCL_LOCKED(old_infomask))
    5905             :                     {
    5906           0 :                         if (old_infomask2 & HEAP_KEYS_UPDATED)
    5907           0 :                             status = MultiXactStatusForUpdate;
    5908             :                         else
    5909           0 :                             status = MultiXactStatusForNoKeyUpdate;
    5910             :                     }
    5911             :                     else
    5912             :                     {
    5913             :                         /*
    5914             :                          * LOCK_ONLY present alone (a pg_upgraded tuple marked
    5915             :                          * as share-locked in the old cluster) shouldn't be
    5916             :                          * seen in the middle of an update chain.
    5917             :                          */
    5918           0 :                         elog(ERROR, "invalid lock status in tuple");
    5919             :                     }
    5920             :                 }
    5921             :                 else
    5922             :                 {
    5923             :                     /* it's an update, but which kind? */
    5924          38 :                     if (old_infomask2 & HEAP_KEYS_UPDATED)
    5925          28 :                         status = MultiXactStatusUpdate;
    5926             :                     else
    5927          10 :                         status = MultiXactStatusNoKeyUpdate;
    5928             :                 }
    5929             : 
    5930          58 :                 result = test_lockmode_for_conflict(status, rawxmax, mode,
    5931             :                                                     &mytup, &needwait);
    5932             : 
    5933             :                 /*
    5934             :                  * If the tuple was already locked by ourselves in a previous
    5935             :                  * iteration of this (say heap_lock_tuple was forced to
    5936             :                  * restart the locking loop because of a change in xmax), then
    5937             :                  * we hold the lock already on this tuple version and we don't
    5938             :                  * need to do anything; and this is not an error condition
    5939             :                  * either.  We just need to skip this tuple and continue
    5940             :                  * locking the next version in the update chain.
    5941             :                  */
    5942          58 :                 if (result == TM_SelfModified)
    5943           0 :                     goto next;
    5944             : 
    5945          58 :                 if (needwait)
    5946             :                 {
    5947          16 :                     LockBuffer(buf, BUFFER_LOCK_UNLOCK);
    5948          16 :                     XactLockTableWait(rawxmax, rel, &mytup.t_self,
    5949             :                                       XLTW_LockUpdated);
    5950          16 :                     goto l4;
    5951             :                 }
    5952          42 :                 if (result != TM_Ok)
    5953             :                 {
    5954          16 :                     goto out_locked;
    5955             :                 }
    5956             :             }
    5957             :         }
    5958             : 
    5959             :         /* compute the new Xmax and infomask values for the tuple ... */
    5960         126 :         compute_new_xmax_infomask(xmax, old_infomask, mytup.t_data->t_infomask2,
    5961             :                                   xid, mode, false,
    5962             :                                   &new_xmax, &new_infomask, &new_infomask2);
    5963             : 
    5964         126 :         if (PageIsAllVisible(BufferGetPage(buf)) &&
    5965           0 :             visibilitymap_clear(rel, block, vmbuffer,
    5966             :                                 VISIBILITYMAP_ALL_FROZEN))
    5967           0 :             cleared_all_frozen = true;
    5968             : 
    5969         126 :         START_CRIT_SECTION();
    5970             : 
    5971             :         /* ... and set them */
    5972         126 :         HeapTupleHeaderSetXmax(mytup.t_data, new_xmax);
    5973         126 :         mytup.t_data->t_infomask &= ~HEAP_XMAX_BITS;
    5974         126 :         mytup.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    5975         126 :         mytup.t_data->t_infomask |= new_infomask;
    5976         126 :         mytup.t_data->t_infomask2 |= new_infomask2;
    5977             : 
    5978         126 :         MarkBufferDirty(buf);
    5979             : 
    5980             :         /* XLOG stuff */
    5981         126 :         if (RelationNeedsWAL(rel))
    5982             :         {
    5983             :             xl_heap_lock_updated xlrec;
    5984             :             XLogRecPtr  recptr;
    5985         126 :             Page        page = BufferGetPage(buf);
    5986             : 
    5987         126 :             XLogBeginInsert();
    5988         126 :             XLogRegisterBuffer(0, buf, REGBUF_STANDARD);
    5989             : 
    5990         126 :             xlrec.offnum = ItemPointerGetOffsetNumber(&mytup.t_self);
    5991         126 :             xlrec.xmax = new_xmax;
    5992         126 :             xlrec.infobits_set = compute_infobits(new_infomask, new_infomask2);
    5993         126 :             xlrec.flags =
    5994         126 :                 cleared_all_frozen ? XLH_LOCK_ALL_FROZEN_CLEARED : 0;
    5995             : 
    5996         126 :             XLogRegisterData(&xlrec, SizeOfHeapLockUpdated);
    5997             : 
    5998         126 :             recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_LOCK_UPDATED);
    5999             : 
    6000         126 :             PageSetLSN(page, recptr);
    6001             :         }
    6002             : 
    6003         126 :         END_CRIT_SECTION();
    6004             : 
    6005         126 : next:
    6006             :         /* if we find the end of update chain, we're done. */
    6007         252 :         if (mytup.t_data->t_infomask & HEAP_XMAX_INVALID ||
    6008         252 :             HeapTupleHeaderIndicatesMovedPartitions(mytup.t_data) ||
    6009         134 :             ItemPointerEquals(&mytup.t_self, &mytup.t_data->t_ctid) ||
    6010           8 :             HeapTupleHeaderIsOnlyLocked(mytup.t_data))
    6011             :         {
    6012         120 :             result = TM_Ok;
    6013         120 :             goto out_locked;
    6014             :         }
    6015             : 
    6016             :         /* tail recursion */
    6017           6 :         priorXmax = HeapTupleHeaderGetUpdateXid(mytup.t_data);
    6018           6 :         ItemPointerCopy(&(mytup.t_data->t_ctid), &tupid);
    6019           6 :         UnlockReleaseBuffer(buf);
    6020             :     }
    6021             : 
    6022             :     result = TM_Ok;
    6023             : 
    6024         162 : out_locked:
    6025         162 :     UnlockReleaseBuffer(buf);
    6026             : 
    6027         162 : out_unlocked:
    6028         162 :     if (vmbuffer != InvalidBuffer)
    6029           0 :         ReleaseBuffer(vmbuffer);
    6030             : 
    6031         162 :     return result;
    6032             : }
    6033             : 
    6034             : /*
    6035             :  * heap_lock_updated_tuple
    6036             :  *      Follow update chain when locking an updated tuple, acquiring locks (row
    6037             :  *      marks) on the updated versions.
    6038             :  *
    6039             :  * The initial tuple is assumed to be already locked.
    6040             :  *
    6041             :  * This function doesn't check visibility, it just unconditionally marks the
    6042             :  * tuple(s) as locked.  If any tuple in the updated chain is being deleted
    6043             :  * concurrently (or updated with the key being modified), sleep until the
    6044             :  * transaction doing it is finished.
    6045             :  *
    6046             :  * Note that we don't acquire heavyweight tuple locks on the tuples we walk
    6047             :  * when we have to wait for other transactions to release them, as opposed to
    6048             :  * what heap_lock_tuple does.  The reason is that having more than one
    6049             :  * transaction walking the chain is probably uncommon enough that risk of
    6050             :  * starvation is not likely: one of the preconditions for being here is that
    6051             :  * the snapshot in use predates the update that created this tuple (because we
    6052             :  * started at an earlier version of the tuple), but at the same time such a
    6053             :  * transaction cannot be using repeatable read or serializable isolation
    6054             :  * levels, because that would lead to a serializability failure.
    6055             :  */
    6056             : static TM_Result
    6057         180 : heap_lock_updated_tuple(Relation rel, HeapTuple tuple, ItemPointer ctid,
    6058             :                         TransactionId xid, LockTupleMode mode)
    6059             : {
    6060             :     /*
    6061             :      * If the tuple has not been updated, or has moved into another partition
    6062             :      * (effectively a delete) stop here.
    6063             :      */
    6064         180 :     if (!HeapTupleHeaderIndicatesMovedPartitions(tuple->t_data) &&
    6065         176 :         !ItemPointerEquals(&tuple->t_self, ctid))
    6066             :     {
    6067             :         /*
    6068             :          * If this is the first possibly-multixact-able operation in the
    6069             :          * current transaction, set my per-backend OldestMemberMXactId
    6070             :          * setting. We can be certain that the transaction will never become a
    6071             :          * member of any older MultiXactIds than that.  (We have to do this
    6072             :          * even if we end up just using our own TransactionId below, since
    6073             :          * some other backend could incorporate our XID into a MultiXact
    6074             :          * immediately afterwards.)
    6075             :          */
    6076         162 :         MultiXactIdSetOldestMember();
    6077             : 
    6078         162 :         return heap_lock_updated_tuple_rec(rel, ctid, xid, mode);
    6079             :     }
    6080             : 
    6081             :     /* nothing to lock */
    6082          18 :     return TM_Ok;
    6083             : }
    6084             : 
    6085             : /*
    6086             :  *  heap_finish_speculative - mark speculative insertion as successful
    6087             :  *
    6088             :  * To successfully finish a speculative insertion we have to clear speculative
    6089             :  * token from tuple.  To do so the t_ctid field, which will contain a
    6090             :  * speculative token value, is modified in place to point to the tuple itself,
    6091             :  * which is characteristic of a newly inserted ordinary tuple.
    6092             :  *
    6093             :  * NB: It is not ok to commit without either finishing or aborting a
    6094             :  * speculative insertion.  We could treat speculative tuples of committed
    6095             :  * transactions implicitly as completed, but then we would have to be prepared
    6096             :  * to deal with speculative tokens on committed tuples.  That wouldn't be
    6097             :  * difficult - no-one looks at the ctid field of a tuple with invalid xmax -
    6098             :  * but clearing the token at completion isn't very expensive either.
    6099             :  * An explicit confirmation WAL record also makes logical decoding simpler.
    6100             :  */
    6101             : void
    6102        4112 : heap_finish_speculative(Relation relation, ItemPointer tid)
    6103             : {
    6104             :     Buffer      buffer;
    6105             :     Page        page;
    6106             :     OffsetNumber offnum;
    6107        4112 :     ItemId      lp = NULL;
    6108             :     HeapTupleHeader htup;
    6109             : 
    6110        4112 :     buffer = ReadBuffer(relation, ItemPointerGetBlockNumber(tid));
    6111        4112 :     LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    6112        4112 :     page = (Page) BufferGetPage(buffer);
    6113             : 
    6114        4112 :     offnum = ItemPointerGetOffsetNumber(tid);
    6115        4112 :     if (PageGetMaxOffsetNumber(page) >= offnum)
    6116        4112 :         lp = PageGetItemId(page, offnum);
    6117             : 
    6118        4112 :     if (PageGetMaxOffsetNumber(page) < offnum || !ItemIdIsNormal(lp))
    6119           0 :         elog(ERROR, "invalid lp");
    6120             : 
    6121        4112 :     htup = (HeapTupleHeader) PageGetItem(page, lp);
    6122             : 
    6123             :     /* NO EREPORT(ERROR) from here till changes are logged */
    6124        4112 :     START_CRIT_SECTION();
    6125             : 
    6126             :     Assert(HeapTupleHeaderIsSpeculative(htup));
    6127             : 
    6128        4112 :     MarkBufferDirty(buffer);
    6129             : 
    6130             :     /*
    6131             :      * Replace the speculative insertion token with a real t_ctid, pointing to
    6132             :      * itself like it does on regular tuples.
    6133             :      */
    6134        4112 :     htup->t_ctid = *tid;
    6135             : 
    6136             :     /* XLOG stuff */
    6137        4112 :     if (RelationNeedsWAL(relation))
    6138             :     {
    6139             :         xl_heap_confirm xlrec;
    6140             :         XLogRecPtr  recptr;
    6141             : 
    6142        4094 :         xlrec.offnum = ItemPointerGetOffsetNumber(tid);
    6143             : 
    6144        4094 :         XLogBeginInsert();
    6145             : 
    6146             :         /* We want the same filtering on this as on a plain insert */
    6147        4094 :         XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
    6148             : 
    6149        4094 :         XLogRegisterData(&xlrec, SizeOfHeapConfirm);
    6150        4094 :         XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
    6151             : 
    6152        4094 :         recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_CONFIRM);
    6153             : 
    6154        4094 :         PageSetLSN(page, recptr);
    6155             :     }
    6156             : 
    6157        4112 :     END_CRIT_SECTION();
    6158             : 
    6159        4112 :     UnlockReleaseBuffer(buffer);
    6160        4112 : }
    6161             : 
    6162             : /*
    6163             :  *  heap_abort_speculative - kill a speculatively inserted tuple
    6164             :  *
    6165             :  * Marks a tuple that was speculatively inserted in the same command as dead,
    6166             :  * by setting its xmin as invalid.  That makes it immediately appear as dead
    6167             :  * to all transactions, including our own.  In particular, it makes
    6168             :  * HeapTupleSatisfiesDirty() regard the tuple as dead, so that another backend
    6169             :  * inserting a duplicate key value won't unnecessarily wait for our whole
    6170             :  * transaction to finish (it'll just wait for our speculative insertion to
    6171             :  * finish).
    6172             :  *
    6173             :  * Killing the tuple prevents "unprincipled deadlocks", which are deadlocks
    6174             :  * that arise due to a mutual dependency that is not user visible.  By
    6175             :  * definition, unprincipled deadlocks cannot be prevented by the user
    6176             :  * reordering lock acquisition in client code, because the implementation level
    6177             :  * lock acquisitions are not under the user's direct control.  If speculative
    6178             :  * inserters did not take this precaution, then under high concurrency they
    6179             :  * could deadlock with each other, which would not be acceptable.
    6180             :  *
    6181             :  * This is somewhat redundant with heap_delete, but we prefer to have a
    6182             :  * dedicated routine with stripped down requirements.  Note that this is also
    6183             :  * used to delete the TOAST tuples created during speculative insertion.
    6184             :  *
    6185             :  * This routine does not affect logical decoding as it only looks at
    6186             :  * confirmation records.
    6187             :  */
    6188             : void
    6189          20 : heap_abort_speculative(Relation relation, ItemPointer tid)
    6190             : {
    6191          20 :     TransactionId xid = GetCurrentTransactionId();
    6192             :     ItemId      lp;
    6193             :     HeapTupleData tp;
    6194             :     Page        page;
    6195             :     BlockNumber block;
    6196             :     Buffer      buffer;
    6197             : 
    6198             :     Assert(ItemPointerIsValid(tid));
    6199             : 
    6200          20 :     block = ItemPointerGetBlockNumber(tid);
    6201          20 :     buffer = ReadBuffer(relation, block);
    6202          20 :     page = BufferGetPage(buffer);
    6203             : 
    6204          20 :     LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    6205             : 
    6206             :     /*
    6207             :      * Page can't be all visible, we just inserted into it, and are still
    6208             :      * running.
    6209             :      */
    6210             :     Assert(!PageIsAllVisible(page));
    6211             : 
    6212          20 :     lp = PageGetItemId(page, ItemPointerGetOffsetNumber(tid));
    6213             :     Assert(ItemIdIsNormal(lp));
    6214             : 
    6215          20 :     tp.t_tableOid = RelationGetRelid(relation);
    6216          20 :     tp.t_data = (HeapTupleHeader) PageGetItem(page, lp);
    6217          20 :     tp.t_len = ItemIdGetLength(lp);
    6218          20 :     tp.t_self = *tid;
    6219             : 
    6220             :     /*
    6221             :      * Sanity check that the tuple really is a speculatively inserted tuple,
    6222             :      * inserted by us.
    6223             :      */
    6224          20 :     if (tp.t_data->t_choice.t_heap.t_xmin != xid)
    6225           0 :         elog(ERROR, "attempted to kill a tuple inserted by another transaction");
    6226          20 :     if (!(IsToastRelation(relation) || HeapTupleHeaderIsSpeculative(tp.t_data)))
    6227           0 :         elog(ERROR, "attempted to kill a non-speculative tuple");
    6228             :     Assert(!HeapTupleHeaderIsHeapOnly(tp.t_data));
    6229             : 
    6230             :     /*
    6231             :      * No need to check for serializable conflicts here.  There is never a
    6232             :      * need for a combo CID, either.  No need to extract replica identity, or
    6233             :      * do anything special with infomask bits.
    6234             :      */
    6235             : 
    6236          20 :     START_CRIT_SECTION();
    6237             : 
    6238             :     /*
    6239             :      * The tuple will become DEAD immediately.  Flag that this page is a
    6240             :      * candidate for pruning by setting xmin to TransactionXmin. While not
    6241             :      * immediately prunable, it is the oldest xid we can cheaply determine
    6242             :      * that's safe against wraparound / being older than the table's
    6243             :      * relfrozenxid.  To defend against the unlikely case of a new relation
    6244             :      * having a newer relfrozenxid than our TransactionXmin, use relfrozenxid
    6245             :      * if so (vacuum can't subsequently move relfrozenxid to beyond
    6246             :      * TransactionXmin, so there's no race here).
    6247             :      */
    6248             :     Assert(TransactionIdIsValid(TransactionXmin));
    6249             :     {
    6250          20 :         TransactionId relfrozenxid = relation->rd_rel->relfrozenxid;
    6251             :         TransactionId prune_xid;
    6252             : 
    6253          20 :         if (TransactionIdPrecedes(TransactionXmin, relfrozenxid))
    6254           0 :             prune_xid = relfrozenxid;
    6255             :         else
    6256          20 :             prune_xid = TransactionXmin;
    6257          20 :         PageSetPrunable(page, prune_xid);
    6258             :     }
    6259             : 
    6260             :     /* store transaction information of xact deleting the tuple */
    6261          20 :     tp.t_data->t_infomask &= ~(HEAP_XMAX_BITS | HEAP_MOVED);
    6262          20 :     tp.t_data->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    6263             : 
    6264             :     /*
    6265             :      * Set the tuple header xmin to InvalidTransactionId.  This makes the
    6266             :      * tuple immediately invisible everyone.  (In particular, to any
    6267             :      * transactions waiting on the speculative token, woken up later.)
    6268             :      */
    6269          20 :     HeapTupleHeaderSetXmin(tp.t_data, InvalidTransactionId);
    6270             : 
    6271             :     /* Clear the speculative insertion token too */
    6272          20 :     tp.t_data->t_ctid = tp.t_self;
    6273             : 
    6274          20 :     MarkBufferDirty(buffer);
    6275             : 
    6276             :     /*
    6277             :      * XLOG stuff
    6278             :      *
    6279             :      * The WAL records generated here match heap_delete().  The same recovery
    6280             :      * routines are used.
    6281             :      */
    6282          20 :     if (RelationNeedsWAL(relation))
    6283             :     {
    6284             :         xl_heap_delete xlrec;
    6285             :         XLogRecPtr  recptr;
    6286             : 
    6287          20 :         xlrec.flags = XLH_DELETE_IS_SUPER;
    6288          40 :         xlrec.infobits_set = compute_infobits(tp.t_data->t_infomask,
    6289          20 :                                               tp.t_data->t_infomask2);
    6290          20 :         xlrec.offnum = ItemPointerGetOffsetNumber(&tp.t_self);
    6291          20 :         xlrec.xmax = xid;
    6292             : 
    6293          20 :         XLogBeginInsert();
    6294          20 :         XLogRegisterData(&xlrec, SizeOfHeapDelete);
    6295          20 :         XLogRegisterBuffer(0, buffer, REGBUF_STANDARD);
    6296             : 
    6297             :         /* No replica identity & replication origin logged */
    6298             : 
    6299          20 :         recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_DELETE);
    6300             : 
    6301          20 :         PageSetLSN(page, recptr);
    6302             :     }
    6303             : 
    6304          20 :     END_CRIT_SECTION();
    6305             : 
    6306          20 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    6307             : 
    6308          20 :     if (HeapTupleHasExternal(&tp))
    6309             :     {
    6310             :         Assert(!IsToastRelation(relation));
    6311           2 :         heap_toast_delete(relation, &tp, true);
    6312             :     }
    6313             : 
    6314             :     /*
    6315             :      * Never need to mark tuple for invalidation, since catalogs don't support
    6316             :      * speculative insertion
    6317             :      */
    6318             : 
    6319             :     /* Now we can release the buffer */
    6320          20 :     ReleaseBuffer(buffer);
    6321             : 
    6322             :     /* count deletion, as we counted the insertion too */
    6323          20 :     pgstat_count_heap_delete(relation);
    6324          20 : }
    6325             : 
    6326             : /*
    6327             :  * heap_inplace_lock - protect inplace update from concurrent heap_update()
    6328             :  *
    6329             :  * Evaluate whether the tuple's state is compatible with a no-key update.
    6330             :  * Current transaction rowmarks are fine, as is KEY SHARE from any
    6331             :  * transaction.  If compatible, return true with the buffer exclusive-locked,
    6332             :  * and the caller must release that by calling
    6333             :  * heap_inplace_update_and_unlock(), calling heap_inplace_unlock(), or raising
    6334             :  * an error.  Otherwise, call release_callback(arg), wait for blocking
    6335             :  * transactions to end, and return false.
    6336             :  *
    6337             :  * Since this is intended for system catalogs and SERIALIZABLE doesn't cover
    6338             :  * DDL, this doesn't guarantee any particular predicate locking.
    6339             :  *
    6340             :  * One could modify this to return true for tuples with delete in progress,
    6341             :  * All inplace updaters take a lock that conflicts with DROP.  If explicit
    6342             :  * "DELETE FROM pg_class" is in progress, we'll wait for it like we would an
    6343             :  * update.
    6344             :  *
    6345             :  * Readers of inplace-updated fields expect changes to those fields are
    6346             :  * durable.  For example, vac_truncate_clog() reads datfrozenxid from
    6347             :  * pg_database tuples via catalog snapshots.  A future snapshot must not
    6348             :  * return a lower datfrozenxid for the same database OID (lower in the
    6349             :  * FullTransactionIdPrecedes() sense).  We achieve that since no update of a
    6350             :  * tuple can start while we hold a lock on its buffer.  In cases like
    6351             :  * BEGIN;GRANT;CREATE INDEX;COMMIT we're inplace-updating a tuple visible only
    6352             :  * to this transaction.  ROLLBACK then is one case where it's okay to lose
    6353             :  * inplace updates.  (Restoring relhasindex=false on ROLLBACK is fine, since
    6354             :  * any concurrent CREATE INDEX would have blocked, then inplace-updated the
    6355             :  * committed tuple.)
    6356             :  *
    6357             :  * In principle, we could avoid waiting by overwriting every tuple in the
    6358             :  * updated tuple chain.  Reader expectations permit updating a tuple only if
    6359             :  * it's aborted, is the tail of the chain, or we already updated the tuple
    6360             :  * referenced in its t_ctid.  Hence, we would need to overwrite the tuples in
    6361             :  * order from tail to head.  That would imply either (a) mutating all tuples
    6362             :  * in one critical section or (b) accepting a chance of partial completion.
    6363             :  * Partial completion of a relfrozenxid update would have the weird
    6364             :  * consequence that the table's next VACUUM could see the table's relfrozenxid
    6365             :  * move forward between vacuum_get_cutoffs() and finishing.
    6366             :  */
    6367             : bool
    6368      269300 : heap_inplace_lock(Relation relation,
    6369             :                   HeapTuple oldtup_ptr, Buffer buffer,
    6370             :                   void (*release_callback) (void *), void *arg)
    6371             : {
    6372      269300 :     HeapTupleData oldtup = *oldtup_ptr; /* minimize diff vs. heap_update() */
    6373             :     TM_Result   result;
    6374             :     bool        ret;
    6375             : 
    6376             : #ifdef USE_ASSERT_CHECKING
    6377             :     if (RelationGetRelid(relation) == RelationRelationId)
    6378             :         check_inplace_rel_lock(oldtup_ptr);
    6379             : #endif
    6380             : 
    6381             :     Assert(BufferIsValid(buffer));
    6382             : 
    6383             :     /*
    6384             :      * Construct shared cache inval if necessary.  Because we pass a tuple
    6385             :      * version without our own inplace changes or inplace changes other
    6386             :      * sessions complete while we wait for locks, inplace update mustn't
    6387             :      * change catcache lookup keys.  But we aren't bothering with index
    6388             :      * updates either, so that's true a fortiori.  After LockBuffer(), it
    6389             :      * would be too late, because this might reach a
    6390             :      * CatalogCacheInitializeCache() that locks "buffer".
    6391             :      */
    6392      269300 :     CacheInvalidateHeapTupleInplace(relation, oldtup_ptr, NULL);
    6393             : 
    6394      269300 :     LockTuple(relation, &oldtup.t_self, InplaceUpdateTupleLock);
    6395      269300 :     LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
    6396             : 
    6397             :     /*----------
    6398             :      * Interpret HeapTupleSatisfiesUpdate() like heap_update() does, except:
    6399             :      *
    6400             :      * - wait unconditionally
    6401             :      * - already locked tuple above, since inplace needs that unconditionally
    6402             :      * - don't recheck header after wait: simpler to defer to next iteration
    6403             :      * - don't try to continue even if the updater aborts: likewise
    6404             :      * - no crosscheck
    6405             :      */
    6406      269300 :     result = HeapTupleSatisfiesUpdate(&oldtup, GetCurrentCommandId(false),
    6407             :                                       buffer);
    6408             : 
    6409      269300 :     if (result == TM_Invisible)
    6410             :     {
    6411             :         /* no known way this can happen */
    6412           0 :         ereport(ERROR,
    6413             :                 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
    6414             :                  errmsg_internal("attempted to overwrite invisible tuple")));
    6415             :     }
    6416      269300 :     else if (result == TM_SelfModified)
    6417             :     {
    6418             :         /*
    6419             :          * CREATE INDEX might reach this if an expression is silly enough to
    6420             :          * call e.g. SELECT ... FROM pg_class FOR SHARE.  C code of other SQL
    6421             :          * statements might get here after a heap_update() of the same row, in
    6422             :          * the absence of an intervening CommandCounterIncrement().
    6423             :          */
    6424           0 :         ereport(ERROR,
    6425             :                 (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
    6426             :                  errmsg("tuple to be updated was already modified by an operation triggered by the current command")));
    6427             :     }
    6428      269300 :     else if (result == TM_BeingModified)
    6429             :     {
    6430             :         TransactionId xwait;
    6431             :         uint16      infomask;
    6432             : 
    6433          44 :         xwait = HeapTupleHeaderGetRawXmax(oldtup.t_data);
    6434          44 :         infomask = oldtup.t_data->t_infomask;
    6435             : 
    6436          44 :         if (infomask & HEAP_XMAX_IS_MULTI)
    6437             :         {
    6438          10 :             LockTupleMode lockmode = LockTupleNoKeyExclusive;
    6439          10 :             MultiXactStatus mxact_status = MultiXactStatusNoKeyUpdate;
    6440             :             int         remain;
    6441             : 
    6442          10 :             if (DoesMultiXactIdConflict((MultiXactId) xwait, infomask,
    6443             :                                         lockmode, NULL))
    6444             :             {
    6445           4 :                 LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    6446           4 :                 release_callback(arg);
    6447           4 :                 ret = false;
    6448           4 :                 MultiXactIdWait((MultiXactId) xwait, mxact_status, infomask,
    6449             :                                 relation, &oldtup.t_self, XLTW_Update,
    6450             :                                 &remain);
    6451             :             }
    6452             :             else
    6453           6 :                 ret = true;
    6454             :         }
    6455          34 :         else if (TransactionIdIsCurrentTransactionId(xwait))
    6456           2 :             ret = true;
    6457          32 :         else if (HEAP_XMAX_IS_KEYSHR_LOCKED(infomask))
    6458           2 :             ret = true;
    6459             :         else
    6460             :         {
    6461          30 :             LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    6462          30 :             release_callback(arg);
    6463          30 :             ret = false;
    6464          30 :             XactLockTableWait(xwait, relation, &oldtup.t_self,
    6465             :                               XLTW_Update);
    6466             :         }
    6467             :     }
    6468             :     else
    6469             :     {
    6470      269256 :         ret = (result == TM_Ok);
    6471      269256 :         if (!ret)
    6472             :         {
    6473           0 :             LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    6474           0 :             release_callback(arg);
    6475             :         }
    6476             :     }
    6477             : 
    6478             :     /*
    6479             :      * GetCatalogSnapshot() relies on invalidation messages to know when to
    6480             :      * take a new snapshot.  COMMIT of xwait is responsible for sending the
    6481             :      * invalidation.  We're not acquiring heavyweight locks sufficient to
    6482             :      * block if not yet sent, so we must take a new snapshot to ensure a later
    6483             :      * attempt has a fair chance.  While we don't need this if xwait aborted,
    6484             :      * don't bother optimizing that.
    6485             :      */
    6486      269300 :     if (!ret)
    6487             :     {
    6488          34 :         UnlockTuple(relation, &oldtup.t_self, InplaceUpdateTupleLock);
    6489          34 :         ForgetInplace_Inval();
    6490          34 :         InvalidateCatalogSnapshot();
    6491             :     }
    6492      269300 :     return ret;
    6493             : }
    6494             : 
    6495             : /*
    6496             :  * heap_inplace_update_and_unlock - core of systable_inplace_update_finish
    6497             :  *
    6498             :  * The tuple cannot change size, and therefore its header fields and null
    6499             :  * bitmap (if any) don't change either.
    6500             :  *
    6501             :  * Since we hold LOCKTAG_TUPLE, no updater has a local copy of this tuple.
    6502             :  */
    6503             : void
    6504      156534 : heap_inplace_update_and_unlock(Relation relation,
    6505             :                                HeapTuple oldtup, HeapTuple tuple,
    6506             :                                Buffer buffer)
    6507             : {
    6508      156534 :     HeapTupleHeader htup = oldtup->t_data;
    6509             :     uint32      oldlen;
    6510             :     uint32      newlen;
    6511             :     char       *dst;
    6512             :     char       *src;
    6513      156534 :     int         nmsgs = 0;
    6514      156534 :     SharedInvalidationMessage *invalMessages = NULL;
    6515      156534 :     bool        RelcacheInitFileInval = false;
    6516             : 
    6517             :     Assert(ItemPointerEquals(&oldtup->t_self, &tuple->t_self));
    6518      156534 :     oldlen = oldtup->t_len - htup->t_hoff;
    6519      156534 :     newlen = tuple->t_len - tuple->t_data->t_hoff;
    6520      156534 :     if (oldlen != newlen || htup->t_hoff != tuple->t_data->t_hoff)
    6521           0 :         elog(ERROR, "wrong tuple length");
    6522             : 
    6523      156534 :     dst = (char *) htup + htup->t_hoff;
    6524      156534 :     src = (char *) tuple->t_data + tuple->t_data->t_hoff;
    6525             : 
    6526             :     /* Like RecordTransactionCommit(), log only if needed */
    6527      156534 :     if (XLogStandbyInfoActive())
    6528       92604 :         nmsgs = inplaceGetInvalidationMessages(&invalMessages,
    6529             :                                                &RelcacheInitFileInval);
    6530             : 
    6531             :     /*
    6532             :      * Unlink relcache init files as needed.  If unlinking, acquire
    6533             :      * RelCacheInitLock until after associated invalidations.  By doing this
    6534             :      * in advance, if we checkpoint and then crash between inplace
    6535             :      * XLogInsert() and inval, we don't rely on StartupXLOG() ->
    6536             :      * RelationCacheInitFileRemove().  That uses elevel==LOG, so replay would
    6537             :      * neglect to PANIC on EIO.
    6538             :      */
    6539      156534 :     PreInplace_Inval();
    6540             : 
    6541             :     /*----------
    6542             :      * NO EREPORT(ERROR) from here till changes are complete
    6543             :      *
    6544             :      * Our buffer lock won't stop a reader having already pinned and checked
    6545             :      * visibility for this tuple.  Hence, we write WAL first, then mutate the
    6546             :      * buffer.  Like in MarkBufferDirtyHint() or RecordTransactionCommit(),
    6547             :      * checkpoint delay makes that acceptable.  With the usual order of
    6548             :      * changes, a crash after memcpy() and before XLogInsert() could allow
    6549             :      * datfrozenxid to overtake relfrozenxid:
    6550             :      *
    6551             :      * ["D" is a VACUUM (ONLY_DATABASE_STATS)]
    6552             :      * ["R" is a VACUUM tbl]
    6553             :      * D: vac_update_datfrozenxid() -> systable_beginscan(pg_class)
    6554             :      * D: systable_getnext() returns pg_class tuple of tbl
    6555             :      * R: memcpy() into pg_class tuple of tbl
    6556             :      * D: raise pg_database.datfrozenxid, XLogInsert(), finish
    6557             :      * [crash]
    6558             :      * [recovery restores datfrozenxid w/o relfrozenxid]
    6559             :      *
    6560             :      * Like in MarkBufferDirtyHint() subroutine XLogSaveBufferForHint(), copy
    6561             :      * the buffer to the stack before logging.  Here, that facilitates a FPI
    6562             :      * of the post-mutation block before we accept other sessions seeing it.
    6563             :      */
    6564             :     Assert((MyProc->delayChkptFlags & DELAY_CHKPT_START) == 0);
    6565      156534 :     START_CRIT_SECTION();
    6566      156534 :     MyProc->delayChkptFlags |= DELAY_CHKPT_START;
    6567             : 
    6568             :     /* XLOG stuff */
    6569      156534 :     if (RelationNeedsWAL(relation))
    6570             :     {
    6571             :         xl_heap_inplace xlrec;
    6572             :         PGAlignedBlock copied_buffer;
    6573      156518 :         char       *origdata = (char *) BufferGetBlock(buffer);
    6574      156518 :         Page        page = BufferGetPage(buffer);
    6575      156518 :         uint16      lower = ((PageHeader) page)->pd_lower;
    6576      156518 :         uint16      upper = ((PageHeader) page)->pd_upper;
    6577             :         uintptr_t   dst_offset_in_block;
    6578             :         RelFileLocator rlocator;
    6579             :         ForkNumber  forkno;
    6580             :         BlockNumber blkno;
    6581             :         XLogRecPtr  recptr;
    6582             : 
    6583      156518 :         xlrec.offnum = ItemPointerGetOffsetNumber(&tuple->t_self);
    6584      156518 :         xlrec.dbId = MyDatabaseId;
    6585      156518 :         xlrec.tsId = MyDatabaseTableSpace;
    6586      156518 :         xlrec.relcacheInitFileInval = RelcacheInitFileInval;
    6587      156518 :         xlrec.nmsgs = nmsgs;
    6588             : 
    6589      156518 :         XLogBeginInsert();
    6590      156518 :         XLogRegisterData(&xlrec, MinSizeOfHeapInplace);
    6591      156518 :         if (nmsgs != 0)
    6592       65964 :             XLogRegisterData(invalMessages,
    6593             :                              nmsgs * sizeof(SharedInvalidationMessage));
    6594             : 
    6595             :         /* register block matching what buffer will look like after changes */
    6596      156518 :         memcpy(copied_buffer.data, origdata, lower);
    6597      156518 :         memcpy(copied_buffer.data + upper, origdata + upper, BLCKSZ - upper);
    6598      156518 :         dst_offset_in_block = dst - origdata;
    6599      156518 :         memcpy(copied_buffer.data + dst_offset_in_block, src, newlen);
    6600      156518 :         BufferGetTag(buffer, &rlocator, &forkno, &blkno);
    6601             :         Assert(forkno == MAIN_FORKNUM);
    6602      156518 :         XLogRegisterBlock(0, &rlocator, forkno, blkno, copied_buffer.data,
    6603             :                           REGBUF_STANDARD);
    6604      156518 :         XLogRegisterBufData(0, src, newlen);
    6605             : 
    6606             :         /* inplace updates aren't decoded atm, don't log the origin */
    6607             : 
    6608      156518 :         recptr = XLogInsert(RM_HEAP_ID, XLOG_HEAP_INPLACE);
    6609             : 
    6610      156518 :         PageSetLSN(page, recptr);
    6611             :     }
    6612             : 
    6613      156534 :     memcpy(dst, src, newlen);
    6614             : 
    6615      156534 :     MarkBufferDirty(buffer);
    6616             : 
    6617      156534 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    6618             : 
    6619             :     /*
    6620             :      * Send invalidations to shared queue.  SearchSysCacheLocked1() assumes we
    6621             :      * do this before UnlockTuple().
    6622             :      *
    6623             :      * If we're mutating a tuple visible only to this transaction, there's an
    6624             :      * equivalent transactional inval from the action that created the tuple,
    6625             :      * and this inval is superfluous.
    6626             :      */
    6627      156534 :     AtInplace_Inval();
    6628             : 
    6629      156534 :     MyProc->delayChkptFlags &= ~DELAY_CHKPT_START;
    6630      156534 :     END_CRIT_SECTION();
    6631      156534 :     UnlockTuple(relation, &tuple->t_self, InplaceUpdateTupleLock);
    6632             : 
    6633      156534 :     AcceptInvalidationMessages();   /* local processing of just-sent inval */
    6634             : 
    6635             :     /*
    6636             :      * Queue a transactional inval.  The immediate invalidation we just sent
    6637             :      * is the only one known to be necessary.  To reduce risk from the
    6638             :      * transition to immediate invalidation, continue sending a transactional
    6639             :      * invalidation like we've long done.  Third-party code might rely on it.
    6640             :      */
    6641      156534 :     if (!IsBootstrapProcessingMode())
    6642      129894 :         CacheInvalidateHeapTuple(relation, tuple, NULL);
    6643      156534 : }
    6644             : 
    6645             : /*
    6646             :  * heap_inplace_unlock - reverse of heap_inplace_lock
    6647             :  */
    6648             : void
    6649      112732 : heap_inplace_unlock(Relation relation,
    6650             :                     HeapTuple oldtup, Buffer buffer)
    6651             : {
    6652      112732 :     LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
    6653      112732 :     UnlockTuple(relation, &oldtup->t_self, InplaceUpdateTupleLock);
    6654      112732 :     ForgetInplace_Inval();
    6655      112732 : }
    6656             : 
    6657             : #define     FRM_NOOP                0x0001
    6658             : #define     FRM_INVALIDATE_XMAX     0x0002
    6659             : #define     FRM_RETURN_IS_XID       0x0004
    6660             : #define     FRM_RETURN_IS_MULTI     0x0008
    6661             : #define     FRM_MARK_COMMITTED      0x0010
    6662             : 
    6663             : /*
    6664             :  * FreezeMultiXactId
    6665             :  *      Determine what to do during freezing when a tuple is marked by a
    6666             :  *      MultiXactId.
    6667             :  *
    6668             :  * "flags" is an output value; it's used to tell caller what to do on return.
    6669             :  * "pagefrz" is an input/output value, used to manage page level freezing.
    6670             :  *
    6671             :  * Possible values that we can set in "flags":
    6672             :  * FRM_NOOP
    6673             :  *      don't do anything -- keep existing Xmax
    6674             :  * FRM_INVALIDATE_XMAX
    6675             :  *      mark Xmax as InvalidTransactionId and set XMAX_INVALID flag.
    6676             :  * FRM_RETURN_IS_XID
    6677             :  *      The Xid return value is a single update Xid to set as xmax.
    6678             :  * FRM_MARK_COMMITTED
    6679             :  *      Xmax can be marked as HEAP_XMAX_COMMITTED
    6680             :  * FRM_RETURN_IS_MULTI
    6681             :  *      The return value is a new MultiXactId to set as new Xmax.
    6682             :  *      (caller must obtain proper infomask bits using GetMultiXactIdHintBits)
    6683             :  *
    6684             :  * Caller delegates control of page freezing to us.  In practice we always
    6685             :  * force freezing of caller's page unless FRM_NOOP processing is indicated.
    6686             :  * We help caller ensure that XIDs < FreezeLimit and MXIDs < MultiXactCutoff
    6687             :  * can never be left behind.  We freely choose when and how to process each
    6688             :  * Multi, without ever violating the cutoff postconditions for freezing.
    6689             :  *
    6690             :  * It's useful to remove Multis on a proactive timeline (relative to freezing
    6691             :  * XIDs) to keep MultiXact member SLRU buffer misses to a minimum.  It can also
    6692             :  * be cheaper in the short run, for us, since we too can avoid SLRU buffer
    6693             :  * misses through eager processing.
    6694             :  *
    6695             :  * NB: Creates a _new_ MultiXactId when FRM_RETURN_IS_MULTI is set, though only
    6696             :  * when FreezeLimit and/or MultiXactCutoff cutoffs leave us with no choice.
    6697             :  * This can usually be put off, which is usually enough to avoid it altogether.
    6698             :  * Allocating new multis during VACUUM should be avoided on general principle;
    6699             :  * only VACUUM can advance relminmxid, so allocating new Multis here comes with
    6700             :  * its own special risks.
    6701             :  *
    6702             :  * NB: Caller must maintain "no freeze" NewRelfrozenXid/NewRelminMxid trackers
    6703             :  * using heap_tuple_should_freeze when we haven't forced page-level freezing.
    6704             :  *
    6705             :  * NB: Caller should avoid needlessly calling heap_tuple_should_freeze when we
    6706             :  * have already forced page-level freezing, since that might incur the same
    6707             :  * SLRU buffer misses that we specifically intended to avoid by freezing.
    6708             :  */
    6709             : static TransactionId
    6710          12 : FreezeMultiXactId(MultiXactId multi, uint16 t_infomask,
    6711             :                   const struct VacuumCutoffs *cutoffs, uint16 *flags,
    6712             :                   HeapPageFreeze *pagefrz)
    6713             : {
    6714             :     TransactionId newxmax;
    6715             :     MultiXactMember *members;
    6716             :     int         nmembers;
    6717             :     bool        need_replace;
    6718             :     int         nnewmembers;
    6719             :     MultiXactMember *newmembers;
    6720             :     bool        has_lockers;
    6721             :     TransactionId update_xid;
    6722             :     bool        update_committed;
    6723             :     TransactionId FreezePageRelfrozenXid;
    6724             : 
    6725          12 :     *flags = 0;
    6726             : 
    6727             :     /* We should only be called in Multis */
    6728             :     Assert(t_infomask & HEAP_XMAX_IS_MULTI);
    6729             : 
    6730          24 :     if (!MultiXactIdIsValid(multi) ||
    6731          12 :         HEAP_LOCKED_UPGRADED(t_infomask))
    6732             :     {
    6733           0 :         *flags |= FRM_INVALIDATE_XMAX;
    6734           0 :         pagefrz->freeze_required = true;
    6735           0 :         return InvalidTransactionId;
    6736             :     }
    6737          12 :     else if (MultiXactIdPrecedes(multi, cutoffs->relminmxid))
    6738           0 :         ereport(ERROR,
    6739             :                 (errcode(ERRCODE_DATA_CORRUPTED),
    6740             :                  errmsg_internal("found multixact %u from before relminmxid %u",
    6741             :                                  multi, cutoffs->relminmxid)));
    6742          12 :     else if (MultiXactIdPrecedes(multi, cutoffs->OldestMxact))
    6743             :     {
    6744             :         TransactionId update_xact;
    6745             : 
    6746             :         /*
    6747             :          * This old multi cannot possibly have members still running, but
    6748             :          * verify just in case.  If it was a locker only, it can be removed
    6749             :          * without any further consideration; but if it contained an update,
    6750             :          * we might need to preserve it.
    6751             :          */
    6752           8 :         if (MultiXactIdIsRunning(multi,
    6753           8 :                                  HEAP_XMAX_IS_LOCKED_ONLY(t_infomask)))
    6754           0 :             ereport(ERROR,
    6755             :                     (errcode(ERRCODE_DATA_CORRUPTED),
    6756             :                      errmsg_internal("multixact %u from before multi freeze cutoff %u found to be still running",
    6757             :                                      multi, cutoffs->OldestMxact)));
    6758             : 
    6759           8 :         if (HEAP_XMAX_IS_LOCKED_ONLY(t_infomask))
    6760             :         {
    6761           8 :             *flags |= FRM_INVALIDATE_XMAX;
    6762           8 :             pagefrz->freeze_required = true;
    6763           8 :             return InvalidTransactionId;
    6764             :         }
    6765             : 
    6766             :         /* replace multi with single XID for its updater? */
    6767           0 :         update_xact = MultiXactIdGetUpdateXid(multi, t_infomask);
    6768           0 :         if (TransactionIdPrecedes(update_xact, cutoffs->relfrozenxid))
    6769           0 :             ereport(ERROR,
    6770             :                     (errcode(ERRCODE_DATA_CORRUPTED),
    6771             :                      errmsg_internal("multixact %u contains update XID %u from before relfrozenxid %u",
    6772             :                                      multi, update_xact,
    6773             :                                      cutoffs->relfrozenxid)));
    6774           0 :         else if (TransactionIdPrecedes(update_xact, cutoffs->OldestXmin))
    6775             :         {
    6776             :             /*
    6777             :              * Updater XID has to have aborted (otherwise the tuple would have
    6778             :              * been pruned away instead, since updater XID is < OldestXmin).
    6779             :              * Just remove xmax.
    6780             :              */
    6781           0 :             if (TransactionIdDidCommit(update_xact))
    6782           0 :                 ereport(ERROR,
    6783             :                         (errcode(ERRCODE_DATA_CORRUPTED),
    6784             :                          errmsg_internal("multixact %u contains committed update XID %u from before removable cutoff %u",
    6785             :                                          multi, update_xact,
    6786             :                                          cutoffs->OldestXmin)));
    6787           0 :             *flags |= FRM_INVALIDATE_XMAX;
    6788           0 :             pagefrz->freeze_required = true;
    6789           0 :             return InvalidTransactionId;
    6790             :         }
    6791             : 
    6792             :         /* Have to keep updater XID as new xmax */
    6793           0 :         *flags |= FRM_RETURN_IS_XID;
    6794           0 :         pagefrz->freeze_required = true;
    6795           0 :         return update_xact;
    6796             :     }
    6797             : 
    6798             :     /*
    6799             :      * Some member(s) of this Multi may be below FreezeLimit xid cutoff, so we
    6800             :      * need to walk the whole members array to figure out what to do, if
    6801             :      * anything.
    6802             :      */
    6803             :     nmembers =
    6804           4 :         GetMultiXactIdMembers(multi, &members, false,
    6805           4 :                               HEAP_XMAX_IS_LOCKED_ONLY(t_infomask));
    6806           4 :     if (nmembers <= 0)
    6807             :     {
    6808             :         /* Nothing worth keeping */
    6809           0 :         *flags |= FRM_INVALIDATE_XMAX;
    6810           0 :         pagefrz->freeze_required = true;
    6811           0 :         return InvalidTransactionId;
    6812             :     }
    6813             : 
    6814             :     /*
    6815             :      * The FRM_NOOP case is the only case where we might need to ratchet back
    6816             :      * FreezePageRelfrozenXid or FreezePageRelminMxid.  It is also the only
    6817             :      * case where our caller might ratchet back its NoFreezePageRelfrozenXid
    6818             :      * or NoFreezePageRelminMxid "no freeze" trackers to deal with a multi.
    6819             :      * FRM_NOOP handling should result in the NewRelfrozenXid/NewRelminMxid
    6820             :      * trackers managed by VACUUM being ratcheting back by xmax to the degree
    6821             :      * required to make it safe to leave xmax undisturbed, independent of
    6822             :      * whether or not page freezing is triggered somewhere else.
    6823             :      *
    6824             :      * Our policy is to force freezing in every case other than FRM_NOOP,
    6825             :      * which obviates the need to maintain either set of trackers, anywhere.
    6826             :      * Every other case will reliably execute a freeze plan for xmax that
    6827             :      * either replaces xmax with an XID/MXID >= OldestXmin/OldestMxact, or
    6828             :      * sets xmax to an InvalidTransactionId XID, rendering xmax fully frozen.
    6829             :      * (VACUUM's NewRelfrozenXid/NewRelminMxid trackers are initialized with
    6830             :      * OldestXmin/OldestMxact, so later values never need to be tracked here.)
    6831             :      */
    6832           4 :     need_replace = false;
    6833           4 :     FreezePageRelfrozenXid = pagefrz->FreezePageRelfrozenXid;
    6834           8 :     for (int i = 0; i < nmembers; i++)
    6835             :     {
    6836           6 :         TransactionId xid = members[i].xid;
    6837             : 
    6838             :         Assert(!TransactionIdPrecedes(xid, cutoffs->relfrozenxid));
    6839             : 
    6840           6 :         if (TransactionIdPrecedes(xid, cutoffs->FreezeLimit))
    6841             :         {
    6842             :             /* Can't violate the FreezeLimit postcondition */
    6843           2 :             need_replace = true;
    6844           2 :             break;
    6845             :         }
    6846           4 :         if (TransactionIdPrecedes(xid, FreezePageRelfrozenXid))
    6847           0 :             FreezePageRelfrozenXid = xid;
    6848             :     }
    6849             : 
    6850             :     /* Can't violate the MultiXactCutoff postcondition, either */
    6851           4 :     if (!need_replace)
    6852           2 :         need_replace = MultiXactIdPrecedes(multi, cutoffs->MultiXactCutoff);
    6853             : 
    6854           4 :     if (!need_replace)
    6855             :     {
    6856             :         /*
    6857             :          * vacuumlazy.c might ratchet back NewRelminMxid, NewRelfrozenXid, or
    6858             :          * both together to make it safe to retain this particular multi after
    6859             :          * freezing its page
    6860             :          */
    6861           2 :         *flags |= FRM_NOOP;
    6862           2 :         pagefrz->FreezePageRelfrozenXid = FreezePageRelfrozenXid;
    6863           2 :         if (MultiXactIdPrecedes(multi, pagefrz->FreezePageRelminMxid))
    6864           0 :             pagefrz->FreezePageRelminMxid = multi;
    6865           2 :         pfree(members);
    6866           2 :         return multi;
    6867             :     }
    6868             : 
    6869             :     /*
    6870             :      * Do a more thorough second pass over the multi to figure out which
    6871             :      * member XIDs actually need to be kept.  Checking the precise status of
    6872             :      * individual members might even show that we don't need to keep anything.
    6873             :      * That is quite possible even though the Multi must be >= OldestMxact,
    6874             :      * since our second pass only keeps member XIDs when it's truly necessary;
    6875             :      * even member XIDs >= OldestXmin often won't be kept by second pass.
    6876             :      */
    6877           2 :     nnewmembers = 0;
    6878           2 :     newmembers = palloc(sizeof(MultiXactMember) * nmembers);
    6879           2 :     has_lockers = false;
    6880           2 :     update_xid = InvalidTransactionId;
    6881           2 :     update_committed = false;
    6882             : 
    6883             :     /*
    6884             :      * Determine whether to keep each member xid, or to ignore it instead
    6885             :      */
    6886           6 :     for (int i = 0; i < nmembers; i++)
    6887             :     {
    6888           4 :         TransactionId xid = members[i].xid;
    6889           4 :         MultiXactStatus mstatus = members[i].status;
    6890             : 
    6891             :         Assert(!TransactionIdPrecedes(xid, cutoffs->relfrozenxid));
    6892             : 
    6893           4 :         if (!ISUPDATE_from_mxstatus(mstatus))
    6894             :         {
    6895             :             /*
    6896             :              * Locker XID (not updater XID).  We only keep lockers that are
    6897             :              * still running.
    6898             :              */
    6899           8 :             if (TransactionIdIsCurrentTransactionId(xid) ||
    6900           4 :                 TransactionIdIsInProgress(xid))
    6901             :             {
    6902           2 :                 if (TransactionIdPrecedes(xid, cutoffs->OldestXmin))
    6903           0 :                     ereport(ERROR,
    6904             :                             (errcode(ERRCODE_DATA_CORRUPTED),
    6905             :                              errmsg_internal("multixact %u contains running locker XID %u from before removable cutoff %u",
    6906             :                                              multi, xid,
    6907             :                                              cutoffs->OldestXmin)));
    6908           2 :                 newmembers[nnewmembers++] = members[i];
    6909           2 :                 has_lockers = true;
    6910             :             }
    6911             : 
    6912           4 :             continue;
    6913             :         }
    6914             : 
    6915             :         /*
    6916             :          * Updater XID (not locker XID).  Should we keep it?
    6917             :          *
    6918             :          * Since the tuple wasn't totally removed when vacuum pruned, the
    6919             :          * update Xid cannot possibly be older than OldestXmin cutoff unless
    6920             :          * the updater XID aborted.  If the updater transaction is known
    6921             :          * aborted or crashed then it's okay to ignore it, otherwise not.
    6922             :          *
    6923             :          * In any case the Multi should never contain two updaters, whatever
    6924             :          * their individual commit status.  Check for that first, in passing.
    6925             :          */
    6926           0 :         if (TransactionIdIsValid(update_xid))
    6927           0 :             ereport(ERROR,
    6928             :                     (errcode(ERRCODE_DATA_CORRUPTED),
    6929             :                      errmsg_internal("multixact %u has two or more updating members",
    6930             :                                      multi),
    6931             :                      errdetail_internal("First updater XID=%u second updater XID=%u.",
    6932             :                                         update_xid, xid)));
    6933             : 
    6934             :         /*
    6935             :          * As with all tuple visibility routines, it's critical to test
    6936             :          * TransactionIdIsInProgress before TransactionIdDidCommit, because of
    6937             :          * race conditions explained in detail in heapam_visibility.c.
    6938             :          */
    6939           0 :         if (TransactionIdIsCurrentTransactionId(xid) ||
    6940           0 :             TransactionIdIsInProgress(xid))
    6941           0 :             update_xid = xid;
    6942           0 :         else if (TransactionIdDidCommit(xid))
    6943             :         {
    6944             :             /*
    6945             :              * The transaction committed, so we can tell caller to set
    6946             :              * HEAP_XMAX_COMMITTED.  (We can only do this because we know the
    6947             :              * transaction is not running.)
    6948             :              */
    6949           0 :             update_committed = true;
    6950           0 :             update_xid = xid;
    6951             :         }
    6952             :         else
    6953             :         {
    6954             :             /*
    6955             :              * Not in progress, not committed -- must be aborted or crashed;
    6956             :              * we can ignore it.
    6957             :              */
    6958           0 :             continue;
    6959             :         }
    6960             : 
    6961             :         /*
    6962             :          * We determined that updater must be kept -- add it to pending new
    6963             :          * members list
    6964             :          */
    6965           0 :         if (TransactionIdPrecedes(xid, cutoffs->OldestXmin))
    6966           0 :             ereport(ERROR,
    6967             :                     (errcode(ERRCODE_DATA_CORRUPTED),
    6968             :                      errmsg_internal("multixact %u contains committed update XID %u from before removable cutoff %u",
    6969             :                                      multi, xid, cutoffs->OldestXmin)));
    6970           0 :         newmembers[nnewmembers++] = members[i];
    6971             :     }
    6972             : 
    6973           2 :     pfree(members);
    6974             : 
    6975             :     /*
    6976             :      * Determine what to do with caller's multi based on information gathered
    6977             :      * during our second pass
    6978             :      */
    6979           2 :     if (nnewmembers == 0)
    6980             :     {
    6981             :         /* Nothing worth keeping */
    6982           0 :         *flags |= FRM_INVALIDATE_XMAX;
    6983           0 :         newxmax = InvalidTransactionId;
    6984             :     }
    6985           2 :     else if (TransactionIdIsValid(update_xid) && !has_lockers)
    6986             :     {
    6987             :         /*
    6988             :          * If there's a single member and it's an update, pass it back alone
    6989             :          * without creating a new Multi.  (XXX we could do this when there's a
    6990             :          * single remaining locker, too, but that would complicate the API too
    6991             :          * much; moreover, the case with the single updater is more
    6992             :          * interesting, because those are longer-lived.)
    6993             :          */
    6994             :         Assert(nnewmembers == 1);
    6995           0 :         *flags |= FRM_RETURN_IS_XID;
    6996           0 :         if (update_committed)
    6997           0 :             *flags |= FRM_MARK_COMMITTED;
    6998           0 :         newxmax = update_xid;
    6999             :     }
    7000             :     else
    7001             :     {
    7002             :         /*
    7003             :          * Create a new multixact with the surviving members of the previous
    7004             :          * one, to set as new Xmax in the tuple
    7005             :          */
    7006           2 :         newxmax = MultiXactIdCreateFromMembers(nnewmembers, newmembers);
    7007           2 :         *flags |= FRM_RETURN_IS_MULTI;
    7008             :     }
    7009             : 
    7010           2 :     pfree(newmembers);
    7011             : 
    7012           2 :     pagefrz->freeze_required = true;
    7013           2 :     return newxmax;
    7014             : }
    7015             : 
    7016             : /*
    7017             :  * heap_prepare_freeze_tuple
    7018             :  *
    7019             :  * Check to see whether any of the XID fields of a tuple (xmin, xmax, xvac)
    7020             :  * are older than the OldestXmin and/or OldestMxact freeze cutoffs.  If so,
    7021             :  * setup enough state (in the *frz output argument) to enable caller to
    7022             :  * process this tuple as part of freezing its page, and return true.  Return
    7023             :  * false if nothing can be changed about the tuple right now.
    7024             :  *
    7025             :  * Also sets *totally_frozen to true if the tuple will be totally frozen once
    7026             :  * caller executes returned freeze plan (or if the tuple was already totally
    7027             :  * frozen by an earlier VACUUM).  This indicates that there are no remaining
    7028             :  * XIDs or MultiXactIds that will need to be processed by a future VACUUM.
    7029             :  *
    7030             :  * VACUUM caller must assemble HeapTupleFreeze freeze plan entries for every
    7031             :  * tuple that we returned true for, and then execute freezing.  Caller must
    7032             :  * initialize pagefrz fields for page as a whole before first call here for
    7033             :  * each heap page.
    7034             :  *
    7035             :  * VACUUM caller decides on whether or not to freeze the page as a whole.
    7036             :  * We'll often prepare freeze plans for a page that caller just discards.
    7037             :  * However, VACUUM doesn't always get to make a choice; it must freeze when
    7038             :  * pagefrz.freeze_required is set, to ensure that any XIDs < FreezeLimit (and
    7039             :  * MXIDs < MultiXactCutoff) can never be left behind.  We help to make sure
    7040             :  * that VACUUM always follows that rule.
    7041             :  *
    7042             :  * We sometimes force freezing of xmax MultiXactId values long before it is
    7043             :  * strictly necessary to do so just to ensure the FreezeLimit postcondition.
    7044             :  * It's worth processing MultiXactIds proactively when it is cheap to do so,
    7045             :  * and it's convenient to make that happen by piggy-backing it on the "force
    7046             :  * freezing" mechanism.  Conversely, we sometimes delay freezing MultiXactIds
    7047             :  * because it is expensive right now (though only when it's still possible to
    7048             :  * do so without violating the FreezeLimit/MultiXactCutoff postcondition).
    7049             :  *
    7050             :  * It is assumed that the caller has checked the tuple with
    7051             :  * HeapTupleSatisfiesVacuum() and determined that it is not HEAPTUPLE_DEAD
    7052             :  * (else we should be removing the tuple, not freezing it).
    7053             :  *
    7054             :  * NB: This function has side effects: it might allocate a new MultiXactId.
    7055             :  * It will be set as tuple's new xmax when our *frz output is processed within
    7056             :  * heap_execute_freeze_tuple later on.  If the tuple is in a shared buffer
    7057             :  * then caller had better have an exclusive lock on it already.
    7058             :  */
    7059             : bool
    7060    32113240 : heap_prepare_freeze_tuple(HeapTupleHeader tuple,
    7061             :                           const struct VacuumCutoffs *cutoffs,
    7062             :                           HeapPageFreeze *pagefrz,
    7063             :                           HeapTupleFreeze *frz, bool *totally_frozen)
    7064             : {
    7065    32113240 :     bool        xmin_already_frozen = false,
    7066    32113240 :                 xmax_already_frozen = false;
    7067    32113240 :     bool        freeze_xmin = false,
    7068    32113240 :                 replace_xvac = false,
    7069    32113240 :                 replace_xmax = false,
    7070    32113240 :                 freeze_xmax = false;
    7071             :     TransactionId xid;
    7072             : 
    7073    32113240 :     frz->xmax = HeapTupleHeaderGetRawXmax(tuple);
    7074    32113240 :     frz->t_infomask2 = tuple->t_infomask2;
    7075    32113240 :     frz->t_infomask = tuple->t_infomask;
    7076    32113240 :     frz->frzflags = 0;
    7077    32113240 :     frz->checkflags = 0;
    7078             : 
    7079             :     /*
    7080             :      * Process xmin, while keeping track of whether it's already frozen, or
    7081             :      * will become frozen iff our freeze plan is executed by caller (could be
    7082             :      * neither).
    7083             :      */
    7084    32113240 :     xid = HeapTupleHeaderGetXmin(tuple);
    7085    32113240 :     if (!TransactionIdIsNormal(xid))
    7086    26455174 :         xmin_already_frozen = true;
    7087             :     else
    7088             :     {
    7089     5658066 :         if (TransactionIdPrecedes(xid, cutoffs->relfrozenxid))
    7090           0 :             ereport(ERROR,
    7091             :                     (errcode(ERRCODE_DATA_CORRUPTED),
    7092             :                      errmsg_internal("found xmin %u from before relfrozenxid %u",
    7093             :                                      xid, cutoffs->relfrozenxid)));
    7094             : 
    7095             :         /* Will set freeze_xmin flags in freeze plan below */
    7096     5658066 :         freeze_xmin = TransactionIdPrecedes(xid, cutoffs->OldestXmin);
    7097             : 
    7098             :         /* Verify that xmin committed if and when freeze plan is executed */
    7099     5658066 :         if (freeze_xmin)
    7100     4583760 :             frz->checkflags |= HEAP_FREEZE_CHECK_XMIN_COMMITTED;
    7101             :     }
    7102             : 
    7103             :     /*
    7104             :      * Old-style VACUUM FULL is gone, but we have to process xvac for as long
    7105             :      * as we support having MOVED_OFF/MOVED_IN tuples in the database
    7106             :      */
    7107    32113240 :     xid = HeapTupleHeaderGetXvac(tuple);
    7108    32113240 :     if (TransactionIdIsNormal(xid))
    7109             :     {
    7110             :         Assert(TransactionIdPrecedesOrEquals(cutoffs->relfrozenxid, xid));
    7111             :         Assert(TransactionIdPrecedes(xid, cutoffs->OldestXmin));
    7112             : 
    7113             :         /*
    7114             :          * For Xvac, we always freeze proactively.  This allows totally_frozen
    7115             :          * tracking to ignore xvac.
    7116             :          */
    7117           0 :         replace_xvac = pagefrz->freeze_required = true;
    7118             : 
    7119             :         /* Will set replace_xvac flags in freeze plan below */
    7120             :     }
    7121             : 
    7122             :     /* Now process xmax */
    7123    32113240 :     xid = frz->xmax;
    7124    32113240 :     if (tuple->t_infomask & HEAP_XMAX_IS_MULTI)
    7125             :     {
    7126             :         /* Raw xmax is a MultiXactId */
    7127             :         TransactionId newxmax;
    7128             :         uint16      flags;
    7129             : 
    7130             :         /*
    7131             :          * We will either remove xmax completely (in the "freeze_xmax" path),
    7132             :          * process xmax by replacing it (in the "replace_xmax" path), or
    7133             :          * perform no-op xmax processing.  The only constraint is that the
    7134             :          * FreezeLimit/MultiXactCutoff postcondition must never be violated.
    7135             :          */
    7136          12 :         newxmax = FreezeMultiXactId(xid, tuple->t_infomask, cutoffs,
    7137             :                                     &flags, pagefrz);
    7138             : 
    7139          12 :         if (flags & FRM_NOOP)
    7140             :         {
    7141             :             /*
    7142             :              * xmax is a MultiXactId, and nothing about it changes for now.
    7143             :              * This is the only case where 'freeze_required' won't have been
    7144             :              * set for us by FreezeMultiXactId, as well as the only case where
    7145             :              * neither freeze_xmax nor replace_xmax are set (given a multi).
    7146             :              *
    7147             :              * This is a no-op, but the call to FreezeMultiXactId might have
    7148             :              * ratcheted back NewRelfrozenXid and/or NewRelminMxid trackers
    7149             :              * for us (the "freeze page" variants, specifically).  That'll
    7150             :              * make it safe for our caller to freeze the page later on, while
    7151             :              * leaving this particular xmax undisturbed.
    7152             :              *
    7153             :              * FreezeMultiXactId is _not_ responsible for the "no freeze"
    7154             :              * NewRelfrozenXid/NewRelminMxid trackers, though -- that's our
    7155             :              * job.  A call to heap_tuple_should_freeze for this same tuple
    7156             :              * will take place below if 'freeze_required' isn't set already.
    7157             :              * (This repeats work from FreezeMultiXactId, but allows "no
    7158             :              * freeze" tracker maintenance to happen in only one place.)
    7159             :              */
    7160             :             Assert(!MultiXactIdPrecedes(newxmax, cutoffs->MultiXactCutoff));
    7161             :             Assert(MultiXactIdIsValid(newxmax) && xid == newxmax);
    7162             :         }
    7163          10 :         else if (flags & FRM_RETURN_IS_XID)
    7164             :         {
    7165             :             /*
    7166             :              * xmax will become an updater Xid (original MultiXact's updater
    7167             :              * member Xid will be carried forward as a simple Xid in Xmax).
    7168             :              */
    7169             :             Assert(!TransactionIdPrecedes(newxmax, cutoffs->OldestXmin));
    7170             : 
    7171             :             /*
    7172             :              * NB -- some of these transformations are only valid because we
    7173             :              * know the return Xid is a tuple updater (i.e. not merely a
    7174             :              * locker.) Also note that the only reason we don't explicitly
    7175             :              * worry about HEAP_KEYS_UPDATED is because it lives in
    7176             :              * t_infomask2 rather than t_infomask.
    7177             :              */
    7178           0 :             frz->t_infomask &= ~HEAP_XMAX_BITS;
    7179           0 :             frz->xmax = newxmax;
    7180           0 :             if (flags & FRM_MARK_COMMITTED)
    7181           0 :                 frz->t_infomask |= HEAP_XMAX_COMMITTED;
    7182           0 :             replace_xmax = true;
    7183             :         }
    7184          10 :         else if (flags & FRM_RETURN_IS_MULTI)
    7185             :         {
    7186             :             uint16      newbits;
    7187             :             uint16      newbits2;
    7188             : 
    7189             :             /*
    7190             :              * xmax is an old MultiXactId that we have to replace with a new
    7191             :              * MultiXactId, to carry forward two or more original member XIDs.
    7192             :              */
    7193             :             Assert(!MultiXactIdPrecedes(newxmax, cutoffs->OldestMxact));
    7194             : 
    7195             :             /*
    7196             :              * We can't use GetMultiXactIdHintBits directly on the new multi
    7197             :              * here; that routine initializes the masks to all zeroes, which
    7198             :              * would lose other bits we need.  Doing it this way ensures all
    7199             :              * unrelated bits remain untouched.
    7200             :              */
    7201           2 :             frz->t_infomask &= ~HEAP_XMAX_BITS;
    7202           2 :             frz->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    7203           2 :             GetMultiXactIdHintBits(newxmax, &newbits, &newbits2);
    7204           2 :             frz->t_infomask |= newbits;
    7205           2 :             frz->t_infomask2 |= newbits2;
    7206           2 :             frz->xmax = newxmax;
    7207           2 :             replace_xmax = true;
    7208             :         }
    7209             :         else
    7210             :         {
    7211             :             /*
    7212             :              * Freeze plan for tuple "freezes xmax" in the strictest sense:
    7213             :              * it'll leave nothing in xmax (neither an Xid nor a MultiXactId).
    7214             :              */
    7215             :             Assert(flags & FRM_INVALIDATE_XMAX);
    7216             :             Assert(!TransactionIdIsValid(newxmax));
    7217             : 
    7218             :             /* Will set freeze_xmax flags in freeze plan below */
    7219           8 :             freeze_xmax = true;
    7220             :         }
    7221             : 
    7222             :         /* MultiXactId processing forces freezing (barring FRM_NOOP case) */
    7223             :         Assert(pagefrz->freeze_required || (!freeze_xmax && !replace_xmax));
    7224             :     }
    7225    32113228 :     else if (TransactionIdIsNormal(xid))
    7226             :     {
    7227             :         /* Raw xmax is normal XID */
    7228    11842528 :         if (TransactionIdPrecedes(xid, cutoffs->relfrozenxid))
    7229           0 :             ereport(ERROR,
    7230             :                     (errcode(ERRCODE_DATA_CORRUPTED),
    7231             :                      errmsg_internal("found xmax %u from before relfrozenxid %u",
    7232             :                                      xid, cutoffs->relfrozenxid)));
    7233             : 
    7234             :         /* Will set freeze_xmax flags in freeze plan below */
    7235    11842528 :         freeze_xmax = TransactionIdPrecedes(xid, cutoffs->OldestXmin);
    7236             : 
    7237             :         /*
    7238             :          * Verify that xmax aborted if and when freeze plan is executed,
    7239             :          * provided it's from an update. (A lock-only xmax can be removed
    7240             :          * independent of this, since the lock is released at xact end.)
    7241             :          */
    7242    11842528 :         if (freeze_xmax && !HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_infomask))
    7243        1188 :             frz->checkflags |= HEAP_FREEZE_CHECK_XMAX_ABORTED;
    7244             :     }
    7245    20270700 :     else if (!TransactionIdIsValid(xid))
    7246             :     {
    7247             :         /* Raw xmax is InvalidTransactionId XID */
    7248             :         Assert((tuple->t_infomask & HEAP_XMAX_IS_MULTI) == 0);
    7249    20270700 :         xmax_already_frozen = true;
    7250             :     }
    7251             :     else
    7252           0 :         ereport(ERROR,
    7253             :                 (errcode(ERRCODE_DATA_CORRUPTED),
    7254             :                  errmsg_internal("found raw xmax %u (infomask 0x%04x) not invalid and not multi",
    7255             :                                  xid, tuple->t_infomask)));
    7256             : 
    7257    32113240 :     if (freeze_xmin)
    7258             :     {
    7259             :         Assert(!xmin_already_frozen);
    7260             : 
    7261     4583760 :         frz->t_infomask |= HEAP_XMIN_FROZEN;
    7262             :     }
    7263    32113240 :     if (replace_xvac)
    7264             :     {
    7265             :         /*
    7266             :          * If a MOVED_OFF tuple is not dead, the xvac transaction must have
    7267             :          * failed; whereas a non-dead MOVED_IN tuple must mean the xvac
    7268             :          * transaction succeeded.
    7269             :          */
    7270             :         Assert(pagefrz->freeze_required);
    7271           0 :         if (tuple->t_infomask & HEAP_MOVED_OFF)
    7272           0 :             frz->frzflags |= XLH_INVALID_XVAC;
    7273             :         else
    7274           0 :             frz->frzflags |= XLH_FREEZE_XVAC;
    7275             :     }
    7276             :     if (replace_xmax)
    7277             :     {
    7278             :         Assert(!xmax_already_frozen && !freeze_xmax);
    7279             :         Assert(pagefrz->freeze_required);
    7280             : 
    7281             :         /* Already set replace_xmax flags in freeze plan earlier */
    7282             :     }
    7283    32113240 :     if (freeze_xmax)
    7284             :     {
    7285             :         Assert(!xmax_already_frozen && !replace_xmax);
    7286             : 
    7287        3126 :         frz->xmax = InvalidTransactionId;
    7288             : 
    7289             :         /*
    7290             :          * The tuple might be marked either XMAX_INVALID or XMAX_COMMITTED +
    7291             :          * LOCKED.  Normalize to INVALID just to be sure no one gets confused.
    7292             :          * Also get rid of the HEAP_KEYS_UPDATED bit.
    7293             :          */
    7294        3126 :         frz->t_infomask &= ~HEAP_XMAX_BITS;
    7295        3126 :         frz->t_infomask |= HEAP_XMAX_INVALID;
    7296        3126 :         frz->t_infomask2 &= ~HEAP_HOT_UPDATED;
    7297        3126 :         frz->t_infomask2 &= ~HEAP_KEYS_UPDATED;
    7298             :     }
    7299             : 
    7300             :     /*
    7301             :      * Determine if this tuple is already totally frozen, or will become
    7302             :      * totally frozen (provided caller executes freeze plans for the page)
    7303             :      */
    7304    63149048 :     *totally_frozen = ((freeze_xmin || xmin_already_frozen) &&
    7305    31035808 :                        (freeze_xmax || xmax_already_frozen));
    7306             : 
    7307    32113240 :     if (!pagefrz->freeze_required && !(xmin_already_frozen &&
    7308             :                                        xmax_already_frozen))
    7309             :     {
    7310             :         /*
    7311             :          * So far no previous tuple from the page made freezing mandatory.
    7312             :          * Does this tuple force caller to freeze the entire page?
    7313             :          */
    7314    14581744 :         pagefrz->freeze_required =
    7315    14581744 :             heap_tuple_should_freeze(tuple, cutoffs,
    7316             :                                      &pagefrz->NoFreezePageRelfrozenXid,
    7317             :                                      &pagefrz->NoFreezePageRelminMxid);
    7318             :     }
    7319             : 
    7320             :     /* Tell caller if this tuple has a usable freeze plan set in *frz */
    7321    32113240 :     return freeze_xmin || replace_xvac || replace_xmax || freeze_xmax;
    7322             : }
    7323             : 
    7324             : /*
    7325             :  * Perform xmin/xmax XID status sanity checks before actually executing freeze
    7326             :  * plans.
    7327             :  *
    7328             :  * heap_prepare_freeze_tuple doesn't perform these checks directly because
    7329             :  * pg_xact lookups are relatively expensive.  They shouldn't be repeated by
    7330             :  * successive VACUUMs that each decide against freezing the same page.
    7331             :  */
    7332             : void
    7333       40176 : heap_pre_freeze_checks(Buffer buffer,
    7334             :                        HeapTupleFreeze *tuples, int ntuples)
    7335             : {
    7336       40176 :     Page        page = BufferGetPage(buffer);
    7337             : 
    7338     1774622 :     for (int i = 0; i < ntuples; i++)
    7339             :     {
    7340     1734446 :         HeapTupleFreeze *frz = tuples + i;
    7341     1734446 :         ItemId      itemid = PageGetItemId(page, frz->offset);
    7342             :         HeapTupleHeader htup;
    7343             : 
    7344     1734446 :         htup = (HeapTupleHeader) PageGetItem(page, itemid);
    7345             : 
    7346             :         /* Deliberately avoid relying on tuple hint bits here */
    7347     1734446 :         if (frz->checkflags & HEAP_FREEZE_CHECK_XMIN_COMMITTED)
    7348             :         {
    7349     1734444 :             TransactionId xmin = HeapTupleHeaderGetRawXmin(htup);
    7350             : 
    7351             :             Assert(!HeapTupleHeaderXminFrozen(htup));
    7352     1734444 :             if (unlikely(!TransactionIdDidCommit(xmin)))
    7353           0 :                 ereport(ERROR,
    7354             :                         (errcode(ERRCODE_DATA_CORRUPTED),
    7355             :                          errmsg_internal("uncommitted xmin %u needs to be frozen",
    7356             :                                          xmin)));
    7357             :         }
    7358             : 
    7359             :         /*
    7360             :          * TransactionIdDidAbort won't work reliably in the presence of XIDs
    7361             :          * left behind by transactions that were in progress during a crash,
    7362             :          * so we can only check that xmax didn't commit
    7363             :          */
    7364     1734446 :         if (frz->checkflags & HEAP_FREEZE_CHECK_XMAX_ABORTED)
    7365             :         {
    7366          94 :             TransactionId xmax = HeapTupleHeaderGetRawXmax(htup);
    7367             : 
    7368             :             Assert(TransactionIdIsNormal(xmax));
    7369          94 :             if (unlikely(TransactionIdDidCommit(xmax)))
    7370           0 :                 ereport(ERROR,
    7371             :                         (errcode(ERRCODE_DATA_CORRUPTED),
    7372             :                          errmsg_internal("cannot freeze committed xmax %u",
    7373             :                                          xmax)));
    7374             :         }
    7375             :     }
    7376       40176 : }
    7377             : 
    7378             : /*
    7379             :  * Helper which executes freezing of one or more heap tuples on a page on
    7380             :  * behalf of caller.  Caller passes an array of tuple plans from
    7381             :  * heap_prepare_freeze_tuple.  Caller must set 'offset' in each plan for us.
    7382             :  * Must be called in a critical section that also marks the buffer dirty and,
    7383             :  * if needed, emits WAL.
    7384             :  */
    7385             : void
    7386       40176 : heap_freeze_prepared_tuples(Buffer buffer, HeapTupleFreeze *tuples, int ntuples)
    7387             : {
    7388       40176 :     Page        page = BufferGetPage(buffer);
    7389             : 
    7390     1774622 :     for (int i = 0; i < ntuples; i++)
    7391             :     {
    7392     1734446 :         HeapTupleFreeze *frz = tuples + i;
    7393     1734446 :         ItemId      itemid = PageGetItemId(page, frz->offset);
    7394             :         HeapTupleHeader htup;
    7395             : 
    7396     1734446 :         htup = (HeapTupleHeader) PageGetItem(page, itemid);
    7397     1734446 :         heap_execute_freeze_tuple(htup, frz);
    7398             :     }
    7399       40176 : }
    7400             : 
    7401             : /*
    7402             :  * heap_freeze_tuple
    7403             :  *      Freeze tuple in place, without WAL logging.
    7404             :  *
    7405             :  * Useful for callers like CLUSTER that perform their own WAL logging.
    7406             :  */
    7407             : bool
    7408      748098 : heap_freeze_tuple(HeapTupleHeader tuple,
    7409             :                   TransactionId relfrozenxid, TransactionId relminmxid,
    7410             :                   TransactionId FreezeLimit, TransactionId MultiXactCutoff)
    7411             : {
    7412             :     HeapTupleFreeze frz;
    7413             :     bool        do_freeze;
    7414             :     bool        totally_frozen;
    7415             :     struct VacuumCutoffs cutoffs;
    7416             :     HeapPageFreeze pagefrz;
    7417             : 
    7418      748098 :     cutoffs.relfrozenxid = relfrozenxid;
    7419      748098 :     cutoffs.relminmxid = relminmxid;
    7420      748098 :     cutoffs.OldestXmin = FreezeLimit;
    7421      748098 :     cutoffs.OldestMxact = MultiXactCutoff;
    7422      748098 :     cutoffs.FreezeLimit = FreezeLimit;
    7423      748098 :     cutoffs.MultiXactCutoff = MultiXactCutoff;
    7424             : 
    7425      748098 :     pagefrz.freeze_required = true;
    7426      748098 :     pagefrz.FreezePageRelfrozenXid = FreezeLimit;
    7427      748098 :     pagefrz.FreezePageRelminMxid = MultiXactCutoff;
    7428      748098 :     pagefrz.NoFreezePageRelfrozenXid = FreezeLimit;
    7429      748098 :     pagefrz.NoFreezePageRelminMxid = MultiXactCutoff;
    7430             : 
    7431      748098 :     do_freeze = heap_prepare_freeze_tuple(tuple, &cutoffs,
    7432             :                                           &pagefrz, &frz, &totally_frozen);
    7433             : 
    7434             :     /*
    7435             :      * Note that because this is not a WAL-logged operation, we don't need to
    7436             :      * fill in the offset in the freeze record.
    7437             :      */
    7438             : 
    7439      748098 :     if (do_freeze)
    7440      531988 :         heap_execute_freeze_tuple(tuple, &frz);
    7441      748098 :     return do_freeze;
    7442             : }
    7443             : 
    7444             : /*
    7445             :  * For a given MultiXactId, return the hint bits that should be set in the
    7446             :  * tuple's infomask.
    7447             :  *
    7448             :  * Normally this should be called for a multixact that was just created, and
    7449             :  * so is on our local cache, so the GetMembers call is fast.
    7450             :  */
    7451             : static void
    7452        2378 : GetMultiXactIdHintBits(MultiXactId multi, uint16 *new_infomask,
    7453             :                        uint16 *new_infomask2)
    7454             : {
    7455             :     int         nmembers;
    7456             :     MultiXactMember *members;
    7457             :     int         i;
    7458        2378 :     uint16      bits = HEAP_XMAX_IS_MULTI;
    7459        2378 :     uint16      bits2 = 0;
    7460        2378 :     bool        has_update = false;
    7461        2378 :     LockTupleMode strongest = LockTupleKeyShare;
    7462             : 
    7463             :     /*
    7464             :      * We only use this in multis we just created, so they cannot be values
    7465             :      * pre-pg_upgrade.
    7466             :      */
    7467        2378 :     nmembers = GetMultiXactIdMembers(multi, &members, false, false);
    7468             : 
    7469        7270 :     for (i = 0; i < nmembers; i++)
    7470             :     {
    7471             :         LockTupleMode mode;
    7472             : 
    7473             :         /*
    7474             :          * Remember the strongest lock mode held by any member of the
    7475             :          * multixact.
    7476             :          */
    7477        4892 :         mode = TUPLOCK_from_mxstatus(members[i].status);
    7478        4892 :         if (mode > strongest)
    7479        1318 :             strongest = mode;
    7480             : 
    7481             :         /* See what other bits we need */
    7482        4892 :         switch (members[i].status)
    7483             :         {
    7484        4510 :             case MultiXactStatusForKeyShare:
    7485             :             case MultiXactStatusForShare:
    7486             :             case MultiXactStatusForNoKeyUpdate:
    7487        4510 :                 break;
    7488             : 
    7489         104 :             case MultiXactStatusForUpdate:
    7490         104 :                 bits2 |= HEAP_KEYS_UPDATED;
    7491         104 :                 break;
    7492             : 
    7493         258 :             case MultiXactStatusNoKeyUpdate:
    7494         258 :                 has_update = true;
    7495         258 :                 break;
    7496             : 
    7497          20 :             case MultiXactStatusUpdate:
    7498          20 :                 bits2 |= HEAP_KEYS_UPDATED;
    7499          20 :                 has_update = true;
    7500          20 :                 break;
    7501             :         }
    7502        4892 :     }
    7503             : 
    7504        2378 :     if (strongest == LockTupleExclusive ||
    7505             :         strongest == LockTupleNoKeyExclusive)
    7506         438 :         bits |= HEAP_XMAX_EXCL_LOCK;
    7507        1940 :     else if (strongest == LockTupleShare)
    7508         874 :         bits |= HEAP_XMAX_SHR_LOCK;
    7509        1066 :     else if (strongest == LockTupleKeyShare)
    7510        1066 :         bits |= HEAP_XMAX_KEYSHR_LOCK;
    7511             : 
    7512        2378 :     if (!has_update)
    7513        2100 :         bits |= HEAP_XMAX_LOCK_ONLY;
    7514             : 
    7515        2378 :     if (nmembers > 0)
    7516        2378 :         pfree(members);
    7517             : 
    7518        2378 :     *new_infomask = bits;
    7519        2378 :     *new_infomask2 = bits2;
    7520        2378 : }
    7521             : 
    7522             : /*
    7523             :  * MultiXactIdGetUpdateXid
    7524             :  *
    7525             :  * Given a multixact Xmax and corresponding infomask, which does not have the
    7526             :  * HEAP_XMAX_LOCK_ONLY bit set, obtain and return the Xid of the updating
    7527             :  * transaction.
    7528             :  *
    7529             :  * Caller is expected to check the status of the updating transaction, if
    7530             :  * necessary.
    7531             :  */
    7532             : static TransactionId
    7533        1094 : MultiXactIdGetUpdateXid(TransactionId xmax, uint16 t_infomask)
    7534             : {
    7535        1094 :     TransactionId update_xact = InvalidTransactionId;
    7536             :     MultiXactMember *members;
    7537             :     int         nmembers;
    7538             : 
    7539             :     Assert(!(t_infomask & HEAP_XMAX_LOCK_ONLY));
    7540             :     Assert(t_infomask & HEAP_XMAX_IS_MULTI);
    7541             : 
    7542             :     /*
    7543             :      * Since we know the LOCK_ONLY bit is not set, this cannot be a multi from
    7544             :      * pre-pg_upgrade.
    7545             :      */
    7546        1094 :     nmembers = GetMultiXactIdMembers(xmax, &members, false, false);
    7547             : 
    7548        1094 :     if (nmembers > 0)
    7549             :     {
    7550             :         int         i;
    7551             : 
    7552        2890 :         for (i = 0; i < nmembers; i++)
    7553             :         {
    7554             :             /* Ignore lockers */
    7555        2890 :             if (!ISUPDATE_from_mxstatus(members[i].status))
    7556        1796 :                 continue;
    7557             : 
    7558             :             /* there can be at most one updater */
    7559             :             Assert(update_xact == InvalidTransactionId);
    7560        1094 :             update_xact = members[i].xid;
    7561             : #ifndef USE_ASSERT_CHECKING
    7562             : 
    7563             :             /*
    7564             :              * in an assert-enabled build, walk the whole array to ensure
    7565             :              * there's no other updater.
    7566             :              */
    7567        1094 :             break;
    7568             : #endif
    7569             :         }
    7570             : 
    7571        1094 :         pfree(members);
    7572             :     }
    7573             : 
    7574        1094 :     return update_xact;
    7575             : }
    7576             : 
    7577             : /*
    7578             :  * HeapTupleGetUpdateXid
    7579             :  *      As above, but use a HeapTupleHeader
    7580             :  *
    7581             :  * See also HeapTupleHeaderGetUpdateXid, which can be used without previously
    7582             :  * checking the hint bits.
    7583             :  */
    7584             : TransactionId
    7585        1078 : HeapTupleGetUpdateXid(const HeapTupleHeaderData *tup)
    7586             : {
    7587        1078 :     return MultiXactIdGetUpdateXid(HeapTupleHeaderGetRawXmax(tup),
    7588        1078 :                                    tup->t_infomask);
    7589             : }
    7590             : 
    7591             : /*
    7592             :  * Does the given multixact conflict with the current transaction grabbing a
    7593             :  * tuple lock of the given strength?
    7594             :  *
    7595             :  * The passed infomask pairs up with the given multixact in the tuple header.
    7596             :  *
    7597             :  * If current_is_member is not NULL, it is set to 'true' if the current
    7598             :  * transaction is a member of the given multixact.
    7599             :  */
    7600             : static bool
    7601         198 : DoesMultiXactIdConflict(MultiXactId multi, uint16 infomask,
    7602             :                         LockTupleMode lockmode, bool *current_is_member)
    7603             : {
    7604             :     int         nmembers;
    7605             :     MultiXactMember *members;
    7606         198 :     bool        result = false;
    7607         198 :     LOCKMODE    wanted = tupleLockExtraInfo[lockmode].hwlock;
    7608             : 
    7609         198 :     if (HEAP_LOCKED_UPGRADED(infomask))
    7610           0 :         return false;
    7611             : 
    7612         198 :     nmembers = GetMultiXactIdMembers(multi, &members, false,
    7613         198 :                                      HEAP_XMAX_IS_LOCKED_ONLY(infomask));
    7614         198 :     if (nmembers >= 0)
    7615             :     {
    7616             :         int         i;
    7617             : 
    7618         618 :         for (i = 0; i < nmembers; i++)
    7619             :         {
    7620             :             TransactionId memxid;
    7621             :             LOCKMODE    memlockmode;
    7622             : 
    7623         434 :             if (result && (current_is_member == NULL || *current_is_member))
    7624             :                 break;
    7625             : 
    7626         420 :             memlockmode = LOCKMODE_from_mxstatus(members[i].status);
    7627             : 
    7628             :             /* ignore members from current xact (but track their presence) */
    7629         420 :             memxid = members[i].xid;
    7630         420 :             if (TransactionIdIsCurrentTransactionId(memxid))
    7631             :             {
    7632         184 :                 if (current_is_member != NULL)
    7633         156 :                     *current_is_member = true;
    7634         184 :                 continue;
    7635             :             }
    7636         236 :             else if (result)
    7637          16 :                 continue;
    7638             : 
    7639             :             /* ignore members that don't conflict with the lock we want */
    7640         220 :             if (!DoLockModesConflict(memlockmode, wanted))
    7641         142 :                 continue;
    7642             : 
    7643          78 :             if (ISUPDATE_from_mxstatus(members[i].status))
    7644             :             {
    7645             :                 /* ignore aborted updaters */
    7646          34 :                 if (TransactionIdDidAbort(memxid))
    7647           2 :                     continue;
    7648             :             }
    7649             :             else
    7650             :             {
    7651             :                 /* ignore lockers-only that are no longer in progress */
    7652          44 :                 if (!TransactionIdIsInProgress(memxid))
    7653          14 :                     continue;
    7654             :             }
    7655             : 
    7656             :             /*
    7657             :              * Whatever remains are either live lockers that conflict with our
    7658             :              * wanted lock, and updaters that are not aborted.  Those conflict
    7659             :              * with what we want.  Set up to return true, but keep going to
    7660             :              * look for the current transaction among the multixact members,
    7661             :              * if needed.
    7662             :              */
    7663          62 :             result = true;
    7664             :         }
    7665         198 :         pfree(members);
    7666             :     }
    7667             : 
    7668         198 :     return result;
    7669             : }
    7670             : 
    7671             : /*
    7672             :  * Do_MultiXactIdWait
    7673             :  *      Actual implementation for the two functions below.
    7674             :  *
    7675             :  * 'multi', 'status' and 'infomask' indicate what to sleep on (the status is
    7676             :  * needed to ensure we only sleep on conflicting members, and the infomask is
    7677             :  * used to optimize multixact access in case it's a lock-only multi); 'nowait'
    7678             :  * indicates whether to use conditional lock acquisition, to allow callers to
    7679             :  * fail if lock is unavailable.  'rel', 'ctid' and 'oper' are used to set up
    7680             :  * context information for error messages.  'remaining', if not NULL, receives
    7681             :  * the number of members that are still running, including any (non-aborted)
    7682             :  * subtransactions of our own transaction.  'logLockFailure' indicates whether
    7683             :  * to log details when a lock acquisition fails with 'nowait' enabled.
    7684             :  *
    7685             :  * We do this by sleeping on each member using XactLockTableWait.  Any
    7686             :  * members that belong to the current backend are *not* waited for, however;
    7687             :  * this would not merely be useless but would lead to Assert failure inside
    7688             :  * XactLockTableWait.  By the time this returns, it is certain that all
    7689             :  * transactions *of other backends* that were members of the MultiXactId
    7690             :  * that conflict with the requested status are dead (and no new ones can have
    7691             :  * been added, since it is not legal to add members to an existing
    7692             :  * MultiXactId).
    7693             :  *
    7694             :  * But by the time we finish sleeping, someone else may have changed the Xmax
    7695             :  * of the containing tuple, so the caller needs to iterate on us somehow.
    7696             :  *
    7697             :  * Note that in case we return false, the number of remaining members is
    7698             :  * not to be trusted.
    7699             :  */
    7700             : static bool
    7701         116 : Do_MultiXactIdWait(MultiXactId multi, MultiXactStatus status,
    7702             :                    uint16 infomask, bool nowait,
    7703             :                    Relation rel, ItemPointer ctid, XLTW_Oper oper,
    7704             :                    int *remaining, bool logLockFailure)
    7705             : {
    7706         116 :     bool        result = true;
    7707             :     MultiXactMember *members;
    7708             :     int         nmembers;
    7709         116 :     int         remain = 0;
    7710             : 
    7711             :     /* for pre-pg_upgrade tuples, no need to sleep at all */
    7712         116 :     nmembers = HEAP_LOCKED_UPGRADED(infomask) ? -1 :
    7713         116 :         GetMultiXactIdMembers(multi, &members, false,
    7714         116 :                               HEAP_XMAX_IS_LOCKED_ONLY(infomask));
    7715             : 
    7716         116 :     if (nmembers >= 0)
    7717             :     {
    7718             :         int         i;
    7719             : 
    7720         374 :         for (i = 0; i < nmembers; i++)
    7721             :         {
    7722         266 :             TransactionId memxid = members[i].xid;
    7723         266 :             MultiXactStatus memstatus = members[i].status;
    7724             : 
    7725         266 :             if (TransactionIdIsCurrentTransactionId(memxid))
    7726             :             {
    7727          48 :                 remain++;
    7728          48 :                 continue;
    7729             :             }
    7730             : 
    7731         218 :             if (!DoLockModesConflict(LOCKMODE_from_mxstatus(memstatus),
    7732         218 :                                      LOCKMODE_from_mxstatus(status)))
    7733             :             {
    7734          44 :                 if (remaining && TransactionIdIsInProgress(memxid))
    7735          16 :                     remain++;
    7736          44 :                 continue;
    7737             :             }
    7738             : 
    7739             :             /*
    7740             :              * This member conflicts with our multi, so we have to sleep (or
    7741             :              * return failure, if asked to avoid waiting.)
    7742             :              *
    7743             :              * Note that we don't set up an error context callback ourselves,
    7744             :              * but instead we pass the info down to XactLockTableWait.  This
    7745             :              * might seem a bit wasteful because the context is set up and
    7746             :              * tore down for each member of the multixact, but in reality it
    7747             :              * should be barely noticeable, and it avoids duplicate code.
    7748             :              */
    7749         174 :             if (nowait)
    7750             :             {
    7751           8 :                 result = ConditionalXactLockTableWait(memxid, logLockFailure);
    7752           8 :                 if (!result)
    7753           8 :                     break;
    7754             :             }
    7755             :             else
    7756         166 :                 XactLockTableWait(memxid, rel, ctid, oper);
    7757             :         }
    7758             : 
    7759         116 :         pfree(members);
    7760             :     }
    7761             : 
    7762         116 :     if (remaining)
    7763          20 :         *remaining = remain;
    7764             : 
    7765         116 :     return result;
    7766             : }
    7767             : 
    7768             : /*
    7769             :  * MultiXactIdWait
    7770             :  *      Sleep on a MultiXactId.
    7771             :  *
    7772             :  * By the time we finish sleeping, someone else may have changed the Xmax
    7773             :  * of the containing tuple, so the caller needs to iterate on us somehow.
    7774             :  *
    7775             :  * We return (in *remaining, if not NULL) the number of members that are still
    7776             :  * running, including any (non-aborted) subtransactions of our own transaction.
    7777             :  */
    7778             : static void
    7779         108 : MultiXactIdWait(MultiXactId multi, MultiXactStatus status, uint16 infomask,
    7780             :                 Relation rel, ItemPointer ctid, XLTW_Oper oper,
    7781             :                 int *remaining)
    7782             : {
    7783         108 :     (void) Do_MultiXactIdWait(multi, status, infomask, false,
    7784             :                               rel, ctid, oper, remaining, false);
    7785         108 : }
    7786             : 
    7787             : /*
    7788             :  * ConditionalMultiXactIdWait
    7789             :  *      As above, but only lock if we can get the lock without blocking.
    7790             :  *
    7791             :  * By the time we finish sleeping, someone else may have changed the Xmax
    7792             :  * of the containing tuple, so the caller needs to iterate on us somehow.
    7793             :  *
    7794             :  * If the multixact is now all gone, return true.  Returns false if some
    7795             :  * transactions might still be running.
    7796             :  *
    7797             :  * We return (in *remaining, if not NULL) the number of members that are still
    7798             :  * running, including any (non-aborted) subtransactions of our own transaction.
    7799             :  */
    7800             : static bool
    7801           8 : ConditionalMultiXactIdWait(MultiXactId multi, MultiXactStatus status,
    7802             :                            uint16 infomask, Relation rel, int *remaining,
    7803             :                            bool logLockFailure)
    7804             : {
    7805           8 :     return Do_MultiXactIdWait(multi, status, infomask, true,
    7806             :                               rel, NULL, XLTW_None, remaining, logLockFailure);
    7807             : }
    7808             : 
    7809             : /*
    7810             :  * heap_tuple_needs_eventual_freeze
    7811             :  *
    7812             :  * Check to see whether any of the XID fields of a tuple (xmin, xmax, xvac)
    7813             :  * will eventually require freezing (if tuple isn't removed by pruning first).
    7814             :  */
    7815             : bool
    7816      218420 : heap_tuple_needs_eventual_freeze(HeapTupleHeader tuple)
    7817             : {
    7818             :     TransactionId xid;
    7819             : 
    7820             :     /*
    7821             :      * If xmin is a normal transaction ID, this tuple is definitely not
    7822             :      * frozen.
    7823             :      */
    7824      218420 :     xid = HeapTupleHeaderGetXmin(tuple);
    7825      218420 :     if (TransactionIdIsNormal(xid))
    7826        5350 :         return true;
    7827             : 
    7828             :     /*
    7829             :      * If xmax is a valid xact or multixact, this tuple is also not frozen.
    7830             :      */
    7831      213070 :     if (tuple->t_infomask & HEAP_XMAX_IS_MULTI)
    7832             :     {
    7833             :         MultiXactId multi;
    7834             : 
    7835           0 :         multi = HeapTupleHeaderGetRawXmax(tuple);
    7836           0 :         if (MultiXactIdIsValid(multi))
    7837           0 :             return true;
    7838             :     }
    7839             :     else
    7840             :     {
    7841      213070 :         xid = HeapTupleHeaderGetRawXmax(tuple);
    7842      213070 :         if (TransactionIdIsNormal(xid))
    7843          10 :             return true;
    7844             :     }
    7845             : 
    7846      213060 :     if (tuple->t_infomask & HEAP_MOVED)
    7847             :     {
    7848           0 :         xid = HeapTupleHeaderGetXvac(tuple);
    7849           0 :         if (TransactionIdIsNormal(xid))
    7850           0 :             return true;
    7851             :     }
    7852             : 
    7853      213060 :     return false;
    7854             : }
    7855             : 
    7856             : /*
    7857             :  * heap_tuple_should_freeze
    7858             :  *
    7859             :  * Return value indicates if heap_prepare_freeze_tuple sibling function would
    7860             :  * (or should) force freezing of the heap page that contains caller's tuple.
    7861             :  * Tuple header XIDs/MXIDs < FreezeLimit/MultiXactCutoff trigger freezing.
    7862             :  * This includes (xmin, xmax, xvac) fields, as well as MultiXact member XIDs.
    7863             :  *
    7864             :  * The *NoFreezePageRelfrozenXid and *NoFreezePageRelminMxid input/output
    7865             :  * arguments help VACUUM track the oldest extant XID/MXID remaining in rel.
    7866             :  * Our working assumption is that caller won't decide to freeze this tuple.
    7867             :  * It's up to caller to only ratchet back its own top-level trackers after the
    7868             :  * point that it fully commits to not freezing the tuple/page in question.
    7869             :  */
    7870             : bool
    7871    14582804 : heap_tuple_should_freeze(HeapTupleHeader tuple,
    7872             :                          const struct VacuumCutoffs *cutoffs,
    7873             :                          TransactionId *NoFreezePageRelfrozenXid,
    7874             :                          MultiXactId *NoFreezePageRelminMxid)
    7875             : {
    7876             :     TransactionId xid;
    7877             :     MultiXactId multi;
    7878    14582804 :     bool        freeze = false;
    7879             : 
    7880             :     /* First deal with xmin */
    7881    14582804 :     xid = HeapTupleHeaderGetXmin(tuple);
    7882    14582804 :     if (TransactionIdIsNormal(xid))
    7883             :     {
    7884             :         Assert(TransactionIdPrecedesOrEquals(cutoffs->relfrozenxid, xid));
    7885     3394362 :         if (TransactionIdPrecedes(xid, *NoFreezePageRelfrozenXid))
    7886       41604 :             *NoFreezePageRelfrozenXid = xid;
    7887     3394362 :         if (TransactionIdPrecedes(xid, cutoffs->FreezeLimit))
    7888       38588 :             freeze = true;
    7889             :     }
    7890             : 
    7891             :     /* Now deal with xmax */
    7892    14582804 :     xid = InvalidTransactionId;
    7893    14582804 :     multi = InvalidMultiXactId;
    7894    14582804 :     if (tuple->t_infomask & HEAP_XMAX_IS_MULTI)
    7895           4 :         multi = HeapTupleHeaderGetRawXmax(tuple);
    7896             :     else
    7897    14582800 :         xid = HeapTupleHeaderGetRawXmax(tuple);
    7898             : 
    7899    14582804 :     if (TransactionIdIsNormal(xid))
    7900             :     {
    7901             :         Assert(TransactionIdPrecedesOrEquals(cutoffs->relfrozenxid, xid));
    7902             :         /* xmax is a non-permanent XID */
    7903    11677492 :         if (TransactionIdPrecedes(xid, *NoFreezePageRelfrozenXid))
    7904           2 :             *NoFreezePageRelfrozenXid = xid;
    7905    11677492 :         if (TransactionIdPrecedes(xid, cutoffs->FreezeLimit))
    7906           8 :             freeze = true;
    7907             :     }
    7908     2905312 :     else if (!MultiXactIdIsValid(multi))
    7909             :     {
    7910             :         /* xmax is a permanent XID or invalid MultiXactId/XID */
    7911             :     }
    7912           4 :     else if (HEAP_LOCKED_UPGRADED(tuple->t_infomask))
    7913             :     {
    7914             :         /* xmax is a pg_upgrade'd MultiXact, which can't have updater XID */
    7915           0 :         if (MultiXactIdPrecedes(multi, *NoFreezePageRelminMxid))
    7916           0 :             *NoFreezePageRelminMxid = multi;
    7917             :         /* heap_prepare_freeze_tuple always freezes pg_upgrade'd xmax */
    7918           0 :         freeze = true;
    7919             :     }
    7920             :     else
    7921             :     {
    7922             :         /* xmax is a MultiXactId that may have an updater XID */
    7923             :         MultiXactMember *members;
    7924             :         int         nmembers;
    7925             : 
    7926             :         Assert(MultiXactIdPrecedesOrEquals(cutoffs->relminmxid, multi));
    7927           4 :         if (MultiXactIdPrecedes(multi, *NoFreezePageRelminMxid))
    7928           4 :             *NoFreezePageRelminMxid = multi;
    7929           4 :         if (MultiXactIdPrecedes(multi, cutoffs->MultiXactCutoff))
    7930           4 :             freeze = true;
    7931             : 
    7932             :         /* need to check whether any member of the mxact is old */
    7933           4 :         nmembers = GetMultiXactIdMembers(multi, &members, false,
    7934           4 :                                          HEAP_XMAX_IS_LOCKED_ONLY(tuple->t_infomask));
    7935             : 
    7936          10 :         for (int i = 0; i < nmembers; i++)
    7937             :         {
    7938           6 :             xid = members[i].xid;
    7939             :             Assert(TransactionIdPrecedesOrEquals(cutoffs->relfrozenxid, xid));
    7940           6 :             if (TransactionIdPrecedes(xid, *NoFreezePageRelfrozenXid))
    7941           0 :                 *NoFreezePageRelfrozenXid = xid;
    7942           6 :             if (TransactionIdPrecedes(xid, cutoffs->FreezeLimit))
    7943           0 :                 freeze = true;
    7944             :         }
    7945           4 :         if (nmembers > 0)
    7946           2 :             pfree(members);
    7947             :     }
    7948             : 
    7949    14582804 :     if (tuple->t_infomask & HEAP_MOVED)
    7950             :     {
    7951           0 :         xid = HeapTupleHeaderGetXvac(tuple);
    7952           0 :         if (TransactionIdIsNormal(xid))
    7953             :         {
    7954             :             Assert(TransactionIdPrecedesOrEquals(cutoffs->relfrozenxid, xid));
    7955           0 :             if (TransactionIdPrecedes(xid, *NoFreezePageRelfrozenXid))
    7956           0 :                 *NoFreezePageRelfrozenXid = xid;
    7957             :             /* heap_prepare_freeze_tuple forces xvac freezing */
    7958           0 :             freeze = true;
    7959             :         }
    7960             :     }
    7961             : 
    7962    14582804 :     return freeze;
    7963             : }
    7964             : 
    7965             : /*
    7966             :  * Maintain snapshotConflictHorizon for caller by ratcheting forward its value
    7967             :  * using any committed XIDs contained in 'tuple', an obsolescent heap tuple
    7968             :  * that caller is in the process of physically removing, e.g. via HOT pruning
    7969             :  * or index deletion.
    7970             :  *
    7971             :  * Caller must initialize its value to InvalidTransactionId, which is
    7972             :  * generally interpreted as "definitely no need for a recovery conflict".
    7973             :  * Final value must reflect all heap tuples that caller will physically remove
    7974             :  * (or remove TID references to) via its ongoing pruning/deletion operation.
    7975             :  * ResolveRecoveryConflictWithSnapshot() is passed the final value (taken from
    7976             :  * caller's WAL record) by REDO routine when it replays caller's operation.
    7977             :  */
    7978             : void
    7979     3121238 : HeapTupleHeaderAdvanceConflictHorizon(HeapTupleHeader tuple,
    7980             :                                       TransactionId *snapshotConflictHorizon)
    7981             : {
    7982     3121238 :     TransactionId xmin = HeapTupleHeaderGetXmin(tuple);
    7983     3121238 :     TransactionId xmax = HeapTupleHeaderGetUpdateXid(tuple);
    7984     3121238 :     TransactionId xvac = HeapTupleHeaderGetXvac(tuple);
    7985             : 
    7986     3121238 :     if (tuple->t_infomask & HEAP_MOVED)
    7987             :     {
    7988           0 :         if (TransactionIdPrecedes(*snapshotConflictHorizon, xvac))
    7989           0 :             *snapshotConflictHorizon = xvac;
    7990             :     }
    7991             : 
    7992             :     /*
    7993             :      * Ignore tuples inserted by an aborted transaction or if the tuple was
    7994             :      * updated/deleted by the inserting transaction.
    7995             :      *
    7996             :      * Look for a committed hint bit, or if no xmin bit is set, check clog.
    7997             :      */
    7998     3121238 :     if (HeapTupleHeaderXminCommitted(tuple) ||
    7999      206982 :         (!HeapTupleHeaderXminInvalid(tuple) && TransactionIdDidCommit(xmin)))
    8000             :     {
    8001     5597392 :         if (xmax != xmin &&
    8002     2626334 :             TransactionIdFollows(xmax, *snapshotConflictHorizon))
    8003      187156 :             *snapshotConflictHorizon = xmax;
    8004             :     }
    8005     3121238 : }
    8006             : 
    8007             : #ifdef USE_PREFETCH
    8008             : /*
    8009             :  * Helper function for heap_index_delete_tuples.  Issues prefetch requests for
    8010             :  * prefetch_count buffers.  The prefetch_state keeps track of all the buffers
    8011             :  * we can prefetch, and which have already been prefetched; each call to this
    8012             :  * function picks up where the previous call left off.
    8013             :  *
    8014             :  * Note: we expect the deltids array to be sorted in an order that groups TIDs
    8015             :  * by heap block, with all TIDs for each block appearing together in exactly
    8016             :  * one group.
    8017             :  */
    8018             : static void
    8019       37380 : index_delete_prefetch_buffer(Relation rel,
    8020             :                              IndexDeletePrefetchState *prefetch_state,
    8021             :                              int prefetch_count)
    8022             : {
    8023       37380 :     BlockNumber cur_hblkno = prefetch_state->cur_hblkno;
    8024       37380 :     int         count = 0;
    8025             :     int         i;
    8026       37380 :     int         ndeltids = prefetch_state->ndeltids;
    8027       37380 :     TM_IndexDelete *deltids = prefetch_state->deltids;
    8028             : 
    8029     1306732 :     for (i = prefetch_state->next_item;
    8030     1277020 :          i < ndeltids && count < prefetch_count;
    8031     1269352 :          i++)
    8032             :     {
    8033     1269352 :         ItemPointer htid = &deltids[i].tid;
    8034             : 
    8035     2527540 :         if (cur_hblkno == InvalidBlockNumber ||
    8036     1258188 :             ItemPointerGetBlockNumber(htid) != cur_hblkno)
    8037             :         {
    8038       33744 :             cur_hblkno = ItemPointerGetBlockNumber(htid);
    8039       33744 :             PrefetchBuffer(rel, MAIN_FORKNUM, cur_hblkno);
    8040       33744 :             count++;
    8041             :         }
    8042             :     }
    8043             : 
    8044             :     /*
    8045             :      * Save the prefetch position so that next time we can continue from that
    8046             :      * position.
    8047             :      */
    8048       37380 :     prefetch_state->next_item = i;
    8049       37380 :     prefetch_state->cur_hblkno = cur_hblkno;
    8050       37380 : }
    8051             : #endif
    8052             : 
    8053             : /*
    8054             :  * Helper function for heap_index_delete_tuples.  Checks for index corruption
    8055             :  * involving an invalid TID in index AM caller's index page.
    8056             :  *
    8057             :  * This is an ideal place for these checks.  The index AM must hold a buffer
    8058             :  * lock on the index page containing the TIDs we examine here, so we don't
    8059             :  * have to worry about concurrent VACUUMs at all.  We can be sure that the
    8060             :  * index is corrupt when htid points directly to an LP_UNUSED item or
    8061             :  * heap-only tuple, which is not the case during standard index scans.
    8062             :  */
    8063             : static inline void
    8064     1067118 : index_delete_check_htid(TM_IndexDeleteOp *delstate,
    8065             :                         Page page, OffsetNumber maxoff,
    8066             :                         ItemPointer htid, TM_IndexStatus *istatus)
    8067             : {
    8068     1067118 :     OffsetNumber indexpagehoffnum = ItemPointerGetOffsetNumber(htid);
    8069             :     ItemId      iid;
    8070             : 
    8071             :     Assert(OffsetNumberIsValid(istatus->idxoffnum));
    8072             : 
    8073     1067118 :     if (unlikely(indexpagehoffnum > maxoff))
    8074           0 :         ereport(ERROR,
    8075             :                 (errcode(ERRCODE_INDEX_CORRUPTED),
    8076             :                  errmsg_internal("heap tid from index tuple (%u,%u) points past end of heap page line pointer array at offset %u of block %u in index \"%s\"",
    8077             :                                  ItemPointerGetBlockNumber(htid),
    8078             :                                  indexpagehoffnum,
    8079             :                                  istatus->idxoffnum, delstate->iblknum,
    8080             :                                  RelationGetRelationName(delstate->irel))));
    8081             : 
    8082     1067118 :     iid = PageGetItemId(page, indexpagehoffnum);
    8083     1067118 :     if (unlikely(!ItemIdIsUsed(iid)))
    8084           0 :         ereport(ERROR,
    8085             :                 (errcode(ERRCODE_INDEX_CORRUPTED),
    8086             :                  errmsg_internal("heap tid from index tuple (%u,%u) points to unused heap page item at offset %u of block %u in index \"%s\"",
    8087             :                                  ItemPointerGetBlockNumber(htid),
    8088             :                                  indexpagehoffnum,
    8089             :                                  istatus->idxoffnum, delstate->iblknum,
    8090             :                                  RelationGetRelationName(delstate->irel))));
    8091             : 
    8092     1067118 :     if (ItemIdHasStorage(iid))
    8093             :     {
    8094             :         HeapTupleHeader htup;
    8095             : 
    8096             :         Assert(ItemIdIsNormal(iid));
    8097      626020 :         htup = (HeapTupleHeader) PageGetItem(page, iid);
    8098             : 
    8099      626020 :         if (unlikely(HeapTupleHeaderIsHeapOnly(htup)))
    8100           0 :             ereport(ERROR,
    8101             :                     (errcode(ERRCODE_INDEX_CORRUPTED),
    8102             :                      errmsg_internal("heap tid from index tuple (%u,%u) points to heap-only tuple at offset %u of block %u in index \"%s\"",
    8103             :                                      ItemPointerGetBlockNumber(htid),
    8104             :                                      indexpagehoffnum,
    8105             :                                      istatus->idxoffnum, delstate->iblknum,
    8106             :                                      RelationGetRelationName(delstate->irel))));
    8107             :     }
    8108     1067118 : }
    8109             : 
    8110             : /*
    8111             :  * heapam implementation of tableam's index_delete_tuples interface.
    8112             :  *
    8113             :  * This helper function is called by index AMs during index tuple deletion.
    8114             :  * See tableam header comments for an explanation of the interface implemented
    8115             :  * here and a general theory of operation.  Note that each call here is either
    8116             :  * a simple index deletion call, or a bottom-up index deletion call.
    8117             :  *
    8118             :  * It's possible for this to generate a fair amount of I/O, since we may be
    8119             :  * deleting hundreds of tuples from a single index block.  To amortize that
    8120             :  * cost to some degree, this uses prefetching and combines repeat accesses to
    8121             :  * the same heap block.
    8122             :  */
    8123             : TransactionId
    8124       11164 : heap_index_delete_tuples(Relation rel, TM_IndexDeleteOp *delstate)
    8125             : {
    8126             :     /* Initial assumption is that earlier pruning took care of conflict */
    8127       11164 :     TransactionId snapshotConflictHorizon = InvalidTransactionId;
    8128       11164 :     BlockNumber blkno = InvalidBlockNumber;
    8129       11164 :     Buffer      buf = InvalidBuffer;
    8130       11164 :     Page        page = NULL;
    8131       11164 :     OffsetNumber maxoff = InvalidOffsetNumber;
    8132             :     TransactionId priorXmax;
    8133             : #ifdef USE_PREFETCH
    8134             :     IndexDeletePrefetchState prefetch_state;
    8135             :     int         prefetch_distance;
    8136             : #endif
    8137             :     SnapshotData SnapshotNonVacuumable;
    8138       11164 :     int         finalndeltids = 0,
    8139       11164 :                 nblocksaccessed = 0;
    8140             : 
    8141             :     /* State that's only used in bottom-up index deletion case */
    8142       11164 :     int         nblocksfavorable = 0;
    8143       11164 :     int         curtargetfreespace = delstate->bottomupfreespace,
    8144       11164 :                 lastfreespace = 0,
    8145       11164 :                 actualfreespace = 0;
    8146       11164 :     bool        bottomup_final_block = false;
    8147             : 
    8148       11164 :     InitNonVacuumableSnapshot(SnapshotNonVacuumable, GlobalVisTestFor(rel));
    8149             : 
    8150             :     /* Sort caller's deltids array by TID for further processing */
    8151       11164 :     index_delete_sort(delstate);
    8152             : 
    8153             :     /*
    8154             :      * Bottom-up case: resort deltids array in an order attuned to where the
    8155             :      * greatest number of promising TIDs are to be found, and determine how
    8156             :      * many blocks from the start of sorted array should be considered
    8157             :      * favorable.  This will also shrink the deltids array in order to
    8158             :      * eliminate completely unfavorable blocks up front.
    8159             :      */
    8160       11164 :     if (delstate->bottomup)
    8161        3828 :         nblocksfavorable = bottomup_sort_and_shrink(delstate);
    8162             : 
    8163             : #ifdef USE_PREFETCH
    8164             :     /* Initialize prefetch state. */
    8165       11164 :     prefetch_state.cur_hblkno = InvalidBlockNumber;
    8166       11164 :     prefetch_state.next_item = 0;
    8167       11164 :     prefetch_state.ndeltids = delstate->ndeltids;
    8168       11164 :     prefetch_state.deltids = delstate->deltids;
    8169             : 
    8170             :     /*
    8171             :      * Determine the prefetch distance that we will attempt to maintain.
    8172             :      *
    8173             :      * Since the caller holds a buffer lock somewhere in rel, we'd better make
    8174             :      * sure that isn't a catalog relation before we call code that does
    8175             :      * syscache lookups, to avoid risk of deadlock.
    8176             :      */
    8177       11164 :     if (IsCatalogRelation(rel))
    8178        7968 :         prefetch_distance = maintenance_io_concurrency;
    8179             :     else
    8180             :         prefetch_distance =
    8181        3196 :             get_tablespace_maintenance_io_concurrency(rel->rd_rel->reltablespace);
    8182             : 
    8183             :     /* Cap initial prefetch distance for bottom-up deletion caller */
    8184       11164 :     if (delstate->bottomup)
    8185             :     {
    8186             :         Assert(nblocksfavorable >= 1);
    8187             :         Assert(nblocksfavorable <= BOTTOMUP_MAX_NBLOCKS);
    8188        3828 :         prefetch_distance = Min(prefetch_distance, nblocksfavorable);
    8189             :     }
    8190             : 
    8191             :     /* Start prefetching. */
    8192       11164 :     index_delete_prefetch_buffer(rel, &prefetch_state, prefetch_distance);
    8193             : #endif
    8194             : 
    8195             :     /* Iterate over deltids, determine which to delete, check their horizon */
    8196             :     Assert(delstate->ndeltids > 0);
    8197     1078282 :     for (int i = 0; i < delstate->ndeltids; i++)
    8198             :     {
    8199     1070946 :         TM_IndexDelete *ideltid = &delstate->deltids[i];
    8200     1070946 :         TM_IndexStatus *istatus = delstate->status + ideltid->id;
    8201     1070946 :         ItemPointer htid = &ideltid->tid;
    8202             :         OffsetNumber offnum;
    8203             : 
    8204             :         /*
    8205             :          * Read buffer, and perform required extra steps each time a new block
    8206             :          * is encountered.  Avoid refetching if it's the same block as the one
    8207             :          * from the last htid.
    8208             :          */
    8209     2130728 :         if (blkno == InvalidBlockNumber ||
    8210     1059782 :             ItemPointerGetBlockNumber(htid) != blkno)
    8211             :         {
    8212             :             /*
    8213             :              * Consider giving up early for bottom-up index deletion caller
    8214             :              * first. (Only prefetch next-next block afterwards, when it
    8215             :              * becomes clear that we're at least going to access the next
    8216             :              * block in line.)
    8217             :              *
    8218             :              * Sometimes the first block frees so much space for bottom-up
    8219             :              * caller that the deletion process can end without accessing any
    8220             :              * more blocks.  It is usually necessary to access 2 or 3 blocks
    8221             :              * per bottom-up deletion operation, though.
    8222             :              */
    8223       30044 :             if (delstate->bottomup)
    8224             :             {
    8225             :                 /*
    8226             :                  * We often allow caller to delete a few additional items
    8227             :                  * whose entries we reached after the point that space target
    8228             :                  * from caller was satisfied.  The cost of accessing the page
    8229             :                  * was already paid at that point, so it made sense to finish
    8230             :                  * it off.  When that happened, we finalize everything here
    8231             :                  * (by finishing off the whole bottom-up deletion operation
    8232             :                  * without needlessly paying the cost of accessing any more
    8233             :                  * blocks).
    8234             :                  */
    8235        8254 :                 if (bottomup_final_block)
    8236         272 :                     break;
    8237             : 
    8238             :                 /*
    8239             :                  * Give up when we didn't enable our caller to free any
    8240             :                  * additional space as a result of processing the page that we
    8241             :                  * just finished up with.  This rule is the main way in which
    8242             :                  * we keep the cost of bottom-up deletion under control.
    8243             :                  */
    8244        7982 :                 if (nblocksaccessed >= 1 && actualfreespace == lastfreespace)
    8245        3556 :                     break;
    8246        4426 :                 lastfreespace = actualfreespace;    /* for next time */
    8247             : 
    8248             :                 /*
    8249             :                  * Deletion operation (which is bottom-up) will definitely
    8250             :                  * access the next block in line.  Prepare for that now.
    8251             :                  *
    8252             :                  * Decay target free space so that we don't hang on for too
    8253             :                  * long with a marginal case. (Space target is only truly
    8254             :                  * helpful when it allows us to recognize that we don't need
    8255             :                  * to access more than 1 or 2 blocks to satisfy caller due to
    8256             :                  * agreeable workload characteristics.)
    8257             :                  *
    8258             :                  * We are a bit more patient when we encounter contiguous
    8259             :                  * blocks, though: these are treated as favorable blocks.  The
    8260             :                  * decay process is only applied when the next block in line
    8261             :                  * is not a favorable/contiguous block.  This is not an
    8262             :                  * exception to the general rule; we still insist on finding
    8263             :                  * at least one deletable item per block accessed.  See
    8264             :                  * bottomup_nblocksfavorable() for full details of the theory
    8265             :                  * behind favorable blocks and heap block locality in general.
    8266             :                  *
    8267             :                  * Note: The first block in line is always treated as a
    8268             :                  * favorable block, so the earliest possible point that the
    8269             :                  * decay can be applied is just before we access the second
    8270             :                  * block in line.  The Assert() verifies this for us.
    8271             :                  */
    8272             :                 Assert(nblocksaccessed > 0 || nblocksfavorable > 0);
    8273        4426 :                 if (nblocksfavorable > 0)
    8274        4220 :                     nblocksfavorable--;
    8275             :                 else
    8276         206 :                     curtargetfreespace /= 2;
    8277             :             }
    8278             : 
    8279             :             /* release old buffer */
    8280       26216 :             if (BufferIsValid(buf))
    8281       15052 :                 UnlockReleaseBuffer(buf);
    8282             : 
    8283       26216 :             blkno = ItemPointerGetBlockNumber(htid);
    8284       26216 :             buf = ReadBuffer(rel, blkno);
    8285       26216 :             nblocksaccessed++;
    8286             :             Assert(!delstate->bottomup ||
    8287             :                    nblocksaccessed <= BOTTOMUP_MAX_NBLOCKS);
    8288             : 
    8289             : #ifdef USE_PREFETCH
    8290             : 
    8291             :             /*
    8292             :              * To maintain the prefetch distance, prefetch one more page for
    8293             :              * each page we read.
    8294             :              */
    8295       26216 :             index_delete_prefetch_buffer(rel, &prefetch_state, 1);
    8296             : #endif
    8297             : 
    8298       26216 :             LockBuffer(buf, BUFFER_LOCK_SHARE);
    8299             : 
    8300       26216 :             page = BufferGetPage(buf);
    8301       26216 :             maxoff = PageGetMaxOffsetNumber(page);
    8302             :         }
    8303             : 
    8304             :         /*
    8305             :          * In passing, detect index corruption involving an index page with a
    8306             :          * TID that points to a location in the heap that couldn't possibly be
    8307             :          * correct.  We only do this with actual TIDs from caller's index page
    8308             :          * (not items reached by traversing through a HOT chain).
    8309             :          */
    8310     1067118 :         index_delete_check_htid(delstate, page, maxoff, htid, istatus);
    8311             : 
    8312     1067118 :         if (istatus->knowndeletable)
    8313             :             Assert(!delstate->bottomup && !istatus->promising);
    8314             :         else
    8315             :         {
    8316      801756 :             ItemPointerData tmp = *htid;
    8317             :             HeapTupleData heapTuple;
    8318             : 
    8319             :             /* Are any tuples from this HOT chain non-vacuumable? */
    8320      801756 :             if (heap_hot_search_buffer(&tmp, rel, buf, &SnapshotNonVacuumable,
    8321             :                                        &heapTuple, NULL, true))
    8322      476150 :                 continue;       /* can't delete entry */
    8323             : 
    8324             :             /* Caller will delete, since whole HOT chain is vacuumable */
    8325      325606 :             istatus->knowndeletable = true;
    8326             : 
    8327             :             /* Maintain index free space info for bottom-up deletion case */
    8328      325606 :             if (delstate->bottomup)
    8329             :             {
    8330             :                 Assert(istatus->freespace > 0);
    8331       14928 :                 actualfreespace += istatus->freespace;
    8332       14928 :                 if (actualfreespace >= curtargetfreespace)
    8333        4780 :                     bottomup_final_block = true;
    8334             :             }
    8335             :         }
    8336             : 
    8337             :         /*
    8338             :          * Maintain snapshotConflictHorizon value for deletion operation as a
    8339             :          * whole by advancing current value using heap tuple headers.  This is
    8340             :          * loosely based on the logic for pruning a HOT chain.
    8341             :          */
    8342      590968 :         offnum = ItemPointerGetOffsetNumber(htid);
    8343      590968 :         priorXmax = InvalidTransactionId;   /* cannot check first XMIN */
    8344             :         for (;;)
    8345       41258 :         {
    8346             :             ItemId      lp;
    8347             :             HeapTupleHeader htup;
    8348             : 
    8349             :             /* Sanity check (pure paranoia) */
    8350      632226 :             if (offnum < FirstOffsetNumber)
    8351           0 :                 break;
    8352             : 
    8353             :             /*
    8354             :              * An offset past the end of page's line pointer array is possible
    8355             :              * when the array was truncated
    8356             :              */
    8357      632226 :             if (offnum > maxoff)
    8358           0 :                 break;
    8359             : 
    8360      632226 :             lp = PageGetItemId(page, offnum);
    8361      632226 :             if (ItemIdIsRedirected(lp))
    8362             :             {
    8363       18334 :                 offnum = ItemIdGetRedirect(lp);
    8364       18334 :                 continue;
    8365             :             }
    8366             : 
    8367             :             /*
    8368             :              * We'll often encounter LP_DEAD line pointers (especially with an
    8369             :              * entry marked knowndeletable by our caller up front).  No heap
    8370             :              * tuple headers get examined for an htid that leads us to an
    8371             :              * LP_DEAD item.  This is okay because the earlier pruning
    8372             :              * operation that made the line pointer LP_DEAD in the first place
    8373             :              * must have considered the original tuple header as part of
    8374             :              * generating its own snapshotConflictHorizon value.
    8375             :              *
    8376             :              * Relying on XLOG_HEAP2_PRUNE_VACUUM_SCAN records like this is
    8377             :              * the same strategy that index vacuuming uses in all cases. Index
    8378             :              * VACUUM WAL records don't even have a snapshotConflictHorizon
    8379             :              * field of their own for this reason.
    8380             :              */
    8381      613892 :             if (!ItemIdIsNormal(lp))
    8382      396520 :                 break;
    8383             : 
    8384      217372 :             htup = (HeapTupleHeader) PageGetItem(page, lp);
    8385             : 
    8386             :             /*
    8387             :              * Check the tuple XMIN against prior XMAX, if any
    8388             :              */
    8389      240296 :             if (TransactionIdIsValid(priorXmax) &&
    8390       22924 :                 !TransactionIdEquals(HeapTupleHeaderGetXmin(htup), priorXmax))
    8391           0 :                 break;
    8392             : 
    8393      217372 :             HeapTupleHeaderAdvanceConflictHorizon(htup,
    8394             :                                                   &snapshotConflictHorizon);
    8395             : 
    8396             :             /*
    8397             :              * If the tuple is not HOT-updated, then we are at the end of this
    8398             :              * HOT-chain.  No need to visit later tuples from the same update
    8399             :              * chain (they get their own index entries) -- just move on to
    8400             :              * next htid from index AM caller.
    8401             :              */
    8402      217372 :             if (!HeapTupleHeaderIsHotUpdated(htup))
    8403      194448 :                 break;
    8404             : 
    8405             :             /* Advance to next HOT chain member */
    8406             :             Assert(ItemPointerGetBlockNumber(&htup->t_ctid) == blkno);
    8407       22924 :             offnum = ItemPointerGetOffsetNumber(&htup->t_ctid);
    8408       22924 :             priorXmax = HeapTupleHeaderGetUpdateXid(htup);
    8409             :         }
    8410             : 
    8411             :         /* Enable further/final shrinking of deltids for caller */
    8412      590968 :         finalndeltids = i + 1;
    8413             :     }
    8414             : 
    8415       11164 :     UnlockReleaseBuffer(buf);
    8416             : 
    8417             :     /*
    8418             :      * Shrink deltids array to exclude non-deletable entries at the end.  This
    8419             :      * is not just a minor optimization.  Final deltids array size might be
    8420             :      * zero for a bottom-up caller.  Index AM is explicitly allowed to rely on
    8421             :      * ndeltids being zero in all cases with zero total deletable entries.
    8422             :      */
    8423             :     Assert(finalndeltids > 0 || delstate->bottomup);
    8424       11164 :     delstate->ndeltids = finalndeltids;
    8425             : 
    8426       11164 :     return snapshotConflictHorizon;
    8427             : }
    8428             : 
    8429             : /*
    8430             :  * Specialized inlineable comparison function for index_delete_sort()
    8431             :  */
    8432             : static inline int
    8433    25151366 : index_delete_sort_cmp(TM_IndexDelete *deltid1, TM_IndexDelete *deltid2)
    8434             : {
    8435    25151366 :     ItemPointer tid1 = &deltid1->tid;
    8436    25151366 :     ItemPointer tid2 = &deltid2->tid;
    8437             : 
    8438             :     {
    8439    25151366 :         BlockNumber blk1 = ItemPointerGetBlockNumber(tid1);
    8440    25151366 :         BlockNumber blk2 = ItemPointerGetBlockNumber(tid2);
    8441             : 
    8442    25151366 :         if (blk1 != blk2)
    8443    10295590 :             return (blk1 < blk2) ? -1 : 1;
    8444             :     }
    8445             :     {
    8446    14855776 :         OffsetNumber pos1 = ItemPointerGetOffsetNumber(tid1);
    8447    14855776 :         OffsetNumber pos2 = ItemPointerGetOffsetNumber(tid2);
    8448             : 
    8449    14855776 :         if (pos1 != pos2)
    8450    14855776 :             return (pos1 < pos2) ? -1 : 1;
    8451             :     }
    8452             : 
    8453             :     Assert(false);
    8454             : 
    8455           0 :     return 0;
    8456             : }
    8457             : 
    8458             : /*
    8459             :  * Sort deltids array from delstate by TID.  This prepares it for further
    8460             :  * processing by heap_index_delete_tuples().
    8461             :  *
    8462             :  * This operation becomes a noticeable consumer of CPU cycles with some
    8463             :  * workloads, so we go to the trouble of specialization/micro optimization.
    8464             :  * We use shellsort for this because it's easy to specialize, compiles to
    8465             :  * relatively few instructions, and is adaptive to presorted inputs/subsets
    8466             :  * (which are typical here).
    8467             :  */
    8468             : static void
    8469       11164 : index_delete_sort(TM_IndexDeleteOp *delstate)
    8470             : {
    8471       11164 :     TM_IndexDelete *deltids = delstate->deltids;
    8472       11164 :     int         ndeltids = delstate->ndeltids;
    8473             : 
    8474             :     /*
    8475             :      * Shellsort gap sequence (taken from Sedgewick-Incerpi paper).
    8476             :      *
    8477             :      * This implementation is fast with array sizes up to ~4500.  This covers
    8478             :      * all supported BLCKSZ values.
    8479             :      */
    8480       11164 :     const int   gaps[9] = {1968, 861, 336, 112, 48, 21, 7, 3, 1};
    8481             : 
    8482             :     /* Think carefully before changing anything here -- keep swaps cheap */
    8483             :     StaticAssertDecl(sizeof(TM_IndexDelete) <= 8,
    8484             :                      "element size exceeds 8 bytes");
    8485             : 
    8486      111640 :     for (int g = 0; g < lengthof(gaps); g++)
    8487             :     {
    8488    15053840 :         for (int hi = gaps[g], i = hi; i < ndeltids; i++)
    8489             :         {
    8490    14953364 :             TM_IndexDelete d = deltids[i];
    8491    14953364 :             int         j = i;
    8492             : 
    8493    25877794 :             while (j >= hi && index_delete_sort_cmp(&deltids[j - hi], &d) >= 0)
    8494             :             {
    8495    10924430 :                 deltids[j] = deltids[j - hi];
    8496    10924430 :                 j -= hi;
    8497             :             }
    8498    14953364 :             deltids[j] = d;
    8499             :         }
    8500             :     }
    8501       11164 : }
    8502             : 
    8503             : /*
    8504             :  * Returns how many blocks should be considered favorable/contiguous for a
    8505             :  * bottom-up index deletion pass.  This is a number of heap blocks that starts
    8506             :  * from and includes the first block in line.
    8507             :  *
    8508             :  * There is always at least one favorable block during bottom-up index
    8509             :  * deletion.  In the worst case (i.e. with totally random heap blocks) the
    8510             :  * first block in line (the only favorable block) can be thought of as a
    8511             :  * degenerate array of contiguous blocks that consists of a single block.
    8512             :  * heap_index_delete_tuples() will expect this.
    8513             :  *
    8514             :  * Caller passes blockgroups, a description of the final order that deltids
    8515             :  * will be sorted in for heap_index_delete_tuples() bottom-up index deletion
    8516             :  * processing.  Note that deltids need not actually be sorted just yet (caller
    8517             :  * only passes deltids to us so that we can interpret blockgroups).
    8518             :  *
    8519             :  * You might guess that the existence of contiguous blocks cannot matter much,
    8520             :  * since in general the main factor that determines which blocks we visit is
    8521             :  * the number of promising TIDs, which is a fixed hint from the index AM.
    8522             :  * We're not really targeting the general case, though -- the actual goal is
    8523             :  * to adapt our behavior to a wide variety of naturally occurring conditions.
    8524             :  * The effects of most of the heuristics we apply are only noticeable in the
    8525             :  * aggregate, over time and across many _related_ bottom-up index deletion
    8526             :  * passes.
    8527             :  *
    8528             :  * Deeming certain blocks favorable allows heapam to recognize and adapt to
    8529             :  * workloads where heap blocks visited during bottom-up index deletion can be
    8530             :  * accessed contiguously, in the sense that each newly visited block is the
    8531             :  * neighbor of the block that bottom-up deletion just finished processing (or
    8532             :  * close enough to it).  It will likely be cheaper to access more favorable
    8533             :  * blocks sooner rather than later (e.g. in this pass, not across a series of
    8534             :  * related bottom-up passes).  Either way it is probably only a matter of time
    8535             :  * (or a matter of further correlated version churn) before all blocks that
    8536             :  * appear together as a single large batch of favorable blocks get accessed by
    8537             :  * _some_ bottom-up pass.  Large batches of favorable blocks tend to either
    8538             :  * appear almost constantly or not even once (it all depends on per-index
    8539             :  * workload characteristics).
    8540             :  *
    8541             :  * Note that the blockgroups sort order applies a power-of-two bucketing
    8542             :  * scheme that creates opportunities for contiguous groups of blocks to get
    8543             :  * batched together, at least with workloads that are naturally amenable to
    8544             :  * being driven by heap block locality.  This doesn't just enhance the spatial
    8545             :  * locality of bottom-up heap block processing in the obvious way.  It also
    8546             :  * enables temporal locality of access, since sorting by heap block number
    8547             :  * naturally tends to make the bottom-up processing order deterministic.
    8548             :  *
    8549             :  * Consider the following example to get a sense of how temporal locality
    8550             :  * might matter: There is a heap relation with several indexes, each of which
    8551             :  * is low to medium cardinality.  It is subject to constant non-HOT updates.
    8552             :  * The updates are skewed (in one part of the primary key, perhaps).  None of
    8553             :  * the indexes are logically modified by the UPDATE statements (if they were
    8554             :  * then bottom-up index deletion would not be triggered in the first place).
    8555             :  * Naturally, each new round of index tuples (for each heap tuple that gets a
    8556             :  * heap_update() call) will have the same heap TID in each and every index.
    8557             :  * Since these indexes are low cardinality and never get logically modified,
    8558             :  * heapam processing during bottom-up deletion passes will access heap blocks
    8559             :  * in approximately sequential order.  Temporal locality of access occurs due
    8560             :  * to bottom-up deletion passes behaving very similarly across each of the
    8561             :  * indexes at any given moment.  This keeps the number of buffer misses needed
    8562             :  * to visit heap blocks to a minimum.
    8563             :  */
    8564             : static int
    8565        3828 : bottomup_nblocksfavorable(IndexDeleteCounts *blockgroups, int nblockgroups,
    8566             :                           TM_IndexDelete *deltids)
    8567             : {
    8568        3828 :     int64       lastblock = -1;
    8569        3828 :     int         nblocksfavorable = 0;
    8570             : 
    8571             :     Assert(nblockgroups >= 1);
    8572             :     Assert(nblockgroups <= BOTTOMUP_MAX_NBLOCKS);
    8573             : 
    8574             :     /*
    8575             :      * We tolerate heap blocks that will be accessed only slightly out of
    8576             :      * physical order.  Small blips occur when a pair of almost-contiguous
    8577             :      * blocks happen to fall into different buckets (perhaps due only to a
    8578             :      * small difference in npromisingtids that the bucketing scheme didn't
    8579             :      * quite manage to ignore).  We effectively ignore these blips by applying
    8580             :      * a small tolerance.  The precise tolerance we use is a little arbitrary,
    8581             :      * but it works well enough in practice.
    8582             :      */
    8583       12104 :     for (int b = 0; b < nblockgroups; b++)
    8584             :     {
    8585       11574 :         IndexDeleteCounts *group = blockgroups + b;
    8586       11574 :         TM_IndexDelete *firstdtid = deltids + group->ifirsttid;
    8587       11574 :         BlockNumber block = ItemPointerGetBlockNumber(&firstdtid->tid);
    8588             : 
    8589       11574 :         if (lastblock != -1 &&
    8590        7746 :             ((int64) block < lastblock - BOTTOMUP_TOLERANCE_NBLOCKS ||
    8591        6540 :              (int64) block > lastblock + BOTTOMUP_TOLERANCE_NBLOCKS))
    8592             :             break;
    8593             : 
    8594        8276 :         nblocksfavorable++;
    8595        8276 :         lastblock = block;
    8596             :     }
    8597             : 
    8598             :     /* Always indicate that there is at least 1 favorable block */
    8599             :     Assert(nblocksfavorable >= 1);
    8600             : 
    8601        3828 :     return nblocksfavorable;
    8602             : }
    8603             : 
    8604             : /*
    8605             :  * qsort comparison function for bottomup_sort_and_shrink()
    8606             :  */
    8607             : static int
    8608      384564 : bottomup_sort_and_shrink_cmp(const void *arg1, const void *arg2)
    8609             : {
    8610      384564 :     const IndexDeleteCounts *group1 = (const IndexDeleteCounts *) arg1;
    8611      384564 :     const IndexDeleteCounts *group2 = (const IndexDeleteCounts *) arg2;
    8612             : 
    8613             :     /*
    8614             :      * Most significant field is npromisingtids (which we invert the order of
    8615             :      * so as to sort in desc order).
    8616             :      *
    8617             :      * Caller should have already normalized npromisingtids fields into
    8618             :      * power-of-two values (buckets).
    8619             :      */
    8620      384564 :     if (group1->npromisingtids > group2->npromisingtids)
    8621       18830 :         return -1;
    8622      365734 :     if (group1->npromisingtids < group2->npromisingtids)
    8623       21698 :         return 1;
    8624             : 
    8625             :     /*
    8626             :      * Tiebreak: desc ntids sort order.
    8627             :      *
    8628             :      * We cannot expect power-of-two values for ntids fields.  We should
    8629             :      * behave as if they were already rounded up for us instead.
    8630             :      */
    8631      344036 :     if (group1->ntids != group2->ntids)
    8632             :     {
    8633      248074 :         uint32      ntids1 = pg_nextpower2_32((uint32) group1->ntids);
    8634      248074 :         uint32      ntids2 = pg_nextpower2_32((uint32) group2->ntids);
    8635             : 
    8636      248074 :         if (ntids1 > ntids2)
    8637       35230 :             return -1;
    8638      212844 :         if (ntids1 < ntids2)
    8639       45282 :             return 1;
    8640             :     }
    8641             : 
    8642             :     /*
    8643             :      * Tiebreak: asc offset-into-deltids-for-block (offset to first TID for
    8644             :      * block in deltids array) order.
    8645             :      *
    8646             :      * This is equivalent to sorting in ascending heap block number order
    8647             :      * (among otherwise equal subsets of the array).  This approach allows us
    8648             :      * to avoid accessing the out-of-line TID.  (We rely on the assumption
    8649             :      * that the deltids array was sorted in ascending heap TID order when
    8650             :      * these offsets to the first TID from each heap block group were formed.)
    8651             :      */
    8652      263524 :     if (group1->ifirsttid > group2->ifirsttid)
    8653      129048 :         return 1;
    8654      134476 :     if (group1->ifirsttid < group2->ifirsttid)
    8655      134476 :         return -1;
    8656             : 
    8657           0 :     pg_unreachable();
    8658             : 
    8659             :     return 0;
    8660             : }
    8661             : 
    8662             : /*
    8663             :  * heap_index_delete_tuples() helper function for bottom-up deletion callers.
    8664             :  *
    8665             :  * Sorts deltids array in the order needed for useful processing by bottom-up
    8666             :  * deletion.  The array should already be sorted in TID order when we're
    8667             :  * called.  The sort process groups heap TIDs from deltids into heap block
    8668             :  * groupings.  Earlier/more-promising groups/blocks are usually those that are
    8669             :  * known to have the most "promising" TIDs.
    8670             :  *
    8671             :  * Sets new size of deltids array (ndeltids) in state.  deltids will only have
    8672             :  * TIDs from the BOTTOMUP_MAX_NBLOCKS most promising heap blocks when we
    8673             :  * return.  This often means that deltids will be shrunk to a small fraction
    8674             :  * of its original size (we eliminate many heap blocks from consideration for
    8675             :  * caller up front).
    8676             :  *
    8677             :  * Returns the number of "favorable" blocks.  See bottomup_nblocksfavorable()
    8678             :  * for a definition and full details.
    8679             :  */
    8680             : static int
    8681        3828 : bottomup_sort_and_shrink(TM_IndexDeleteOp *delstate)
    8682             : {
    8683             :     IndexDeleteCounts *blockgroups;
    8684             :     TM_IndexDelete *reordereddeltids;
    8685        3828 :     BlockNumber curblock = InvalidBlockNumber;
    8686        3828 :     int         nblockgroups = 0;
    8687        3828 :     int         ncopied = 0;
    8688        3828 :     int         nblocksfavorable = 0;
    8689             : 
    8690             :     Assert(delstate->bottomup);
    8691             :     Assert(delstate->ndeltids > 0);
    8692             : 
    8693             :     /* Calculate per-heap-block count of TIDs */
    8694        3828 :     blockgroups = palloc(sizeof(IndexDeleteCounts) * delstate->ndeltids);
    8695     1857900 :     for (int i = 0; i < delstate->ndeltids; i++)
    8696             :     {
    8697     1854072 :         TM_IndexDelete *ideltid = &delstate->deltids[i];
    8698     1854072 :         TM_IndexStatus *istatus = delstate->status + ideltid->id;
    8699     1854072 :         ItemPointer htid = &ideltid->tid;
    8700     1854072 :         bool        promising = istatus->promising;
    8701             : 
    8702     1854072 :         if (curblock != ItemPointerGetBlockNumber(htid))
    8703             :         {
    8704             :             /* New block group */
    8705       74754 :             nblockgroups++;
    8706             : 
    8707             :             Assert(curblock < ItemPointerGetBlockNumber(htid) ||
    8708             :                    !BlockNumberIsValid(curblock));
    8709             : 
    8710       74754 :             curblock = ItemPointerGetBlockNumber(htid);
    8711       74754 :             blockgroups[nblockgroups - 1].ifirsttid = i;
    8712       74754 :             blockgroups[nblockgroups - 1].ntids = 1;
    8713       74754 :             blockgroups[nblockgroups - 1].npromisingtids = 0;
    8714             :         }
    8715             :         else
    8716             :         {
    8717     1779318 :             blockgroups[nblockgroups - 1].ntids++;
    8718             :         }
    8719             : 
    8720     1854072 :         if (promising)
    8721      235858 :             blockgroups[nblockgroups - 1].npromisingtids++;
    8722             :     }
    8723             : 
    8724             :     /*
    8725             :      * We're about ready to sort block groups to determine the optimal order
    8726             :      * for visiting heap blocks.  But before we do, round the number of
    8727             :      * promising tuples for each block group up to the next power-of-two,
    8728             :      * unless it is very low (less than 4), in which case we round up to 4.
    8729             :      * npromisingtids is far too noisy to trust when choosing between a pair
    8730             :      * of block groups that both have very low values.
    8731             :      *
    8732             :      * This scheme divides heap blocks/block groups into buckets.  Each bucket
    8733             :      * contains blocks that have _approximately_ the same number of promising
    8734             :      * TIDs as each other.  The goal is to ignore relatively small differences
    8735             :      * in the total number of promising entries, so that the whole process can
    8736             :      * give a little weight to heapam factors (like heap block locality)
    8737             :      * instead.  This isn't a trade-off, really -- we have nothing to lose. It
    8738             :      * would be foolish to interpret small differences in npromisingtids
    8739             :      * values as anything more than noise.
    8740             :      *
    8741             :      * We tiebreak on nhtids when sorting block group subsets that have the
    8742             :      * same npromisingtids, but this has the same issues as npromisingtids,
    8743             :      * and so nhtids is subject to the same power-of-two bucketing scheme. The
    8744             :      * only reason that we don't fix nhtids in the same way here too is that
    8745             :      * we'll need accurate nhtids values after the sort.  We handle nhtids
    8746             :      * bucketization dynamically instead (in the sort comparator).
    8747             :      *
    8748             :      * See bottomup_nblocksfavorable() for a full explanation of when and how
    8749             :      * heap locality/favorable blocks can significantly influence when and how
    8750             :      * heap blocks are accessed.
    8751             :      */
    8752       78582 :     for (int b = 0; b < nblockgroups; b++)
    8753             :     {
    8754       74754 :         IndexDeleteCounts *group = blockgroups + b;
    8755             : 
    8756             :         /* Better off falling back on nhtids with low npromisingtids */
    8757       74754 :         if (group->npromisingtids <= 4)
    8758       63854 :             group->npromisingtids = 4;
    8759             :         else
    8760       10900 :             group->npromisingtids =
    8761       10900 :                 pg_nextpower2_32((uint32) group->npromisingtids);
    8762             :     }
    8763             : 
    8764             :     /* Sort groups and rearrange caller's deltids array */
    8765        3828 :     qsort(blockgroups, nblockgroups, sizeof(IndexDeleteCounts),
    8766             :           bottomup_sort_and_shrink_cmp);
    8767        3828 :     reordereddeltids = palloc(delstate->ndeltids * sizeof(TM_IndexDelete));
    8768             : 
    8769        3828 :     nblockgroups = Min(BOTTOMUP_MAX_NBLOCKS, nblockgroups);
    8770             :     /* Determine number of favorable blocks at the start of final deltids */
    8771        3828 :     nblocksfavorable = bottomup_nblocksfavorable(blockgroups, nblockgroups,
    8772             :                                                  delstate->deltids);
    8773             : 
    8774       25574 :     for (int b = 0; b < nblockgroups; b++)
    8775             :     {
    8776       21746 :         IndexDeleteCounts *group = blockgroups + b;
    8777       21746 :         TM_IndexDelete *firstdtid = delstate->deltids + group->ifirsttid;
    8778             : 
    8779       21746 :         memcpy(reordereddeltids + ncopied, firstdtid,
    8780       21746 :                sizeof(TM_IndexDelete) * group->ntids);
    8781       21746 :         ncopied += group->ntids;
    8782             :     }
    8783             : 
    8784             :     /* Copy final grouped and sorted TIDs back into start of caller's array */
    8785        3828 :     memcpy(delstate->deltids, reordereddeltids,
    8786             :            sizeof(TM_IndexDelete) * ncopied);
    8787        3828 :     delstate->ndeltids = ncopied;
    8788             : 
    8789        3828 :     pfree(reordereddeltids);
    8790        3828 :     pfree(blockgroups);
    8791             : 
    8792        3828 :     return nblocksfavorable;
    8793             : }
    8794             : 
    8795             : /*
    8796             :  * Perform XLogInsert for a heap-visible operation.  'block' is the block
    8797             :  * being marked all-visible, and vm_buffer is the buffer containing the
    8798             :  * corresponding visibility map block.  Both should have already been modified
    8799             :  * and dirtied.
    8800             :  *
    8801             :  * snapshotConflictHorizon comes from the largest xmin on the page being
    8802             :  * marked all-visible.  REDO routine uses it to generate recovery conflicts.
    8803             :  *
    8804             :  * If checksums or wal_log_hints are enabled, we may also generate a full-page
    8805             :  * image of heap_buffer. Otherwise, we optimize away the FPI (by specifying
    8806             :  * REGBUF_NO_IMAGE for the heap buffer), in which case the caller should *not*
    8807             :  * update the heap page's LSN.
    8808             :  */
    8809             : XLogRecPtr
    8810       83652 : log_heap_visible(Relation rel, Buffer heap_buffer, Buffer vm_buffer,
    8811             :                  TransactionId snapshotConflictHorizon, uint8 vmflags)
    8812             : {
    8813             :     xl_heap_visible xlrec;
    8814             :     XLogRecPtr  recptr;
    8815             :     uint8       flags;
    8816             : 
    8817             :     Assert(BufferIsValid(heap_buffer));
    8818             :     Assert(BufferIsValid(vm_buffer));
    8819             : 
    8820       83652 :     xlrec.snapshotConflictHorizon = snapshotConflictHorizon;
    8821       83652 :     xlrec.flags = vmflags;
    8822       83652 :     if (RelationIsAccessibleInLogicalDecoding(rel))
    8823         266 :         xlrec.flags |= VISIBILITYMAP_XLOG_CATALOG_REL;
    8824       83652 :     XLogBeginInsert();
    8825       83652 :     XLogRegisterData(&xlrec, SizeOfHeapVisible);
    8826             : 
    8827       83652 :     XLogRegisterBuffer(0, vm_buffer, 0);
    8828             : 
    8829       83652 :     flags = REGBUF_STANDARD;
    8830       83652 :     if (!XLogHintBitIsNeeded())
    8831        5854 :         flags |= REGBUF_NO_IMAGE;
    8832       83652 :     XLogRegisterBuffer(1, heap_buffer, flags);
    8833             : 
    8834       83652 :     recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_VISIBLE);
    8835             : 
    8836       83652 :     return recptr;
    8837             : }
    8838             : 
    8839             : /*
    8840             :  * Perform XLogInsert for a heap-update operation.  Caller must already
    8841             :  * have modified the buffer(s) and marked them dirty.
    8842             :  */
    8843             : static XLogRecPtr
    8844      572742 : log_heap_update(Relation reln, Buffer oldbuf,
    8845             :                 Buffer newbuf, HeapTuple oldtup, HeapTuple newtup,
    8846             :                 HeapTuple old_key_tuple,
    8847             :                 bool all_visible_cleared, bool new_all_visible_cleared)
    8848             : {
    8849             :     xl_heap_update xlrec;
    8850             :     xl_heap_header xlhdr;
    8851             :     xl_heap_header xlhdr_idx;
    8852             :     uint8       info;
    8853             :     uint16      prefix_suffix[2];
    8854      572742 :     uint16      prefixlen = 0,
    8855      572742 :                 suffixlen = 0;
    8856             :     XLogRecPtr  recptr;
    8857      572742 :     Page        page = BufferGetPage(newbuf);
    8858      572742 :     bool        need_tuple_data = RelationIsLogicallyLogged(reln);
    8859             :     bool        init;
    8860             :     int         bufflags;
    8861             : 
    8862             :     /* Caller should not call me on a non-WAL-logged relation */
    8863             :     Assert(RelationNeedsWAL(reln));
    8864             : 
    8865      572742 :     XLogBeginInsert();
    8866             : 
    8867      572742 :     if (HeapTupleIsHeapOnly(newtup))
    8868      276768 :         info = XLOG_HEAP_HOT_UPDATE;
    8869             :     else
    8870      295974 :         info = XLOG_HEAP_UPDATE;
    8871             : 
    8872             :     /*
    8873             :      * If the old and new tuple are on the same page, we only need to log the
    8874             :      * parts of the new tuple that were changed.  That saves on the amount of
    8875             :      * WAL we need to write.  Currently, we just count any unchanged bytes in
    8876             :      * the beginning and end of the tuple.  That's quick to check, and
    8877             :      * perfectly covers the common case that only one field is updated.
    8878             :      *
    8879             :      * We could do this even if the old and new tuple are on different pages,
    8880             :      * but only if we don't make a full-page image of the old page, which is
    8881             :      * difficult to know in advance.  Also, if the old tuple is corrupt for
    8882             :      * some reason, it would allow the corruption to propagate the new page,
    8883             :      * so it seems best to avoid.  Under the general assumption that most
    8884             :      * updates tend to create the new tuple version on the same page, there
    8885             :      * isn't much to be gained by doing this across pages anyway.
    8886             :      *
    8887             :      * Skip this if we're taking a full-page image of the new page, as we
    8888             :      * don't include the new tuple in the WAL record in that case.  Also
    8889             :      * disable if wal_level='logical', as logical decoding needs to be able to
    8890             :      * read the new tuple in whole from the WAL record alone.
    8891             :      */
    8892      572742 :     if (oldbuf == newbuf && !need_tuple_data &&
    8893      276612 :         !XLogCheckBufferNeedsBackup(newbuf))
    8894             :     {
    8895      275696 :         char       *oldp = (char *) oldtup->t_data + oldtup->t_data->t_hoff;
    8896      275696 :         char       *newp = (char *) newtup->t_data + newtup->t_data->t_hoff;
    8897      275696 :         int         oldlen = oldtup->t_len - oldtup->t_data->t_hoff;
    8898      275696 :         int         newlen = newtup->t_len - newtup->t_data->t_hoff;
    8899             : 
    8900             :         /* Check for common prefix between old and new tuple */
    8901    22655448 :         for (prefixlen = 0; prefixlen < Min(oldlen, newlen); prefixlen++)
    8902             :         {
    8903    22606490 :             if (newp[prefixlen] != oldp[prefixlen])
    8904      226738 :                 break;
    8905             :         }
    8906             : 
    8907             :         /*
    8908             :          * Storing the length of the prefix takes 2 bytes, so we need to save
    8909             :          * at least 3 bytes or there's no point.
    8910             :          */
    8911      275696 :         if (prefixlen < 3)
    8912       44148 :             prefixlen = 0;
    8913             : 
    8914             :         /* Same for suffix */
    8915     8867970 :         for (suffixlen = 0; suffixlen < Min(oldlen, newlen) - prefixlen; suffixlen++)
    8916             :         {
    8917     8818514 :             if (newp[newlen - suffixlen - 1] != oldp[oldlen - suffixlen - 1])
    8918      226240 :                 break;
    8919             :         }
    8920      275696 :         if (suffixlen < 3)
    8921       67674 :             suffixlen = 0;
    8922             :     }
    8923             : 
    8924             :     /* Prepare main WAL data chain */
    8925      572742 :     xlrec.flags = 0;
    8926      572742 :     if (all_visible_cleared)
    8927        3230 :         xlrec.flags |= XLH_UPDATE_OLD_ALL_VISIBLE_CLEARED;
    8928      572742 :     if (new_all_visible_cleared)
    8929        1486 :         xlrec.flags |= XLH_UPDATE_NEW_ALL_VISIBLE_CLEARED;
    8930      572742 :     if (prefixlen > 0)
    8931      231548 :         xlrec.flags |= XLH_UPDATE_PREFIX_FROM_OLD;
    8932      572742 :     if (suffixlen > 0)
    8933      208022 :         xlrec.flags |= XLH_UPDATE_SUFFIX_FROM_OLD;
    8934      572742 :     if (need_tuple_data)
    8935             :     {
    8936       94044 :         xlrec.flags |= XLH_UPDATE_CONTAINS_NEW_TUPLE;
    8937       94044 :         if (old_key_tuple)
    8938             :         {
    8939         286 :             if (reln->rd_rel->relreplident == REPLICA_IDENTITY_FULL)
    8940         126 :                 xlrec.flags |= XLH_UPDATE_CONTAINS_OLD_TUPLE;
    8941             :             else
    8942         160 :                 xlrec.flags |= XLH_UPDATE_CONTAINS_OLD_KEY;
    8943             :         }
    8944             :     }
    8945             : 
    8946             :     /* If new tuple is the single and first tuple on page... */
    8947      579276 :     if (ItemPointerGetOffsetNumber(&(newtup->t_self)) == FirstOffsetNumber &&
    8948        6534 :         PageGetMaxOffsetNumber(page) == FirstOffsetNumber)
    8949             :     {
    8950        6258 :         info |= XLOG_HEAP_INIT_PAGE;
    8951        6258 :         init = true;
    8952             :     }
    8953             :     else
    8954      566484 :         init = false;
    8955             : 
    8956             :     /* Prepare WAL data for the old page */
    8957      572742 :     xlrec.old_offnum = ItemPointerGetOffsetNumber(&oldtup->t_self);
    8958      572742 :     xlrec.old_xmax = HeapTupleHeaderGetRawXmax(oldtup->t_data);
    8959     1145484 :     xlrec.old_infobits_set = compute_infobits(oldtup->t_data->t_infomask,
    8960      572742 :                                               oldtup->t_data->t_infomask2);
    8961             : 
    8962             :     /* Prepare WAL data for the new page */
    8963      572742 :     xlrec.new_offnum = ItemPointerGetOffsetNumber(&newtup->t_self);
    8964      572742 :     xlrec.new_xmax = HeapTupleHeaderGetRawXmax(newtup->t_data);
    8965             : 
    8966      572742 :     bufflags = REGBUF_STANDARD;
    8967      572742 :     if (init)
    8968        6258 :         bufflags |= REGBUF_WILL_INIT;
    8969      572742 :     if (need_tuple_data)
    8970       94044 :         bufflags |= REGBUF_KEEP_DATA;
    8971             : 
    8972      572742 :     XLogRegisterBuffer(0, newbuf, bufflags);
    8973      572742 :     if (oldbuf != newbuf)
    8974      272246 :         XLogRegisterBuffer(1, oldbuf, REGBUF_STANDARD);
    8975             : 
    8976      572742 :     XLogRegisterData(&xlrec, SizeOfHeapUpdate);
    8977             : 
    8978             :     /*
    8979             :      * Prepare WAL data for the new tuple.
    8980             :      */
    8981      572742 :     if (prefixlen > 0 || suffixlen > 0)
    8982             :     {
    8983      274788 :         if (prefixlen > 0 && suffixlen > 0)
    8984             :         {
    8985      164782 :             prefix_suffix[0] = prefixlen;
    8986      164782 :             prefix_suffix[1] = suffixlen;
    8987      164782 :             XLogRegisterBufData(0, &prefix_suffix, sizeof(uint16) * 2);
    8988             :         }
    8989      110006 :         else if (prefixlen > 0)
    8990             :         {
    8991       66766 :             XLogRegisterBufData(0, &prefixlen, sizeof(uint16));
    8992             :         }
    8993             :         else
    8994             :         {
    8995       43240 :             XLogRegisterBufData(0, &suffixlen, sizeof(uint16));
    8996             :         }
    8997             :     }
    8998             : 
    8999      572742 :     xlhdr.t_infomask2 = newtup->t_data->t_infomask2;
    9000      572742 :     xlhdr.t_infomask = newtup->t_data->t_infomask;
    9001      572742 :     xlhdr.t_hoff = newtup->t_data->t_hoff;
    9002             :     Assert(SizeofHeapTupleHeader + prefixlen + suffixlen <= newtup->t_len);
    9003             : 
    9004             :     /*
    9005             :      * PG73FORMAT: write bitmap [+ padding] [+ oid] + data
    9006             :      *
    9007             :      * The 'data' doesn't include the common prefix or suffix.
    9008             :      */
    9009      572742 :     XLogRegisterBufData(0, &xlhdr, SizeOfHeapHeader);
    9010      572742 :     if (prefixlen == 0)
    9011             :     {
    9012      341194 :         XLogRegisterBufData(0,
    9013      341194 :                             (char *) newtup->t_data + SizeofHeapTupleHeader,
    9014      341194 :                             newtup->t_len - SizeofHeapTupleHeader - suffixlen);
    9015             :     }
    9016             :     else
    9017             :     {
    9018             :         /*
    9019             :          * Have to write the null bitmap and data after the common prefix as
    9020             :          * two separate rdata entries.
    9021             :          */
    9022             :         /* bitmap [+ padding] [+ oid] */
    9023      231548 :         if (newtup->t_data->t_hoff - SizeofHeapTupleHeader > 0)
    9024             :         {
    9025      231548 :             XLogRegisterBufData(0,
    9026      231548 :                                 (char *) newtup->t_data + SizeofHeapTupleHeader,
    9027      231548 :                                 newtup->t_data->t_hoff - SizeofHeapTupleHeader);
    9028             :         }
    9029             : 
    9030             :         /* data after common prefix */
    9031      231548 :         XLogRegisterBufData(0,
    9032      231548 :                             (char *) newtup->t_data + newtup->t_data->t_hoff + prefixlen,
    9033      231548 :                             newtup->t_len - newtup->t_data->t_hoff - prefixlen - suffixlen);
    9034             :     }
    9035             : 
    9036             :     /* We need to log a tuple identity */
    9037      572742 :     if (need_tuple_data && old_key_tuple)
    9038             :     {
    9039             :         /* don't really need this, but its more comfy to decode */
    9040         286 :         xlhdr_idx.t_infomask2 = old_key_tuple->t_data->t_infomask2;
    9041         286 :         xlhdr_idx.t_infomask = old_key_tuple->t_data->t_infomask;
    9042         286 :         xlhdr_idx.t_hoff = old_key_tuple->t_data->t_hoff;
    9043             : 
    9044         286 :         XLogRegisterData(&xlhdr_idx, SizeOfHeapHeader);
    9045             : 
    9046             :         /* PG73FORMAT: write bitmap [+ padding] [+ oid] + data */
    9047         286 :         XLogRegisterData((char *) old_key_tuple->t_data + SizeofHeapTupleHeader,
    9048         286 :                          old_key_tuple->t_len - SizeofHeapTupleHeader);
    9049             :     }
    9050             : 
    9051             :     /* filtering by origin on a row level is much more efficient */
    9052      572742 :     XLogSetRecordFlags(XLOG_INCLUDE_ORIGIN);
    9053             : 
    9054      572742 :     recptr = XLogInsert(RM_HEAP_ID, info);
    9055             : 
    9056      572742 :     return recptr;
    9057             : }
    9058             : 
    9059             : /*
    9060             :  * Perform XLogInsert of an XLOG_HEAP2_NEW_CID record
    9061             :  *
    9062             :  * This is only used in wal_level >= WAL_LEVEL_LOGICAL, and only for catalog
    9063             :  * tuples.
    9064             :  */
    9065             : static XLogRecPtr
    9066       48200 : log_heap_new_cid(Relation relation, HeapTuple tup)
    9067             : {
    9068             :     xl_heap_new_cid xlrec;
    9069             : 
    9070             :     XLogRecPtr  recptr;
    9071       48200 :     HeapTupleHeader hdr = tup->t_data;
    9072             : 
    9073             :     Assert(ItemPointerIsValid(&tup->t_self));
    9074             :     Assert(tup->t_tableOid != InvalidOid);
    9075             : 
    9076       48200 :     xlrec.top_xid = GetTopTransactionId();
    9077       48200 :     xlrec.target_locator = relation->rd_locator;
    9078       48200 :     xlrec.target_tid = tup->t_self;
    9079             : 
    9080             :     /*
    9081             :      * If the tuple got inserted & deleted in the same TX we definitely have a
    9082             :      * combo CID, set cmin and cmax.
    9083             :      */
    9084       48200 :     if (hdr->t_infomask & HEAP_COMBOCID)
    9085             :     {
    9086             :         Assert(!(hdr->t_infomask & HEAP_XMAX_INVALID));
    9087             :         Assert(!HeapTupleHeaderXminInvalid(hdr));
    9088        4036 :         xlrec.cmin = HeapTupleHeaderGetCmin(hdr);
    9089        4036 :         xlrec.cmax = HeapTupleHeaderGetCmax(hdr);
    9090        4036 :         xlrec.combocid = HeapTupleHeaderGetRawCommandId(hdr);
    9091             :     }
    9092             :     /* No combo CID, so only cmin or cmax can be set by this TX */
    9093             :     else
    9094             :     {
    9095             :         /*
    9096             :          * Tuple inserted.
    9097             :          *
    9098             :          * We need to check for LOCK ONLY because multixacts might be
    9099             :          * transferred to the new tuple in case of FOR KEY SHARE updates in
    9100             :          * which case there will be an xmax, although the tuple just got
    9101             :          * inserted.
    9102             :          */
    9103       57786 :         if (hdr->t_infomask & HEAP_XMAX_INVALID ||
    9104       13622 :             HEAP_XMAX_IS_LOCKED_ONLY(hdr->t_infomask))
    9105             :         {
    9106       30544 :             xlrec.cmin = HeapTupleHeaderGetRawCommandId(hdr);
    9107       30544 :             xlrec.cmax = InvalidCommandId;
    9108             :         }
    9109             :         /* Tuple from a different tx updated or deleted. */
    9110             :         else
    9111             :         {
    9112       13620 :             xlrec.cmin = InvalidCommandId;
    9113       13620 :             xlrec.cmax = HeapTupleHeaderGetRawCommandId(hdr);
    9114             :         }
    9115       44164 :         xlrec.combocid = InvalidCommandId;
    9116             :     }
    9117             : 
    9118             :     /*
    9119             :      * Note that we don't need to register the buffer here, because this
    9120             :      * operation does not modify the page. The insert/update/delete that
    9121             :      * called us certainly did, but that's WAL-logged separately.
    9122             :      */
    9123       48200 :     XLogBeginInsert();
    9124       48200 :     XLogRegisterData(&xlrec, SizeOfHeapNewCid);
    9125             : 
    9126             :     /* will be looked at irrespective of origin */
    9127             : 
    9128       48200 :     recptr = XLogInsert(RM_HEAP2_ID, XLOG_HEAP2_NEW_CID);
    9129             : 
    9130       48200 :     return recptr;
    9131             : }
    9132             : 
    9133             : /*
    9134             :  * Build a heap tuple representing the configured REPLICA IDENTITY to represent
    9135             :  * the old tuple in an UPDATE or DELETE.
    9136             :  *
    9137             :  * Returns NULL if there's no need to log an identity or if there's no suitable
    9138             :  * key defined.
    9139             :  *
    9140             :  * Pass key_required true if any replica identity columns changed value, or if
    9141             :  * any of them have any external data.  Delete must always pass true.
    9142             :  *
    9143             :  * *copy is set to true if the returned tuple is a modified copy rather than
    9144             :  * the same tuple that was passed in.
    9145             :  */
    9146             : static HeapTuple
    9147     3539298 : ExtractReplicaIdentity(Relation relation, HeapTuple tp, bool key_required,
    9148             :                        bool *copy)
    9149             : {
    9150     3539298 :     TupleDesc   desc = RelationGetDescr(relation);
    9151     3539298 :     char        replident = relation->rd_rel->relreplident;
    9152             :     Bitmapset  *idattrs;
    9153             :     HeapTuple   key_tuple;
    9154             :     bool        nulls[MaxHeapAttributeNumber];
    9155             :     Datum       values[MaxHeapAttributeNumber];
    9156             : 
    9157     3539298 :     *copy = false;
    9158             : 
    9159     3539298 :     if (!RelationIsLogicallyLogged(relation))
    9160     3338722 :         return NULL;
    9161             : 
    9162      200576 :     if (replident == REPLICA_IDENTITY_NOTHING)
    9163         462 :         return NULL;
    9164             : 
    9165      200114 :     if (replident == REPLICA_IDENTITY_FULL)
    9166             :     {
    9167             :         /*
    9168             :          * When logging the entire old tuple, it very well could contain
    9169             :          * toasted columns. If so, force them to be inlined.
    9170             :          */
    9171         382 :         if (HeapTupleHasExternal(tp))
    9172             :         {
    9173           8 :             *copy = true;
    9174           8 :             tp = toast_flatten_tuple(tp, desc);
    9175             :         }
    9176         382 :         return tp;
    9177             :     }
    9178             : 
    9179             :     /* if the key isn't required and we're only logging the key, we're done */
    9180      199732 :     if (!key_required)
    9181       93758 :         return NULL;
    9182             : 
    9183             :     /* find out the replica identity columns */
    9184      105974 :     idattrs = RelationGetIndexAttrBitmap(relation,
    9185             :                                          INDEX_ATTR_BITMAP_IDENTITY_KEY);
    9186             : 
    9187             :     /*
    9188             :      * If there's no defined replica identity columns, treat as !key_required.
    9189             :      * (This case should not be reachable from heap_update, since that should
    9190             :      * calculate key_required accurately.  But heap_delete just passes
    9191             :      * constant true for key_required, so we can hit this case in deletes.)
    9192             :      */
    9193      105974 :     if (bms_is_empty(idattrs))
    9194       12042 :         return NULL;
    9195             : 
    9196             :     /*
    9197             :      * Construct a new tuple containing only the replica identity columns,
    9198             :      * with nulls elsewhere.  While we're at it, assert that the replica
    9199             :      * identity columns aren't null.
    9200             :      */
    9201       93932 :     heap_deform_tuple(tp, desc, values, nulls);
    9202             : 
    9203      301782 :     for (int i = 0; i < desc->natts; i++)
    9204             :     {
    9205      207850 :         if (bms_is_member(i + 1 - FirstLowInvalidHeapAttributeNumber,
    9206             :                           idattrs))
    9207             :             Assert(!nulls[i]);
    9208             :         else
    9209      113894 :             nulls[i] = true;
    9210             :     }
    9211             : 
    9212       93932 :     key_tuple = heap_form_tuple(desc, values, nulls);
    9213       93932 :     *copy = true;
    9214             : 
    9215       93932 :     bms_free(idattrs);
    9216             : 
    9217             :     /*
    9218             :      * If the tuple, which by here only contains indexed columns, still has
    9219             :      * toasted columns, force them to be inlined. This is somewhat unlikely
    9220             :      * since there's limits on the size of indexed columns, so we don't
    9221             :      * duplicate toast_flatten_tuple()s functionality in the above loop over
    9222             :      * the indexed columns, even if it would be more efficient.
    9223             :      */
    9224       93932 :     if (HeapTupleHasExternal(key_tuple))
    9225             :     {
    9226           8 :         HeapTuple   oldtup = key_tuple;
    9227             : 
    9228           8 :         key_tuple = toast_flatten_tuple(oldtup, desc);
    9229           8 :         heap_freetuple(oldtup);
    9230             :     }
    9231             : 
    9232       93932 :     return key_tuple;
    9233             : }
    9234             : 
    9235             : /*
    9236             :  * HeapCheckForSerializableConflictOut
    9237             :  *      We are reading a tuple.  If it's not visible, there may be a
    9238             :  *      rw-conflict out with the inserter.  Otherwise, if it is visible to us
    9239             :  *      but has been deleted, there may be a rw-conflict out with the deleter.
    9240             :  *
    9241             :  * We will determine the top level xid of the writing transaction with which
    9242             :  * we may be in conflict, and ask CheckForSerializableConflictOut() to check
    9243             :  * for overlap with our own transaction.
    9244             :  *
    9245             :  * This function should be called just about anywhere in heapam.c where a
    9246             :  * tuple has been read. The caller must hold at least a shared lock on the
    9247             :  * buffer, because this function might set hint bits on the tuple. There is
    9248             :  * currently no known reason to call this function from an index AM.
    9249             :  */
    9250             : void
    9251    60098946 : HeapCheckForSerializableConflictOut(bool visible, Relation relation,
    9252             :                                     HeapTuple tuple, Buffer buffer,
    9253             :                                     Snapshot snapshot)
    9254             : {
    9255             :     TransactionId xid;
    9256             :     HTSV_Result htsvResult;
    9257             : 
    9258    60098946 :     if (!CheckForSerializableConflictOutNeeded(relation, snapshot))
    9259    60048276 :         return;
    9260             : 
    9261             :     /*
    9262             :      * Check to see whether the tuple has been written to by a concurrent
    9263             :      * transaction, either to create it not visible to us, or to delete it
    9264             :      * while it is visible to us.  The "visible" bool indicates whether the
    9265             :      * tuple is visible to us, while HeapTupleSatisfiesVacuum checks what else
    9266             :      * is going on with it.
    9267             :      *
    9268             :      * In the event of a concurrently inserted tuple that also happens to have
    9269             :      * been concurrently updated (by a separate transaction), the xmin of the
    9270             :      * tuple will be used -- not the updater's xid.
    9271             :      */
    9272       50670 :     htsvResult = HeapTupleSatisfiesVacuum(tuple, TransactionXmin, buffer);
    9273       50670 :     switch (htsvResult)
    9274             :     {
    9275       49066 :         case HEAPTUPLE_LIVE:
    9276       49066 :             if (visible)
    9277       49040 :                 return;
    9278          26 :             xid = HeapTupleHeaderGetXmin(tuple->t_data);
    9279          26 :             break;
    9280         704 :         case HEAPTUPLE_RECENTLY_DEAD:
    9281             :         case HEAPTUPLE_DELETE_IN_PROGRESS:
    9282         704 :             if (visible)
    9283         562 :                 xid = HeapTupleHeaderGetUpdateXid(tuple->t_data);
    9284             :             else
    9285         142 :                 xid = HeapTupleHeaderGetXmin(tuple->t_data);
    9286             : 
    9287         704 :             if (TransactionIdPrecedes(xid, TransactionXmin))
    9288             :             {
    9289             :                 /* This is like the HEAPTUPLE_DEAD case */
    9290             :                 Assert(!visible);
    9291         124 :                 return;
    9292             :             }
    9293         580 :             break;
    9294         652 :         case HEAPTUPLE_INSERT_IN_PROGRESS:
    9295         652 :             xid = HeapTupleHeaderGetXmin(tuple->t_data);
    9296         652 :             break;
    9297         248 :         case HEAPTUPLE_DEAD:
    9298             :             Assert(!visible);
    9299         248 :             return;
    9300           0 :         default:
    9301             : 
    9302             :             /*
    9303             :              * The only way to get to this default clause is if a new value is
    9304             :              * added to the enum type without adding it to this switch
    9305             :              * statement.  That's a bug, so elog.
    9306             :              */
    9307           0 :             elog(ERROR, "unrecognized return value from HeapTupleSatisfiesVacuum: %u", htsvResult);
    9308             : 
    9309             :             /*
    9310             :              * In spite of having all enum values covered and calling elog on
    9311             :              * this default, some compilers think this is a code path which
    9312             :              * allows xid to be used below without initialization. Silence
    9313             :              * that warning.
    9314             :              */
    9315             :             xid = InvalidTransactionId;
    9316             :     }
    9317             : 
    9318             :     Assert(TransactionIdIsValid(xid));
    9319             :     Assert(TransactionIdFollowsOrEquals(xid, TransactionXmin));
    9320             : 
    9321             :     /*
    9322             :      * Find top level xid.  Bail out if xid is too early to be a conflict, or
    9323             :      * if it's our own xid.
    9324             :      */
    9325        1258 :     if (TransactionIdEquals(xid, GetTopTransactionIdIfAny()))
    9326         124 :         return;
    9327        1134 :     xid = SubTransGetTopmostTransaction(xid);
    9328        1134 :     if (TransactionIdPrecedes(xid, TransactionXmin))
    9329           0 :         return;
    9330             : 
    9331        1134 :     CheckForSerializableConflictOut(relation, xid, snapshot);
    9332             : }

Generated by: LCOV version 1.14