LCOV - code coverage report
Current view: top level - src/backend/utils/activity - pgstat_shmem.c (source / functions) Hit Total Coverage
Test: PostgreSQL 16devel Lines: 280 293 95.6 %
Date: 2022-08-17 04:10:37 Functions: 33 33 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /* -------------------------------------------------------------------------
       2             :  *
       3             :  * pgstat_shmem.c
       4             :  *    Storage of stats entries in shared memory
       5             :  *
       6             :  * Copyright (c) 2001-2022, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/utils/activity/pgstat_shmem.c
      10             :  * -------------------------------------------------------------------------
      11             :  */
      12             : 
      13             : #include "postgres.h"
      14             : 
      15             : #include "pgstat.h"
      16             : #include "storage/shmem.h"
      17             : #include "utils/memutils.h"
      18             : #include "utils/pgstat_internal.h"
      19             : 
      20             : 
      21             : #define PGSTAT_ENTRY_REF_HASH_SIZE  128
      22             : 
      23             : /* hash table entry for finding the PgStat_EntryRef for a key */
      24             : typedef struct PgStat_EntryRefHashEntry
      25             : {
      26             :     PgStat_HashKey key;         /* hash key */
      27             :     char        status;         /* for simplehash use */
      28             :     PgStat_EntryRef *entry_ref;
      29             : } PgStat_EntryRefHashEntry;
      30             : 
      31             : 
      32             : /* for references to shared statistics entries */
      33             : #define SH_PREFIX pgstat_entry_ref_hash
      34             : #define SH_ELEMENT_TYPE PgStat_EntryRefHashEntry
      35             : #define SH_KEY_TYPE PgStat_HashKey
      36             : #define SH_KEY key
      37             : #define SH_HASH_KEY(tb, key) \
      38             :     pgstat_hash_hash_key(&key, sizeof(PgStat_HashKey), NULL)
      39             : #define SH_EQUAL(tb, a, b) \
      40             :     pgstat_cmp_hash_key(&a, &b, sizeof(PgStat_HashKey), NULL) == 0
      41             : #define SH_SCOPE static inline
      42             : #define SH_DEFINE
      43             : #define SH_DECLARE
      44             : #include "lib/simplehash.h"
      45             : 
      46             : 
      47             : static void pgstat_drop_database_and_contents(Oid dboid);
      48             : 
      49             : static void pgstat_free_entry(PgStatShared_HashEntry *shent, dshash_seq_status *hstat);
      50             : 
      51             : static void pgstat_release_entry_ref(PgStat_HashKey key, PgStat_EntryRef *entry_ref, bool discard_pending);
      52             : static bool pgstat_need_entry_refs_gc(void);
      53             : static void pgstat_gc_entry_refs(void);
      54             : static void pgstat_release_all_entry_refs(bool discard_pending);
      55             : typedef bool (*ReleaseMatchCB) (PgStat_EntryRefHashEntry *, Datum data);
      56             : static void pgstat_release_matching_entry_refs(bool discard_pending, ReleaseMatchCB match, Datum match_data);
      57             : 
      58             : static void pgstat_setup_memcxt(void);
      59             : 
      60             : 
      61             : /* parameter for the shared hash */
      62             : static const dshash_parameters dsh_params = {
      63             :     sizeof(PgStat_HashKey),
      64             :     sizeof(PgStatShared_HashEntry),
      65             :     pgstat_cmp_hash_key,
      66             :     pgstat_hash_hash_key,
      67             :     LWTRANCHE_PGSTATS_HASH
      68             : };
      69             : 
      70             : 
      71             : /*
      72             :  * Backend local references to shared stats entries. If there are pending
      73             :  * updates to a stats entry, the PgStat_EntryRef is added to the pgStatPending
      74             :  * list.
      75             :  *
      76             :  * When a stats entry is dropped each backend needs to release its reference
      77             :  * to it before the memory can be released. To trigger that
      78             :  * pgStatLocal.shmem->gc_request_count is incremented - which each backend
      79             :  * compares to their copy of pgStatSharedRefAge on a regular basis.
      80             :  */
      81             : static pgstat_entry_ref_hash_hash *pgStatEntryRefHash = NULL;
      82             : static int  pgStatSharedRefAge = 0; /* cache age of pgStatShmLookupCache */
      83             : 
      84             : /*
      85             :  * Memory contexts containing the pgStatEntryRefHash table and the
      86             :  * pgStatSharedRef entries respectively. Kept separate to make it easier to
      87             :  * track / attribute memory usage.
      88             :  */
      89             : static MemoryContext pgStatSharedRefContext = NULL;
      90             : static MemoryContext pgStatEntryRefHashContext = NULL;
      91             : 
      92             : 
      93             : /* ------------------------------------------------------------
      94             :  * Public functions called from postmaster follow
      95             :  * ------------------------------------------------------------
      96             :  */
      97             : 
      98             : /*
      99             :  * The size of the shared memory allocation for stats stored in the shared
     100             :  * stats hash table. This allocation will be done as part of the main shared
     101             :  * memory, rather than dynamic shared memory, allowing it to be initialized in
     102             :  * postmaster.
     103             :  */
     104             : static Size
     105       18308 : pgstat_dsa_init_size(void)
     106             : {
     107             :     Size        sz;
     108             : 
     109             :     /*
     110             :      * The dshash header / initial buckets array needs to fit into "plain"
     111             :      * shared memory, but it's beneficial to not need dsm segments
     112             :      * immediately. A size of 256kB seems works well and is not
     113             :      * disproportional compared to other constant sized shared memory
     114             :      * allocations. NB: To avoid DSMs further, the user can configure
     115             :      * min_dynamic_shared_memory.
     116             :      */
     117       18308 :     sz = 256 * 1024;
     118             :     Assert(dsa_minimum_size() <= sz);
     119       18308 :     return MAXALIGN(sz);
     120             : }
     121             : 
     122             : /*
     123             :  * Compute shared memory space needed for cumulative statistics
     124             :  */
     125             : Size
     126        8324 : StatsShmemSize(void)
     127             : {
     128             :     Size        sz;
     129             : 
     130        8324 :     sz = MAXALIGN(sizeof(PgStat_ShmemControl));
     131        8324 :     sz = add_size(sz, pgstat_dsa_init_size());
     132             : 
     133        8324 :     return sz;
     134             : }
     135             : 
     136             : /*
     137             :  * Initialize cumulative statistics system during startup
     138             :  */
     139             : void
     140        3328 : StatsShmemInit(void)
     141             : {
     142             :     bool        found;
     143             :     Size        sz;
     144             : 
     145        3328 :     sz = StatsShmemSize();
     146        3328 :     pgStatLocal.shmem = (PgStat_ShmemControl *)
     147        3328 :         ShmemInitStruct("Shared Memory Stats", sz, &found);
     148             : 
     149        3328 :     if (!IsUnderPostmaster)
     150             :     {
     151             :         dsa_area   *dsa;
     152             :         dshash_table *dsh;
     153        3328 :         PgStat_ShmemControl *ctl = pgStatLocal.shmem;
     154        3328 :         char       *p = (char *) ctl;
     155             : 
     156             :         Assert(!found);
     157             : 
     158             :         /* the allocation of pgStatLocal.shmem itself */
     159        3328 :         p += MAXALIGN(sizeof(PgStat_ShmemControl));
     160             : 
     161             :         /*
     162             :          * Create a small dsa allocation in plain shared memory. This is
     163             :          * required because postmaster cannot use dsm segments. It also
     164             :          * provides a small efficiency win.
     165             :          */
     166        3328 :         ctl->raw_dsa_area = p;
     167        3328 :         p += MAXALIGN(pgstat_dsa_init_size());
     168        3328 :         dsa = dsa_create_in_place(ctl->raw_dsa_area,
     169             :                                   pgstat_dsa_init_size(),
     170             :                                   LWTRANCHE_PGSTATS_DSA, 0);
     171        3328 :         dsa_pin(dsa);
     172             : 
     173             :         /*
     174             :          * To ensure dshash is created in "plain" shared memory, temporarily
     175             :          * limit size of dsa to the initial size of the dsa.
     176             :          */
     177        3328 :         dsa_set_size_limit(dsa, pgstat_dsa_init_size());
     178             : 
     179             :         /*
     180             :          * With the limit in place, create the dshash table. XXX: It'd be nice
     181             :          * if there were dshash_create_in_place().
     182             :          */
     183        3328 :         dsh = dshash_create(dsa, &dsh_params, 0);
     184        3328 :         ctl->hash_handle = dshash_get_hash_table_handle(dsh);
     185             : 
     186             :         /* lift limit set above */
     187        3328 :         dsa_set_size_limit(dsa, -1);
     188             : 
     189             :         /*
     190             :          * Postmaster will never access these again, thus free the local
     191             :          * dsa/dshash references.
     192             :          */
     193        3328 :         dshash_detach(dsh);
     194        3328 :         dsa_detach(dsa);
     195             : 
     196        3328 :         pg_atomic_init_u64(&ctl->gc_request_count, 1);
     197             : 
     198             : 
     199             :         /* initialize fixed-numbered stats */
     200        3328 :         LWLockInitialize(&ctl->archiver.lock, LWTRANCHE_PGSTATS_DATA);
     201        3328 :         LWLockInitialize(&ctl->bgwriter.lock, LWTRANCHE_PGSTATS_DATA);
     202        3328 :         LWLockInitialize(&ctl->checkpointer.lock, LWTRANCHE_PGSTATS_DATA);
     203        3328 :         LWLockInitialize(&ctl->slru.lock, LWTRANCHE_PGSTATS_DATA);
     204        3328 :         LWLockInitialize(&ctl->wal.lock, LWTRANCHE_PGSTATS_DATA);
     205             :     }
     206             :     else
     207             :     {
     208             :         Assert(found);
     209             :     }
     210        3328 : }
     211             : 
     212             : void
     213       24048 : pgstat_attach_shmem(void)
     214             : {
     215             :     MemoryContext oldcontext;
     216             : 
     217             :     Assert(pgStatLocal.dsa == NULL);
     218             : 
     219             :     /* stats shared memory persists for the backend lifetime */
     220       24048 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     221             : 
     222       24048 :     pgStatLocal.dsa = dsa_attach_in_place(pgStatLocal.shmem->raw_dsa_area,
     223             :                                           NULL);
     224       24048 :     dsa_pin_mapping(pgStatLocal.dsa);
     225             : 
     226       48096 :     pgStatLocal.shared_hash = dshash_attach(pgStatLocal.dsa, &dsh_params,
     227       24048 :                                             pgStatLocal.shmem->hash_handle, 0);
     228             : 
     229       24048 :     MemoryContextSwitchTo(oldcontext);
     230       24048 : }
     231             : 
     232             : void
     233       24048 : pgstat_detach_shmem(void)
     234             : {
     235             :     Assert(pgStatLocal.dsa);
     236             : 
     237             :     /* we shouldn't leave references to shared stats */
     238       24048 :     pgstat_release_all_entry_refs(false);
     239             : 
     240       24048 :     dshash_detach(pgStatLocal.shared_hash);
     241       24048 :     pgStatLocal.shared_hash = NULL;
     242             : 
     243       24048 :     dsa_detach(pgStatLocal.dsa);
     244       24048 :     pgStatLocal.dsa = NULL;
     245       24048 : }
     246             : 
     247             : 
     248             : /* ------------------------------------------------------------
     249             :  * Maintenance of shared memory stats entries
     250             :  * ------------------------------------------------------------
     251             :  */
     252             : 
     253             : PgStatShared_Common *
     254      537166 : pgstat_init_entry(PgStat_Kind kind,
     255             :                   PgStatShared_HashEntry *shhashent)
     256             : {
     257             :     /* Create new stats entry. */
     258             :     dsa_pointer chunk;
     259             :     PgStatShared_Common *shheader;
     260             : 
     261             :     /*
     262             :      * Initialize refcount to 1, marking it as valid / not dropped. The entry
     263             :      * can't be freed before the initialization because it can't be found as
     264             :      * long as we hold the dshash partition lock. Caller needs to increase
     265             :      * further if a longer lived reference is needed.
     266             :      */
     267      537166 :     pg_atomic_init_u32(&shhashent->refcount, 1);
     268      537166 :     shhashent->dropped = false;
     269             : 
     270      537166 :     chunk = dsa_allocate0(pgStatLocal.dsa, pgstat_get_kind_info(kind)->shared_size);
     271      537166 :     shheader = dsa_get_address(pgStatLocal.dsa, chunk);
     272      537166 :     shheader->magic = 0xdeadbeef;
     273             : 
     274             :     /* Link the new entry from the hash entry. */
     275      537166 :     shhashent->body = chunk;
     276             : 
     277      537166 :     LWLockInitialize(&shheader->lock, LWTRANCHE_PGSTATS_DATA);
     278             : 
     279      537166 :     return shheader;
     280             : }
     281             : 
     282             : static PgStatShared_Common *
     283          50 : pgstat_reinit_entry(PgStat_Kind kind, PgStatShared_HashEntry *shhashent)
     284             : {
     285             :     PgStatShared_Common *shheader;
     286             : 
     287          50 :     shheader = dsa_get_address(pgStatLocal.dsa, shhashent->body);
     288             : 
     289             :     /* mark as not dropped anymore */
     290          50 :     pg_atomic_fetch_add_u32(&shhashent->refcount, 1);
     291          50 :     shhashent->dropped = false;
     292             : 
     293             :     /* reinitialize content */
     294             :     Assert(shheader->magic == 0xdeadbeef);
     295          50 :     memset(pgstat_get_entry_data(kind, shheader), 0,
     296             :            pgstat_get_entry_len(kind));
     297             : 
     298          50 :     return shheader;
     299             : }
     300             : 
     301             : static void
     302     2899988 : pgstat_setup_shared_refs(void)
     303             : {
     304     2899988 :     if (likely(pgStatEntryRefHash != NULL))
     305     2877486 :         return;
     306             : 
     307       22502 :     pgStatEntryRefHash =
     308       22502 :         pgstat_entry_ref_hash_create(pgStatEntryRefHashContext,
     309             :                                      PGSTAT_ENTRY_REF_HASH_SIZE, NULL);
     310       22502 :     pgStatSharedRefAge = pg_atomic_read_u64(&pgStatLocal.shmem->gc_request_count);
     311             :     Assert(pgStatSharedRefAge != 0);
     312             : }
     313             : 
     314             : /*
     315             :  * Helper function for pgstat_get_entry_ref().
     316             :  */
     317             : static void
     318     1068328 : pgstat_acquire_entry_ref(PgStat_EntryRef *entry_ref,
     319             :                          PgStatShared_HashEntry *shhashent,
     320             :                          PgStatShared_Common *shheader)
     321             : {
     322             :     Assert(shheader->magic == 0xdeadbeef);
     323             :     Assert(pg_atomic_read_u32(&shhashent->refcount) > 0);
     324             : 
     325     1068328 :     pg_atomic_fetch_add_u32(&shhashent->refcount, 1);
     326             : 
     327     1068328 :     dshash_release_lock(pgStatLocal.shared_hash, shhashent);
     328             : 
     329     1068328 :     entry_ref->shared_stats = shheader;
     330     1068328 :     entry_ref->shared_entry = shhashent;
     331     1068328 : }
     332             : 
     333             : /*
     334             :  * Helper function for pgstat_get_entry_ref().
     335             :  */
     336             : static bool
     337     2899988 : pgstat_get_entry_ref_cached(PgStat_HashKey key, PgStat_EntryRef **entry_ref_p)
     338             : {
     339             :     bool        found;
     340             :     PgStat_EntryRefHashEntry *cache_entry;
     341             : 
     342             :     /*
     343             :      * We immediately insert a cache entry, because it avoids 1) multiple
     344             :      * hashtable lookups in case of a cache miss 2) having to deal with
     345             :      * out-of-memory errors after incrementing PgStatShared_Common->refcount.
     346             :      */
     347             : 
     348     2899988 :     cache_entry = pgstat_entry_ref_hash_insert(pgStatEntryRefHash, key, &found);
     349             : 
     350     2899988 :     if (!found || !cache_entry->entry_ref)
     351     1275810 :     {
     352             :         PgStat_EntryRef *entry_ref;
     353             : 
     354     1275810 :         cache_entry->entry_ref = entry_ref =
     355     1275810 :             MemoryContextAlloc(pgStatSharedRefContext,
     356             :                                sizeof(PgStat_EntryRef));
     357     1275810 :         entry_ref->shared_stats = NULL;
     358     1275810 :         entry_ref->shared_entry = NULL;
     359     1275810 :         entry_ref->pending = NULL;
     360             : 
     361     1275810 :         found = false;
     362             :     }
     363     1624178 :     else if (cache_entry->entry_ref->shared_stats == NULL)
     364             :     {
     365             :         Assert(cache_entry->entry_ref->pending == NULL);
     366           0 :         found = false;
     367             :     }
     368             :     else
     369             :     {
     370             :         PgStat_EntryRef *entry_ref PG_USED_FOR_ASSERTS_ONLY;
     371             : 
     372     1624178 :         entry_ref = cache_entry->entry_ref;
     373             :         Assert(entry_ref->shared_entry != NULL);
     374             :         Assert(entry_ref->shared_stats != NULL);
     375             : 
     376             :         Assert(entry_ref->shared_stats->magic == 0xdeadbeef);
     377             :         /* should have at least our reference */
     378             :         Assert(pg_atomic_read_u32(&entry_ref->shared_entry->refcount) > 0);
     379             :     }
     380             : 
     381     2899988 :     *entry_ref_p = cache_entry->entry_ref;
     382     2899988 :     return found;
     383             : }
     384             : 
     385             : /*
     386             :  * Get a shared stats reference. If create is true, the shared stats object is
     387             :  * created if it does not exist.
     388             :  *
     389             :  * When create is true, and created_entry is non-NULL, it'll be set to true
     390             :  * if the entry is newly created, false otherwise.
     391             :  */
     392             : PgStat_EntryRef *
     393     2899988 : pgstat_get_entry_ref(PgStat_Kind kind, Oid dboid, Oid objoid, bool create,
     394             :                      bool *created_entry)
     395             : {
     396     2899988 :     PgStat_HashKey key = {.kind = kind,.dboid = dboid,.objoid = objoid};
     397             :     PgStatShared_HashEntry *shhashent;
     398     2899988 :     PgStatShared_Common *shheader = NULL;
     399             :     PgStat_EntryRef *entry_ref;
     400             : 
     401             :     /*
     402             :      * passing in created_entry only makes sense if we possibly could create
     403             :      * entry.
     404             :      */
     405             :     AssertArg(create || created_entry == NULL);
     406             :     pgstat_assert_is_up();
     407             :     Assert(pgStatLocal.shared_hash != NULL);
     408             :     Assert(!pgStatLocal.shmem->is_shutdown);
     409             : 
     410     2899988 :     pgstat_setup_memcxt();
     411     2899988 :     pgstat_setup_shared_refs();
     412             : 
     413     2899988 :     if (created_entry != NULL)
     414         208 :         *created_entry = false;
     415             : 
     416             :     /*
     417             :      * Check if other backends dropped stats that could not be deleted because
     418             :      * somebody held references to it. If so, check this backend's references.
     419             :      * This is not expected to happen often. The location of the check is a
     420             :      * bit random, but this is a relatively frequently called path, so better
     421             :      * than most.
     422             :      */
     423     2899988 :     if (pgstat_need_entry_refs_gc())
     424       10124 :         pgstat_gc_entry_refs();
     425             : 
     426             :     /*
     427             :      * First check the lookup cache hashtable in local memory. If we find a
     428             :      * match here we can avoid taking locks / causing contention.
     429             :      */
     430     2899988 :     if (pgstat_get_entry_ref_cached(key, &entry_ref))
     431     1624178 :         return entry_ref;
     432             : 
     433             :     Assert(entry_ref != NULL);
     434             : 
     435             :     /*
     436             :      * Do a lookup in the hash table first - it's quite likely that the entry
     437             :      * already exists, and that way we only need a shared lock.
     438             :      */
     439     1275810 :     shhashent = dshash_find(pgStatLocal.shared_hash, &key, false);
     440             : 
     441     1275810 :     if (create && !shhashent)
     442             :     {
     443             :         bool        shfound;
     444             : 
     445             :         /*
     446             :          * It's possible that somebody created the entry since the above
     447             :          * lookup. If so, fall through to the same path as if we'd have if it
     448             :          * already had been created before the dshash_find() calls.
     449             :          */
     450      261352 :         shhashent = dshash_find_or_insert(pgStatLocal.shared_hash, &key, &shfound);
     451      261352 :         if (!shfound)
     452             :         {
     453      261352 :             shheader = pgstat_init_entry(kind, shhashent);
     454      261352 :             pgstat_acquire_entry_ref(entry_ref, shhashent, shheader);
     455             : 
     456      261352 :             if (created_entry != NULL)
     457          90 :                 *created_entry = true;
     458             : 
     459      261352 :             return entry_ref;
     460             :         }
     461             :     }
     462             : 
     463     1014458 :     if (!shhashent)
     464             :     {
     465             :         /*
     466             :          * If we're not creating, delete the reference again. In all
     467             :          * likelihood it's just a stats lookup - no point wasting memory for a
     468             :          * shared ref to nothing...
     469             :          */
     470      207408 :         pgstat_release_entry_ref(key, entry_ref, false);
     471             : 
     472      207408 :         return NULL;
     473             :     }
     474             :     else
     475             :     {
     476             :         /*
     477             :          * Can get here either because dshash_find() found a match, or if
     478             :          * dshash_find_or_insert() found a concurrently inserted entry.
     479             :          */
     480             : 
     481      807050 :         if (shhashent->dropped && create)
     482             :         {
     483             :             /*
     484             :              * There are legitimate cases where the old stats entry might not
     485             :              * yet have been dropped by the time it's reused. The most obvious
     486             :              * case are replication slot stats, where a new slot can be
     487             :              * created with the same index just after dropping. But oid
     488             :              * wraparound can lead to other cases as well. We just reset the
     489             :              * stats to their plain state.
     490             :              */
     491          50 :             shheader = pgstat_reinit_entry(kind, shhashent);
     492          50 :             pgstat_acquire_entry_ref(entry_ref, shhashent, shheader);
     493             : 
     494          50 :             if (created_entry != NULL)
     495           0 :                 *created_entry = true;
     496             : 
     497          50 :             return entry_ref;
     498             :         }
     499      807000 :         else if (shhashent->dropped)
     500             :         {
     501          74 :             dshash_release_lock(pgStatLocal.shared_hash, shhashent);
     502          74 :             pgstat_release_entry_ref(key, entry_ref, false);
     503             : 
     504          74 :             return NULL;
     505             :         }
     506             :         else
     507             :         {
     508      806926 :             shheader = dsa_get_address(pgStatLocal.dsa, shhashent->body);
     509      806926 :             pgstat_acquire_entry_ref(entry_ref, shhashent, shheader);
     510             : 
     511      806926 :             return entry_ref;
     512             :         }
     513             :     }
     514             : }
     515             : 
     516             : static void
     517     1275810 : pgstat_release_entry_ref(PgStat_HashKey key, PgStat_EntryRef *entry_ref,
     518             :                          bool discard_pending)
     519             : {
     520     1275810 :     if (entry_ref && entry_ref->pending)
     521             :     {
     522       34288 :         if (discard_pending)
     523       34288 :             pgstat_delete_pending_entry(entry_ref);
     524             :         else
     525           0 :             elog(ERROR, "releasing ref with pending data");
     526             :     }
     527             : 
     528     1275810 :     if (entry_ref && entry_ref->shared_stats)
     529             :     {
     530             :         Assert(entry_ref->shared_stats->magic == 0xdeadbeef);
     531             :         Assert(entry_ref->pending == NULL);
     532             : 
     533             :         /*
     534             :          * This can't race with another backend looking up the stats entry and
     535             :          * increasing the refcount because it is not "legal" to create
     536             :          * additional references to dropped entries.
     537             :          */
     538     1068328 :         if (pg_atomic_fetch_sub_u32(&entry_ref->shared_entry->refcount, 1) == 1)
     539             :         {
     540             :             PgStatShared_HashEntry *shent;
     541             : 
     542             :             /*
     543             :              * We're the last referrer to this entry, try to drop the shared
     544             :              * entry.
     545             :              */
     546             : 
     547             :             /* only dropped entries can reach a 0 refcount */
     548             :             Assert(entry_ref->shared_entry->dropped);
     549             : 
     550        5410 :             shent = dshash_find(pgStatLocal.shared_hash,
     551        5410 :                                 &entry_ref->shared_entry->key,
     552             :                                 true);
     553        5410 :             if (!shent)
     554           0 :                 elog(ERROR, "could not find just referenced shared stats entry");
     555             : 
     556             :             Assert(pg_atomic_read_u32(&entry_ref->shared_entry->refcount) == 0);
     557             :             Assert(entry_ref->shared_entry == shent);
     558             : 
     559        5410 :             pgstat_free_entry(shent, NULL);
     560             :         }
     561             :     }
     562             : 
     563     1275810 :     if (!pgstat_entry_ref_hash_delete(pgStatEntryRefHash, key))
     564           0 :         elog(ERROR, "entry ref vanished before deletion");
     565             : 
     566     1275810 :     if (entry_ref)
     567     1275810 :         pfree(entry_ref);
     568     1275810 : }
     569             : 
     570             : bool
     571     1361468 : pgstat_lock_entry(PgStat_EntryRef *entry_ref, bool nowait)
     572             : {
     573     1361468 :     LWLock     *lock = &entry_ref->shared_stats->lock;
     574             : 
     575     1361468 :     if (nowait)
     576      422086 :         return LWLockConditionalAcquire(lock, LW_EXCLUSIVE);
     577             : 
     578      939382 :     LWLockAcquire(lock, LW_EXCLUSIVE);
     579      939382 :     return true;
     580             : }
     581             : 
     582             : void
     583     1361446 : pgstat_unlock_entry(PgStat_EntryRef *entry_ref)
     584             : {
     585     1361446 :     LWLockRelease(&entry_ref->shared_stats->lock);
     586     1361446 : }
     587             : 
     588             : /*
     589             :  * Helper function to fetch and lock shared stats.
     590             :  */
     591             : PgStat_EntryRef *
     592      122392 : pgstat_get_entry_ref_locked(PgStat_Kind kind, Oid dboid, Oid objoid,
     593             :                             bool nowait)
     594             : {
     595             :     PgStat_EntryRef *entry_ref;
     596             : 
     597             :     /* find shared table stats entry corresponding to the local entry */
     598      122392 :     entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, true, NULL);
     599             : 
     600             :     /* lock the shared entry to protect the content, skip if failed */
     601      122392 :     if (!pgstat_lock_entry(entry_ref, nowait))
     602           0 :         return NULL;
     603             : 
     604      122392 :     return entry_ref;
     605             : }
     606             : 
     607             : void
     608        3204 : pgstat_request_entry_refs_gc(void)
     609             : {
     610        3204 :     pg_atomic_fetch_add_u64(&pgStatLocal.shmem->gc_request_count, 1);
     611        3204 : }
     612             : 
     613             : static bool
     614     2899988 : pgstat_need_entry_refs_gc(void)
     615             : {
     616             :     uint64      curage;
     617             : 
     618     2899988 :     if (!pgStatEntryRefHash)
     619           0 :         return false;
     620             : 
     621             :     /* should have been initialized when creating pgStatEntryRefHash */
     622             :     Assert(pgStatSharedRefAge != 0);
     623             : 
     624     2899988 :     curage = pg_atomic_read_u64(&pgStatLocal.shmem->gc_request_count);
     625             : 
     626     2899988 :     return pgStatSharedRefAge != curage;
     627             : }
     628             : 
     629             : static void
     630       10124 : pgstat_gc_entry_refs(void)
     631             : {
     632             :     pgstat_entry_ref_hash_iterator i;
     633             :     PgStat_EntryRefHashEntry *ent;
     634             :     uint64      curage;
     635             : 
     636       10124 :     curage = pg_atomic_read_u64(&pgStatLocal.shmem->gc_request_count);
     637             :     Assert(curage != 0);
     638             : 
     639             :     /*
     640             :      * Some entries have been dropped. Invalidate cache pointer to them.
     641             :      */
     642       10124 :     pgstat_entry_ref_hash_start_iterate(pgStatEntryRefHash, &i);
     643     1146044 :     while ((ent = pgstat_entry_ref_hash_iterate(pgStatEntryRefHash, &i)) != NULL)
     644             :     {
     645     1135920 :         PgStat_EntryRef *entry_ref = ent->entry_ref;
     646             : 
     647             :         Assert(!entry_ref->shared_stats ||
     648             :                entry_ref->shared_stats->magic == 0xdeadbeef);
     649             : 
     650     1135920 :         if (!entry_ref->shared_entry->dropped)
     651      960916 :             continue;
     652             : 
     653             :         /* cannot gc shared ref that has pending data */
     654      175004 :         if (entry_ref->pending != NULL)
     655      170742 :             continue;
     656             : 
     657        4262 :         pgstat_release_entry_ref(ent->key, entry_ref, false);
     658             :     }
     659             : 
     660       10124 :     pgStatSharedRefAge = curage;
     661       10124 : }
     662             : 
     663             : static void
     664       22528 : pgstat_release_matching_entry_refs(bool discard_pending, ReleaseMatchCB match,
     665             :                                    Datum match_data)
     666             : {
     667             :     pgstat_entry_ref_hash_iterator i;
     668             :     PgStat_EntryRefHashEntry *ent;
     669             : 
     670       22528 :     if (pgStatEntryRefHash == NULL)
     671           8 :         return;
     672             : 
     673       22520 :     pgstat_entry_ref_hash_start_iterate(pgStatEntryRefHash, &i);
     674             : 
     675     1052326 :     while ((ent = pgstat_entry_ref_hash_iterate(pgStatEntryRefHash, &i))
     676             :            != NULL)
     677             :     {
     678             :         Assert(ent->entry_ref != NULL);
     679             : 
     680     1029806 :         if (match && !match(ent, match_data))
     681         542 :             continue;
     682             : 
     683     1029264 :         pgstat_release_entry_ref(ent->key, ent->entry_ref, discard_pending);
     684             :     }
     685             : }
     686             : 
     687             : /*
     688             :  * Release all local references to shared stats entries.
     689             :  *
     690             :  * When a process exits it cannot do so while still holding references onto
     691             :  * stats entries, otherwise the shared stats entries could never be freed.
     692             :  */
     693             : static void
     694       24048 : pgstat_release_all_entry_refs(bool discard_pending)
     695             : {
     696       24048 :     if (pgStatEntryRefHash == NULL)
     697        1546 :         return;
     698             : 
     699       22502 :     pgstat_release_matching_entry_refs(discard_pending, NULL, 0);
     700             :     Assert(pgStatEntryRefHash->members == 0);
     701       22502 :     pgstat_entry_ref_hash_destroy(pgStatEntryRefHash);
     702       22502 :     pgStatEntryRefHash = NULL;
     703             : }
     704             : 
     705             : static bool
     706         542 : match_db(PgStat_EntryRefHashEntry *ent, Datum match_data)
     707             : {
     708         542 :     Oid         dboid = DatumGetObjectId(match_data);
     709             : 
     710         542 :     return ent->key.dboid == dboid;
     711             : }
     712             : 
     713             : static void
     714          26 : pgstat_release_db_entry_refs(Oid dboid)
     715             : {
     716          26 :     pgstat_release_matching_entry_refs( /* discard pending = */ true,
     717             :                                        match_db,
     718             :                                        ObjectIdGetDatum(dboid));
     719          26 : }
     720             : 
     721             : 
     722             : /* ------------------------------------------------------------
     723             :  * Dropping and resetting of stats entries
     724             :  * ------------------------------------------------------------
     725             :  */
     726             : 
     727             : static void
     728       36808 : pgstat_free_entry(PgStatShared_HashEntry *shent, dshash_seq_status *hstat)
     729             : {
     730             :     dsa_pointer pdsa;
     731             : 
     732             :     /*
     733             :      * Fetch dsa pointer before deleting entry - that way we can free the
     734             :      * memory after releasing the lock.
     735             :      */
     736       36808 :     pdsa = shent->body;
     737             : 
     738       36808 :     if (!hstat)
     739       34862 :         dshash_delete_entry(pgStatLocal.shared_hash, shent);
     740             :     else
     741        1946 :         dshash_delete_current(hstat);
     742             : 
     743       36808 :     dsa_free(pgStatLocal.dsa, pdsa);
     744       36808 : }
     745             : 
     746             : /*
     747             :  * Helper for both pgstat_drop_database_and_contents() and
     748             :  * pgstat_drop_entry(). If hstat is non-null delete the shared entry using
     749             :  * dshash_delete_current(), otherwise use dshash_delete_entry(). In either
     750             :  * case the entry needs to be already locked.
     751             :  */
     752             : static bool
     753       36860 : pgstat_drop_entry_internal(PgStatShared_HashEntry *shent,
     754             :                            dshash_seq_status *hstat)
     755             : {
     756             :     Assert(shent->body != InvalidDsaPointer);
     757             : 
     758             :     /* should already have released local reference */
     759       36860 :     if (pgStatEntryRefHash)
     760             :         Assert(!pgstat_entry_ref_hash_lookup(pgStatEntryRefHash, shent->key));
     761             : 
     762             :     /*
     763             :      * Signal that the entry is dropped - this will eventually cause other
     764             :      * backends to release their references.
     765             :      */
     766       36860 :     if (shent->dropped)
     767           0 :         elog(ERROR, "can only drop stats once");
     768       36860 :     shent->dropped = true;
     769             : 
     770             :     /* release refcount marking entry as not dropped */
     771       36860 :     if (pg_atomic_sub_fetch_u32(&shent->refcount, 1) == 0)
     772             :     {
     773       31398 :         pgstat_free_entry(shent, hstat);
     774       31398 :         return true;
     775             :     }
     776             :     else
     777             :     {
     778        5462 :         if (!hstat)
     779        5462 :             dshash_release_lock(pgStatLocal.shared_hash, shent);
     780        5462 :         return false;
     781             :     }
     782             : }
     783             : 
     784             : /*
     785             :  * Drop stats for the database and all the objects inside that database.
     786             :  */
     787             : static void
     788          26 : pgstat_drop_database_and_contents(Oid dboid)
     789             : {
     790             :     dshash_seq_status hstat;
     791             :     PgStatShared_HashEntry *p;
     792          26 :     uint64      not_freed_count = 0;
     793             : 
     794             :     Assert(OidIsValid(dboid));
     795             : 
     796             :     Assert(pgStatLocal.shared_hash != NULL);
     797             : 
     798             :     /*
     799             :      * This backend might very well be the only backend holding a reference to
     800             :      * about-to-be-dropped entries. Ensure that we're not preventing it from
     801             :      * being cleaned up till later.
     802             :      *
     803             :      * Doing this separately from the dshash iteration below avoids having to
     804             :      * do so while holding a partition lock on the shared hashtable.
     805             :      */
     806          26 :     pgstat_release_db_entry_refs(dboid);
     807             : 
     808             :     /* some of the dshash entries are to be removed, take exclusive lock. */
     809          26 :     dshash_seq_init(&hstat, pgStatLocal.shared_hash, true);
     810        9034 :     while ((p = dshash_seq_next(&hstat)) != NULL)
     811             :     {
     812        9008 :         if (p->dropped)
     813           2 :             continue;
     814             : 
     815        9006 :         if (p->key.dboid != dboid)
     816        7172 :             continue;
     817             : 
     818        1834 :         if (!pgstat_drop_entry_internal(p, &hstat))
     819             :         {
     820             :             /*
     821             :              * Even statistics for a dropped database might currently be
     822             :              * accessed (consider e.g. database stats for pg_stat_database).
     823             :              */
     824           0 :             not_freed_count++;
     825             :         }
     826             :     }
     827          26 :     dshash_seq_term(&hstat);
     828             : 
     829             :     /*
     830             :      * If some of the stats data could not be freed, signal the reference
     831             :      * holders to run garbage collection of their cached pgStatShmLookupCache.
     832             :      */
     833          26 :     if (not_freed_count > 0)
     834           0 :         pgstat_request_entry_refs_gc();
     835          26 : }
     836             : 
     837             : bool
     838       54558 : pgstat_drop_entry(PgStat_Kind kind, Oid dboid, Oid objoid)
     839             : {
     840       54558 :     PgStat_HashKey key = {.kind = kind,.dboid = dboid,.objoid = objoid};
     841             :     PgStatShared_HashEntry *shent;
     842       54558 :     bool        freed = true;
     843             : 
     844             :     /* delete local reference */
     845       54558 :     if (pgStatEntryRefHash)
     846             :     {
     847             :         PgStat_EntryRefHashEntry *lohashent =
     848       44486 :         pgstat_entry_ref_hash_lookup(pgStatEntryRefHash, key);
     849             : 
     850       44486 :         if (lohashent)
     851       34802 :             pgstat_release_entry_ref(lohashent->key, lohashent->entry_ref,
     852             :                                      true);
     853             :     }
     854             : 
     855             :     /* mark entry in shared hashtable as deleted, drop if possible */
     856       54558 :     shent = dshash_find(pgStatLocal.shared_hash, &key, true);
     857       54558 :     if (shent)
     858             :     {
     859       34914 :         freed = pgstat_drop_entry_internal(shent, NULL);
     860             : 
     861             :         /*
     862             :          * Database stats contain other stats. Drop those as well when
     863             :          * dropping the database. XXX: Perhaps this should be done in a
     864             :          * slightly more principled way? But not obvious what that'd look
     865             :          * like, and so far this is the only case...
     866             :          */
     867       34914 :         if (key.kind == PGSTAT_KIND_DATABASE)
     868          26 :             pgstat_drop_database_and_contents(key.dboid);
     869             :     }
     870             : 
     871       54558 :     return freed;
     872             : }
     873             : 
     874             : void
     875         810 : pgstat_drop_all_entries(void)
     876             : {
     877             :     dshash_seq_status hstat;
     878             :     PgStatShared_HashEntry *ps;
     879         810 :     uint64      not_freed_count = 0;
     880             : 
     881         810 :     dshash_seq_init(&hstat, pgStatLocal.shared_hash, true);
     882         922 :     while ((ps = dshash_seq_next(&hstat)) != NULL)
     883             :     {
     884         112 :         if (ps->dropped)
     885           0 :             continue;
     886             : 
     887         112 :         if (!pgstat_drop_entry_internal(ps, &hstat))
     888           0 :             not_freed_count++;
     889             :     }
     890         810 :     dshash_seq_term(&hstat);
     891             : 
     892         810 :     if (not_freed_count > 0)
     893           0 :         pgstat_request_entry_refs_gc();
     894         810 : }
     895             : 
     896             : static void
     897       24582 : shared_stat_reset_contents(PgStat_Kind kind, PgStatShared_Common *header,
     898             :                            TimestampTz ts)
     899             : {
     900       24582 :     const PgStat_KindInfo *kind_info = pgstat_get_kind_info(kind);
     901             : 
     902       24582 :     memset(pgstat_get_entry_data(kind, header), 0,
     903             :            pgstat_get_entry_len(kind));
     904             : 
     905       24582 :     if (kind_info->reset_timestamp_cb)
     906          42 :         kind_info->reset_timestamp_cb(header, ts);
     907       24582 : }
     908             : 
     909             : /*
     910             :  * Reset one variable-numbered stats entry.
     911             :  */
     912             : void
     913          14 : pgstat_reset_entry(PgStat_Kind kind, Oid dboid, Oid objoid, TimestampTz ts)
     914             : {
     915             :     PgStat_EntryRef *entry_ref;
     916             : 
     917             :     Assert(!pgstat_get_kind_info(kind)->fixed_amount);
     918             : 
     919          14 :     entry_ref = pgstat_get_entry_ref(kind, dboid, objoid, false, NULL);
     920          14 :     if (!entry_ref || entry_ref->shared_entry->dropped)
     921           2 :         return;
     922             : 
     923          12 :     (void) pgstat_lock_entry(entry_ref, false);
     924          12 :     shared_stat_reset_contents(kind, entry_ref->shared_stats, ts);
     925          12 :     pgstat_unlock_entry(entry_ref);
     926             : }
     927             : 
     928             : /*
     929             :  * Scan through the shared hashtable of stats, resetting statistics if
     930             :  * approved by the provided do_reset() function.
     931             :  */
     932             : void
     933          22 : pgstat_reset_matching_entries(bool (*do_reset) (PgStatShared_HashEntry *, Datum),
     934             :                               Datum match_data, TimestampTz ts)
     935             : {
     936             :     dshash_seq_status hstat;
     937             :     PgStatShared_HashEntry *p;
     938             : 
     939             :     /* dshash entry is not modified, take shared lock */
     940          22 :     dshash_seq_init(&hstat, pgStatLocal.shared_hash, false);
     941       31662 :     while ((p = dshash_seq_next(&hstat)) != NULL)
     942             :     {
     943             :         PgStatShared_Common *header;
     944             : 
     945       31640 :         if (p->dropped)
     946           2 :             continue;
     947             : 
     948       31638 :         if (!do_reset(p, match_data))
     949        7068 :             continue;
     950             : 
     951       24570 :         header = dsa_get_address(pgStatLocal.dsa, p->body);
     952             : 
     953       24570 :         LWLockAcquire(&header->lock, LW_EXCLUSIVE);
     954             : 
     955       24570 :         shared_stat_reset_contents(p->key.kind, header, ts);
     956             : 
     957       24570 :         LWLockRelease(&header->lock);
     958             :     }
     959          22 :     dshash_seq_term(&hstat);
     960          22 : }
     961             : 
     962             : static bool
     963        2928 : match_kind(PgStatShared_HashEntry *p, Datum match_data)
     964             : {
     965        2928 :     return p->key.kind == DatumGetInt32(match_data);
     966             : }
     967             : 
     968             : void
     969           8 : pgstat_reset_entries_of_kind(PgStat_Kind kind, TimestampTz ts)
     970             : {
     971           8 :     pgstat_reset_matching_entries(match_kind, Int32GetDatum(kind), ts);
     972           8 : }
     973             : 
     974             : static void
     975     2899988 : pgstat_setup_memcxt(void)
     976             : {
     977     2899988 :     if (unlikely(!pgStatSharedRefContext))
     978       22502 :         pgStatSharedRefContext =
     979       22502 :             AllocSetContextCreate(CacheMemoryContext,
     980             :                                   "PgStat Shared Ref",
     981             :                                   ALLOCSET_SMALL_SIZES);
     982     2899988 :     if (unlikely(!pgStatEntryRefHashContext))
     983       22502 :         pgStatEntryRefHashContext =
     984       22502 :             AllocSetContextCreate(CacheMemoryContext,
     985             :                                   "PgStat Shared Ref Hash",
     986             :                                   ALLOCSET_SMALL_SIZES);
     987     2899988 : }

Generated by: LCOV version 1.14