LCOV - code coverage report
Current view: top level - src/backend/access/gin - gininsert.c (source / functions) Hit Total Coverage
Test: PostgreSQL 13devel Lines: 138 161 85.7 %
Date: 2019-11-13 23:06:49 Functions: 8 9 88.9 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * gininsert.c
       4             :  *    insert routines for the postgres inverted index access method.
       5             :  *
       6             :  *
       7             :  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
       8             :  * Portions Copyright (c) 1994, Regents of the University of California
       9             :  *
      10             :  * IDENTIFICATION
      11             :  *          src/backend/access/gin/gininsert.c
      12             :  *-------------------------------------------------------------------------
      13             :  */
      14             : 
      15             : #include "postgres.h"
      16             : 
      17             : #include "access/gin_private.h"
      18             : #include "access/ginxlog.h"
      19             : #include "access/tableam.h"
      20             : #include "access/xloginsert.h"
      21             : #include "catalog/index.h"
      22             : #include "miscadmin.h"
      23             : #include "storage/bufmgr.h"
      24             : #include "storage/indexfsm.h"
      25             : #include "storage/predicate.h"
      26             : #include "storage/smgr.h"
      27             : #include "utils/memutils.h"
      28             : #include "utils/rel.h"
      29             : 
      30             : typedef struct
      31             : {
      32             :     GinState    ginstate;
      33             :     double      indtuples;
      34             :     GinStatsData buildStats;
      35             :     MemoryContext tmpCtx;
      36             :     MemoryContext funcCtx;
      37             :     BuildAccumulator accum;
      38             : } GinBuildState;
      39             : 
      40             : 
      41             : /*
      42             :  * Adds array of item pointers to tuple's posting list, or
      43             :  * creates posting tree and tuple pointing to tree in case
      44             :  * of not enough space.  Max size of tuple is defined in
      45             :  * GinFormTuple().  Returns a new, modified index tuple.
      46             :  * items[] must be in sorted order with no duplicates.
      47             :  */
      48             : static IndexTuple
      49       25308 : addItemPointersToLeafTuple(GinState *ginstate,
      50             :                            IndexTuple old,
      51             :                            ItemPointerData *items, uint32 nitem,
      52             :                            GinStatsData *buildStats, Buffer buffer)
      53             : {
      54             :     OffsetNumber attnum;
      55             :     Datum       key;
      56             :     GinNullCategory category;
      57             :     IndexTuple  res;
      58             :     ItemPointerData *newItems,
      59             :                *oldItems;
      60             :     int         oldNPosting,
      61             :                 newNPosting;
      62             :     GinPostingList *compressedList;
      63             : 
      64             :     Assert(!GinIsPostingTree(old));
      65             : 
      66       25308 :     attnum = gintuple_get_attrnum(ginstate, old);
      67       25308 :     key = gintuple_get_key(ginstate, old, &category);
      68             : 
      69             :     /* merge the old and new posting lists */
      70       25308 :     oldItems = ginReadTuple(ginstate, attnum, old, &oldNPosting);
      71             : 
      72       25308 :     newItems = ginMergeItemPointers(items, nitem,
      73             :                                     oldItems, oldNPosting,
      74             :                                     &newNPosting);
      75             : 
      76             :     /* Compress the posting list, and try to a build tuple with room for it */
      77       25308 :     res = NULL;
      78       25308 :     compressedList = ginCompressPostingList(newItems, newNPosting, GinMaxItemSize,
      79             :                                             NULL);
      80       25308 :     pfree(newItems);
      81       25308 :     if (compressedList)
      82             :     {
      83       50616 :         res = GinFormTuple(ginstate, attnum, key, category,
      84             :                            (char *) compressedList,
      85       25308 :                            SizeOfGinPostingList(compressedList),
      86             :                            newNPosting,
      87             :                            false);
      88       25308 :         pfree(compressedList);
      89             :     }
      90       25308 :     if (!res)
      91             :     {
      92             :         /* posting list would be too big, convert to posting tree */
      93             :         BlockNumber postingRoot;
      94             : 
      95             :         /*
      96             :          * Initialize posting tree with the old tuple's posting list.  It's
      97             :          * surely small enough to fit on one posting-tree page, and should
      98             :          * already be in order with no duplicates.
      99             :          */
     100           8 :         postingRoot = createPostingTree(ginstate->index,
     101             :                                         oldItems,
     102             :                                         oldNPosting,
     103             :                                         buildStats,
     104             :                                         buffer);
     105             : 
     106             :         /* Now insert the TIDs-to-be-added into the posting tree */
     107           8 :         ginInsertItemPointers(ginstate->index, postingRoot,
     108             :                               items, nitem,
     109             :                               buildStats);
     110             : 
     111             :         /* And build a new posting-tree-only result tuple */
     112           8 :         res = GinFormTuple(ginstate, attnum, key, category, NULL, 0, 0, true);
     113           8 :         GinSetPostingTree(res, postingRoot);
     114             :     }
     115       25308 :     pfree(oldItems);
     116             : 
     117       25308 :     return res;
     118             : }
     119             : 
     120             : /*
     121             :  * Build a fresh leaf tuple, either posting-list or posting-tree format
     122             :  * depending on whether the given items list will fit.
     123             :  * items[] must be in sorted order with no duplicates.
     124             :  *
     125             :  * This is basically the same logic as in addItemPointersToLeafTuple,
     126             :  * but working from slightly different input.
     127             :  */
     128             : static IndexTuple
     129      199640 : buildFreshLeafTuple(GinState *ginstate,
     130             :                     OffsetNumber attnum, Datum key, GinNullCategory category,
     131             :                     ItemPointerData *items, uint32 nitem,
     132             :                     GinStatsData *buildStats, Buffer buffer)
     133             : {
     134      199640 :     IndexTuple  res = NULL;
     135             :     GinPostingList *compressedList;
     136             : 
     137             :     /* try to build a posting list tuple with all the items */
     138      199640 :     compressedList = ginCompressPostingList(items, nitem, GinMaxItemSize, NULL);
     139      199640 :     if (compressedList)
     140             :     {
     141      399280 :         res = GinFormTuple(ginstate, attnum, key, category,
     142             :                            (char *) compressedList,
     143      199640 :                            SizeOfGinPostingList(compressedList),
     144             :                            nitem, false);
     145      199640 :         pfree(compressedList);
     146             :     }
     147      199640 :     if (!res)
     148             :     {
     149             :         /* posting list would be too big, build posting tree */
     150             :         BlockNumber postingRoot;
     151             : 
     152             :         /*
     153             :          * Build posting-tree-only result tuple.  We do this first so as to
     154             :          * fail quickly if the key is too big.
     155             :          */
     156          82 :         res = GinFormTuple(ginstate, attnum, key, category, NULL, 0, 0, true);
     157             : 
     158             :         /*
     159             :          * Initialize a new posting tree with the TIDs.
     160             :          */
     161          82 :         postingRoot = createPostingTree(ginstate->index, items, nitem,
     162             :                                         buildStats, buffer);
     163             : 
     164             :         /* And save the root link in the result tuple */
     165          82 :         GinSetPostingTree(res, postingRoot);
     166             :     }
     167             : 
     168      199640 :     return res;
     169             : }
     170             : 
     171             : /*
     172             :  * Insert one or more heap TIDs associated with the given key value.
     173             :  * This will either add a single key entry, or enlarge a pre-existing entry.
     174             :  *
     175             :  * During an index build, buildStats is non-null and the counters
     176             :  * it contains should be incremented as needed.
     177             :  */
     178             : void
     179      267668 : ginEntryInsert(GinState *ginstate,
     180             :                OffsetNumber attnum, Datum key, GinNullCategory category,
     181             :                ItemPointerData *items, uint32 nitem,
     182             :                GinStatsData *buildStats)
     183             : {
     184             :     GinBtreeData btree;
     185             :     GinBtreeEntryInsertData insertdata;
     186             :     GinBtreeStack *stack;
     187             :     IndexTuple  itup;
     188             :     Page        page;
     189             : 
     190      267668 :     insertdata.isDelete = false;
     191             : 
     192      267668 :     ginPrepareEntryScan(&btree, attnum, key, category, ginstate);
     193      267668 :     btree.isBuild = (buildStats != NULL);
     194             : 
     195      267668 :     stack = ginFindLeafPage(&btree, false, false, NULL);
     196      267668 :     page = BufferGetPage(stack->buffer);
     197             : 
     198      267668 :     if (btree.findItem(&btree, stack))
     199             :     {
     200             :         /* found pre-existing entry */
     201       68024 :         itup = (IndexTuple) PageGetItem(page, PageGetItemId(page, stack->off));
     202             : 
     203       68024 :         if (GinIsPostingTree(itup))
     204             :         {
     205             :             /* add entries to existing posting tree */
     206       42708 :             BlockNumber rootPostingTree = GinGetPostingTree(itup);
     207             : 
     208             :             /* release all stack */
     209       42708 :             LockBuffer(stack->buffer, GIN_UNLOCK);
     210       42708 :             freeGinBtreeStack(stack);
     211             : 
     212             :             /* insert into posting tree */
     213       42708 :             ginInsertItemPointers(ginstate->index, rootPostingTree,
     214             :                                   items, nitem,
     215             :                                   buildStats);
     216       42704 :             return;
     217             :         }
     218             : 
     219       25316 :         CheckForSerializableConflictIn(ginstate->index, NULL, stack->buffer);
     220             :         /* modify an existing leaf entry */
     221       25308 :         itup = addItemPointersToLeafTuple(ginstate, itup,
     222             :                                           items, nitem, buildStats, stack->buffer);
     223             : 
     224       25308 :         insertdata.isDelete = true;
     225             :     }
     226             :     else
     227             :     {
     228      199644 :         CheckForSerializableConflictIn(ginstate->index, NULL, stack->buffer);
     229             :         /* no match, so construct a new leaf entry */
     230      199640 :         itup = buildFreshLeafTuple(ginstate, attnum, key, category,
     231             :                                    items, nitem, buildStats, stack->buffer);
     232             : 
     233             :         /*
     234             :          * nEntries counts leaf tuples, so increment it only when we make a
     235             :          * new one.
     236             :          */
     237      199640 :         if (buildStats)
     238      119632 :             buildStats->nEntries++;
     239             :     }
     240             : 
     241             :     /* Insert the new or modified leaf tuple */
     242      224948 :     insertdata.entry = itup;
     243      224948 :     ginInsertValue(&btree, stack, &insertdata, buildStats);
     244      224948 :     pfree(itup);
     245             : }
     246             : 
     247             : /*
     248             :  * Extract index entries for a single indexable item, and add them to the
     249             :  * BuildAccumulator's state.
     250             :  *
     251             :  * This function is used only during initial index creation.
     252             :  */
     253             : static void
     254      834844 : ginHeapTupleBulkInsert(GinBuildState *buildstate, OffsetNumber attnum,
     255             :                        Datum value, bool isNull,
     256             :                        ItemPointer heapptr)
     257             : {
     258             :     Datum      *entries;
     259             :     GinNullCategory *categories;
     260             :     int32       nentries;
     261             :     MemoryContext oldCtx;
     262             : 
     263      834844 :     oldCtx = MemoryContextSwitchTo(buildstate->funcCtx);
     264      834844 :     entries = ginExtractEntries(buildstate->accum.ginstate, attnum,
     265             :                                 value, isNull,
     266             :                                 &nentries, &categories);
     267      834844 :     MemoryContextSwitchTo(oldCtx);
     268             : 
     269      834844 :     ginInsertBAEntries(&buildstate->accum, heapptr, attnum,
     270             :                        entries, categories, nentries);
     271             : 
     272      834844 :     buildstate->indtuples += nentries;
     273             : 
     274      834844 :     MemoryContextReset(buildstate->funcCtx);
     275      834844 : }
     276             : 
     277             : static void
     278      834432 : ginBuildCallback(Relation index, ItemPointer tid, Datum *values,
     279             :                  bool *isnull, bool tupleIsAlive, void *state)
     280             : {
     281      834432 :     GinBuildState *buildstate = (GinBuildState *) state;
     282             :     MemoryContext oldCtx;
     283             :     int         i;
     284             : 
     285      834432 :     oldCtx = MemoryContextSwitchTo(buildstate->tmpCtx);
     286             : 
     287     1669276 :     for (i = 0; i < buildstate->ginstate.origTupdesc->natts; i++)
     288     1669688 :         ginHeapTupleBulkInsert(buildstate, (OffsetNumber) (i + 1),
     289     1669688 :                                values[i], isnull[i], tid);
     290             : 
     291             :     /* If we've maxed out our available memory, dump everything to the index */
     292      834432 :     if (buildstate->accum.allocatedMemory >= (Size) maintenance_work_mem * 1024L)
     293             :     {
     294             :         ItemPointerData *list;
     295             :         Datum       key;
     296             :         GinNullCategory category;
     297             :         uint32      nlist;
     298             :         OffsetNumber attnum;
     299             : 
     300           0 :         ginBeginBAScan(&buildstate->accum);
     301           0 :         while ((list = ginGetBAEntry(&buildstate->accum,
     302             :                                      &attnum, &key, &category, &nlist)) != NULL)
     303             :         {
     304             :             /* there could be many entries, so be willing to abort here */
     305           0 :             CHECK_FOR_INTERRUPTS();
     306           0 :             ginEntryInsert(&buildstate->ginstate, attnum, key, category,
     307             :                            list, nlist, &buildstate->buildStats);
     308             :         }
     309             : 
     310           0 :         MemoryContextReset(buildstate->tmpCtx);
     311           0 :         ginInitBA(&buildstate->accum);
     312             :     }
     313             : 
     314      834432 :     MemoryContextSwitchTo(oldCtx);
     315      834432 : }
     316             : 
     317             : IndexBuildResult *
     318         198 : ginbuild(Relation heap, Relation index, IndexInfo *indexInfo)
     319             : {
     320             :     IndexBuildResult *result;
     321             :     double      reltuples;
     322             :     GinBuildState buildstate;
     323             :     Buffer      RootBuffer,
     324             :                 MetaBuffer;
     325             :     ItemPointerData *list;
     326             :     Datum       key;
     327             :     GinNullCategory category;
     328             :     uint32      nlist;
     329             :     MemoryContext oldCtx;
     330             :     OffsetNumber attnum;
     331             : 
     332         198 :     if (RelationGetNumberOfBlocks(index) != 0)
     333           0 :         elog(ERROR, "index \"%s\" already contains data",
     334             :              RelationGetRelationName(index));
     335             : 
     336         198 :     initGinState(&buildstate.ginstate, index);
     337         198 :     buildstate.indtuples = 0;
     338         198 :     memset(&buildstate.buildStats, 0, sizeof(GinStatsData));
     339             : 
     340             :     /* initialize the meta page */
     341         198 :     MetaBuffer = GinNewBuffer(index);
     342             : 
     343             :     /* initialize the root page */
     344         198 :     RootBuffer = GinNewBuffer(index);
     345             : 
     346         198 :     START_CRIT_SECTION();
     347         198 :     GinInitMetabuffer(MetaBuffer);
     348         198 :     MarkBufferDirty(MetaBuffer);
     349         198 :     GinInitBuffer(RootBuffer, GIN_LEAF);
     350         198 :     MarkBufferDirty(RootBuffer);
     351             : 
     352             : 
     353         198 :     UnlockReleaseBuffer(MetaBuffer);
     354         198 :     UnlockReleaseBuffer(RootBuffer);
     355         198 :     END_CRIT_SECTION();
     356             : 
     357             :     /* count the root as first entry page */
     358         198 :     buildstate.buildStats.nEntryPages++;
     359             : 
     360             :     /*
     361             :      * create a temporary memory context that is used to hold data not yet
     362             :      * dumped out to the index
     363             :      */
     364         198 :     buildstate.tmpCtx = AllocSetContextCreate(CurrentMemoryContext,
     365             :                                               "Gin build temporary context",
     366             :                                               ALLOCSET_DEFAULT_SIZES);
     367             : 
     368             :     /*
     369             :      * create a temporary memory context that is used for calling
     370             :      * ginExtractEntries(), and can be reset after each tuple
     371             :      */
     372         198 :     buildstate.funcCtx = AllocSetContextCreate(CurrentMemoryContext,
     373             :                                                "Gin build temporary context for user-defined function",
     374             :                                                ALLOCSET_DEFAULT_SIZES);
     375             : 
     376         198 :     buildstate.accum.ginstate = &buildstate.ginstate;
     377         198 :     ginInitBA(&buildstate.accum);
     378             : 
     379             :     /*
     380             :      * Do the heap scan.  We disallow sync scan here because dataPlaceToPage
     381             :      * prefers to receive tuples in TID order.
     382             :      */
     383         198 :     reltuples = table_index_build_scan(heap, index, indexInfo, false, true,
     384             :                                        ginBuildCallback, (void *) &buildstate,
     385             :                                        NULL);
     386             : 
     387             :     /* dump remaining entries to the index */
     388         198 :     oldCtx = MemoryContextSwitchTo(buildstate.tmpCtx);
     389         198 :     ginBeginBAScan(&buildstate.accum);
     390      120028 :     while ((list = ginGetBAEntry(&buildstate.accum,
     391             :                                  &attnum, &key, &category, &nlist)) != NULL)
     392             :     {
     393             :         /* there could be many entries, so be willing to abort here */
     394      119632 :         CHECK_FOR_INTERRUPTS();
     395      119632 :         ginEntryInsert(&buildstate.ginstate, attnum, key, category,
     396             :                        list, nlist, &buildstate.buildStats);
     397             :     }
     398         198 :     MemoryContextSwitchTo(oldCtx);
     399             : 
     400         198 :     MemoryContextDelete(buildstate.funcCtx);
     401         198 :     MemoryContextDelete(buildstate.tmpCtx);
     402             : 
     403             :     /*
     404             :      * Update metapage stats
     405             :      */
     406         198 :     buildstate.buildStats.nTotalPages = RelationGetNumberOfBlocks(index);
     407         198 :     ginUpdateStats(index, &buildstate.buildStats, true);
     408             : 
     409             :     /*
     410             :      * We didn't write WAL records as we built the index, so if WAL-logging is
     411             :      * required, write all pages to the WAL now.
     412             :      */
     413         198 :     if (RelationNeedsWAL(index))
     414             :     {
     415         186 :         log_newpage_range(index, MAIN_FORKNUM,
     416             :                           0, RelationGetNumberOfBlocks(index),
     417             :                           true);
     418             :     }
     419             : 
     420             :     /*
     421             :      * Return statistics
     422             :      */
     423         198 :     result = (IndexBuildResult *) palloc(sizeof(IndexBuildResult));
     424             : 
     425         198 :     result->heap_tuples = reltuples;
     426         198 :     result->index_tuples = buildstate.indtuples;
     427             : 
     428         198 :     return result;
     429             : }
     430             : 
     431             : /*
     432             :  *  ginbuildempty() -- build an empty gin index in the initialization fork
     433             :  */
     434             : void
     435           0 : ginbuildempty(Relation index)
     436             : {
     437             :     Buffer      RootBuffer,
     438             :                 MetaBuffer;
     439             : 
     440             :     /* An empty GIN index has two pages. */
     441           0 :     MetaBuffer =
     442             :         ReadBufferExtended(index, INIT_FORKNUM, P_NEW, RBM_NORMAL, NULL);
     443           0 :     LockBuffer(MetaBuffer, BUFFER_LOCK_EXCLUSIVE);
     444           0 :     RootBuffer =
     445             :         ReadBufferExtended(index, INIT_FORKNUM, P_NEW, RBM_NORMAL, NULL);
     446           0 :     LockBuffer(RootBuffer, BUFFER_LOCK_EXCLUSIVE);
     447             : 
     448             :     /* Initialize and xlog metabuffer and root buffer. */
     449           0 :     START_CRIT_SECTION();
     450           0 :     GinInitMetabuffer(MetaBuffer);
     451           0 :     MarkBufferDirty(MetaBuffer);
     452           0 :     log_newpage_buffer(MetaBuffer, true);
     453           0 :     GinInitBuffer(RootBuffer, GIN_LEAF);
     454           0 :     MarkBufferDirty(RootBuffer);
     455           0 :     log_newpage_buffer(RootBuffer, false);
     456           0 :     END_CRIT_SECTION();
     457             : 
     458             :     /* Unlock and release the buffers. */
     459           0 :     UnlockReleaseBuffer(MetaBuffer);
     460           0 :     UnlockReleaseBuffer(RootBuffer);
     461           0 : }
     462             : 
     463             : /*
     464             :  * Insert index entries for a single indexable item during "normal"
     465             :  * (non-fast-update) insertion
     466             :  */
     467             : static void
     468       28054 : ginHeapTupleInsert(GinState *ginstate, OffsetNumber attnum,
     469             :                    Datum value, bool isNull,
     470             :                    ItemPointer item)
     471             : {
     472             :     Datum      *entries;
     473             :     GinNullCategory *categories;
     474             :     int32       i,
     475             :                 nentries;
     476             : 
     477       28054 :     entries = ginExtractEntries(ginstate, attnum, value, isNull,
     478             :                                 &nentries, &categories);
     479             : 
     480       92074 :     for (i = 0; i < nentries; i++)
     481       64036 :         ginEntryInsert(ginstate, attnum, entries[i], categories[i],
     482             :                        item, 1, NULL);
     483       28038 : }
     484             : 
     485             : bool
     486      116084 : gininsert(Relation index, Datum *values, bool *isnull,
     487             :           ItemPointer ht_ctid, Relation heapRel,
     488             :           IndexUniqueCheck checkUnique,
     489             :           IndexInfo *indexInfo)
     490             : {
     491      116084 :     GinState   *ginstate = (GinState *) indexInfo->ii_AmCache;
     492             :     MemoryContext oldCtx;
     493             :     MemoryContext insertCtx;
     494             :     int         i;
     495             : 
     496             :     /* Initialize GinState cache if first call in this statement */
     497      116084 :     if (ginstate == NULL)
     498             :     {
     499         108 :         oldCtx = MemoryContextSwitchTo(indexInfo->ii_Context);
     500         108 :         ginstate = (GinState *) palloc(sizeof(GinState));
     501         108 :         initGinState(ginstate, index);
     502         108 :         indexInfo->ii_AmCache = (void *) ginstate;
     503         108 :         MemoryContextSwitchTo(oldCtx);
     504             :     }
     505             : 
     506      116084 :     insertCtx = AllocSetContextCreate(CurrentMemoryContext,
     507             :                                       "Gin insert temporary context",
     508             :                                       ALLOCSET_DEFAULT_SIZES);
     509             : 
     510      116084 :     oldCtx = MemoryContextSwitchTo(insertCtx);
     511             : 
     512      116084 :     if (GinGetUseFastUpdate(index))
     513       88024 :     {
     514             :         GinTupleCollector collector;
     515             : 
     516       88030 :         memset(&collector, 0, sizeof(GinTupleCollector));
     517             : 
     518      176060 :         for (i = 0; i < ginstate->origTupdesc->natts; i++)
     519      264090 :             ginHeapTupleFastCollect(ginstate, &collector,
     520       88030 :                                     (OffsetNumber) (i + 1),
     521      176060 :                                     values[i], isnull[i],
     522             :                                     ht_ctid);
     523             : 
     524       88030 :         ginHeapTupleFastInsert(ginstate, &collector);
     525             :     }
     526             :     else
     527             :     {
     528       56092 :         for (i = 0; i < ginstate->origTupdesc->natts; i++)
     529       56108 :             ginHeapTupleInsert(ginstate, (OffsetNumber) (i + 1),
     530       56108 :                                values[i], isnull[i],
     531             :                                ht_ctid);
     532             :     }
     533             : 
     534      116062 :     MemoryContextSwitchTo(oldCtx);
     535      116062 :     MemoryContextDelete(insertCtx);
     536             : 
     537      116062 :     return false;
     538             : }

Generated by: LCOV version 1.13