Line data    Source code

       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * toast_helper.c
       4             :  *    Helper functions for table AMs implementing compressed or
       5             :  *    out-of-line storage of varlena attributes.
       6             :  *
       7             :  * Copyright (c) 2000-2023, PostgreSQL Global Development Group
       8             :  *
       9             :  * IDENTIFICATION
      10             :  *    src/backend/access/table/toast_helper.c
      11             :  *
      12             :  *-------------------------------------------------------------------------
      13             :  */
      14             : 
      15             : #include "postgres.h"
      16             : 
      17             : #include "access/detoast.h"
      18             : #include "access/table.h"
      19             : #include "access/toast_helper.h"
      20             : #include "access/toast_internals.h"
      21             : #include "catalog/pg_type_d.h"
      22             : #include "varatt.h"
      23             : 
      24             : 
      25             : /*
      26             :  * Prepare to TOAST a tuple.
      27             :  *
      28             :  * tupleDesc, toast_values, and toast_isnull are required parameters; they
      29             :  * provide the necessary details about the tuple to be toasted.
      30             :  *
      31             :  * toast_oldvalues and toast_oldisnull should be NULL for a newly-inserted
      32             :  * tuple; for an update, they should describe the existing tuple.
      33             :  *
      34             :  * All of these arrays should have a length equal to tupleDesc->natts.
      35             :  *
      36             :  * On return, toast_flags and toast_attr will have been initialized.
      37             :  * toast_flags is just a single uint8, but toast_attr is a caller-provided
      38             :  * array with a length equal to tupleDesc->natts.  The caller need not
      39             :  * perform any initialization of the array before calling this function.
      40             :  */
      41             : void
      42      125968 : toast_tuple_init(ToastTupleContext *ttc)
      43             : {
      44      125968 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
      45      125968 :     int         numAttrs = tupleDesc->natts;
      46             :     int         i;
      47             : 
      48      125968 :     ttc->ttc_flags = 0;
      49             : 
      50     1543648 :     for (i = 0; i < numAttrs; i++)
      51             :     {
      52     1417680 :         Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
      53             :         struct varlena *old_value;
      54             :         struct varlena *new_value;
      55             : 
      56     1417680 :         ttc->ttc_attr[i].tai_colflags = 0;
      57     1417680 :         ttc->ttc_attr[i].tai_oldexternal = NULL;
      58     1417680 :         ttc->ttc_attr[i].tai_compression = att->attcompression;
      59             : 
      60     1417680 :         if (ttc->ttc_oldvalues != NULL)
      61             :         {
      62             :             /*
      63             :              * For UPDATE get the old and new values of this attribute
      64             :              */
      65             :             old_value =
      66      172068 :                 (struct varlena *) DatumGetPointer(ttc->ttc_oldvalues[i]);
      67             :             new_value =
      68      172068 :                 (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
      69             : 
      70             :             /*
      71             :              * If the old value is stored on disk, check if it has changed so
      72             :              * we have to delete it later.
      73             :              */
      74      172068 :             if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
      75       17350 :                 VARATT_IS_EXTERNAL_ONDISK(old_value))
      76             :             {
      77         532 :                 if (ttc->ttc_isnull[i] ||
      78         516 :                     !VARATT_IS_EXTERNAL_ONDISK(new_value) ||
      79          96 :                     memcmp((char *) old_value, (char *) new_value,
      80          96 :                            VARSIZE_EXTERNAL(old_value)) != 0)
      81             :                 {
      82             :                     /*
      83             :                      * The old external stored value isn't needed any more
      84             :                      * after the update
      85             :                      */
      86         438 :                     ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_DELETE_OLD;
      87         438 :                     ttc->ttc_flags |= TOAST_NEEDS_DELETE_OLD;
      88             :                 }
      89             :                 else
      90             :                 {
      91             :                     /*
      92             :                      * This attribute isn't changed by this update so we reuse
      93             :                      * the original reference to the old value in the new
      94             :                      * tuple.
      95             :                      */
      96          94 :                     ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
      97          94 :                     continue;
      98             :                 }
      99             :             }
     100             :         }
     101             :         else
     102             :         {
     103             :             /*
     104             :              * For INSERT simply get the new value
     105             :              */
     106     1245612 :             new_value = (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
     107             :         }
     108             : 
     109             :         /*
     110             :          * Handle NULL attributes
     111             :          */
     112     1417586 :         if (ttc->ttc_isnull[i])
     113             :         {
     114      144686 :             ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     115      144686 :             ttc->ttc_flags |= TOAST_HAS_NULLS;
     116      144686 :             continue;
     117             :         }
     118             : 
     119             :         /*
     120             :          * Now look at varlena attributes
     121             :          */
     122     1272900 :         if (att->attlen == -1)
     123             :         {
     124             :             /*
     125             :              * If the table's attribute says PLAIN always, force it so.
     126             :              */
     127      279344 :             if (att->attstorage == TYPSTORAGE_PLAIN)
     128        9746 :                 ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     129             : 
     130             :             /*
     131             :              * We took care of UPDATE above, so any external value we find
     132             :              * still in the tuple must be someone else's that we cannot reuse
     133             :              * (this includes the case of an out-of-line in-memory datum).
     134             :              * Fetch it back (without decompression, unless we are forcing
     135             :              * PLAIN storage).  If necessary, we'll push it out as a new
     136             :              * external value below.
     137             :              */
     138      279344 :             if (VARATT_IS_EXTERNAL(new_value))
     139             :             {
     140         944 :                 ttc->ttc_attr[i].tai_oldexternal = new_value;
     141         944 :                 if (att->attstorage == TYPSTORAGE_PLAIN)
     142           0 :                     new_value = detoast_attr(new_value);
     143             :                 else
     144         944 :                     new_value = detoast_external_attr(new_value);
     145         944 :                 ttc->ttc_values[i] = PointerGetDatum(new_value);
     146         944 :                 ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_FREE;
     147         944 :                 ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     148             :             }
     149             : 
     150             :             /*
     151             :              * Remember the size of this attribute
     152             :              */
     153      279344 :             ttc->ttc_attr[i].tai_size = VARSIZE_ANY(new_value);
     154             :         }
     155             :         else
     156             :         {
     157             :             /*
     158             :              * Not a varlena attribute, plain storage always
     159             :              */
     160      993556 :             ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     161             :         }
     162             :     }
     163      125968 : }
     164             : 
     165             : /*
     166             :  * Find the largest varlena attribute that satisfies certain criteria.
     167             :  *
     168             :  * The relevant column must not be marked TOASTCOL_IGNORE, and if the
     169             :  * for_compression flag is passed as true, it must also not be marked
     170             :  * TOASTCOL_INCOMPRESSIBLE.
     171             :  *
     172             :  * The column must have attstorage EXTERNAL or EXTENDED if check_main is
     173             :  * false, and must have attstorage MAIN if check_main is true.
     174             :  *
     175             :  * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
     176             :  * if not, no benefit is to be expected by compressing it.
     177             :  *
     178             :  * The return value is the index of the biggest suitable column, or
     179             :  * -1 if there is none.
     180             :  */
     181             : int
     182      152022 : toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
     183             :                                    bool for_compression, bool check_main)
     184             : {
     185      152022 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
     186      152022 :     int         numAttrs = tupleDesc->natts;
     187      152022 :     int         biggest_attno = -1;
     188      152022 :     int32       biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
     189      152022 :     int32       skip_colflags = TOASTCOL_IGNORE;
     190             :     int         i;
     191             : 
     192      152022 :     if (for_compression)
     193      145386 :         skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
     194             : 
     195     2195702 :     for (i = 0; i < numAttrs; i++)
     196             :     {
     197     2043680 :         Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
     198             : 
     199     2043680 :         if ((ttc->ttc_attr[i].tai_colflags & skip_colflags) != 0)
     200     1683870 :             continue;
     201      359810 :         if (VARATT_IS_EXTERNAL(DatumGetPointer(ttc->ttc_values[i])))
     202           0 :             continue;           /* can't happen, toast_action would be PLAIN */
     203      359810 :         if (for_compression &&
     204      341896 :             VARATT_IS_COMPRESSED(DatumGetPointer(ttc->ttc_values[i])))
     205       28554 :             continue;
     206      331256 :         if (check_main && att->attstorage != TYPSTORAGE_MAIN)
     207           0 :             continue;
     208      331256 :         if (!check_main && att->attstorage != TYPSTORAGE_EXTENDED &&
     209        5912 :             att->attstorage != TYPSTORAGE_EXTERNAL)
     210         112 :             continue;
     211             : 
     212      331144 :         if (ttc->ttc_attr[i].tai_size > biggest_size)
     213             :         {
     214      192608 :             biggest_attno = i;
     215      192608 :             biggest_size = ttc->ttc_attr[i].tai_size;
     216             :         }
     217             :     }
     218             : 
     219      152022 :     return biggest_attno;
     220             : }
     221             : 
     222             : /*
     223             :  * Try compression for an attribute.
     224             :  *
     225             :  * If we find that the attribute is not compressible, mark it so.
     226             :  */
     227             : void
     228      132992 : toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
     229             : {
     230      132992 :     Datum      *value = &ttc->ttc_values[attribute];
     231             :     Datum       new_value;
     232      132992 :     ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
     233             : 
     234      132992 :     new_value = toast_compress_datum(*value, attr->tai_compression);
     235             : 
     236      132992 :     if (DatumGetPointer(new_value) != NULL)
     237             :     {
     238             :         /* successful compression */
     239      129900 :         if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     240         122 :             pfree(DatumGetPointer(*value));
     241      129900 :         *value = new_value;
     242      129900 :         attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
     243      129900 :         attr->tai_size = VARSIZE(DatumGetPointer(*value));
     244      129900 :         ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     245             :     }
     246             :     else
     247             :     {
     248             :         /* incompressible, ignore on subsequent compression passes */
     249        3092 :         attr->tai_colflags |= TOASTCOL_INCOMPRESSIBLE;
     250             :     }
     251      132992 : }
     252             : 
     253             : /*
     254             :  * Move an attribute to external storage.
     255             :  */
     256             : void
     257       64328 : toast_tuple_externalize(ToastTupleContext *ttc, int attribute, int options)
     258             : {
     259       64328 :     Datum      *value = &ttc->ttc_values[attribute];
     260       64328 :     Datum       old_value = *value;
     261       64328 :     ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
     262             : 
     263       64328 :     attr->tai_colflags |= TOASTCOL_IGNORE;
     264       64328 :     *value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal,
     265             :                               options);
     266       64328 :     if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     267       58498 :         pfree(DatumGetPointer(old_value));
     268       64328 :     attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
     269       64328 :     ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     270       64328 : }
     271             : 
     272             : /*
     273             :  * Perform appropriate cleanup after one tuple has been subjected to TOAST.
     274             :  */
     275             : void
     276      125968 : toast_tuple_cleanup(ToastTupleContext *ttc)
     277             : {
     278      125968 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
     279      125968 :     int         numAttrs = tupleDesc->natts;
     280             : 
     281             :     /*
     282             :      * Free allocated temp values
     283             :      */
     284      125968 :     if ((ttc->ttc_flags & TOAST_NEEDS_FREE) != 0)
     285             :     {
     286             :         int         i;
     287             : 
     288     1541976 :         for (i = 0; i < numAttrs; i++)
     289             :         {
     290     1416138 :             ToastAttrInfo *attr = &ttc->ttc_attr[i];
     291             : 
     292     1416138 :             if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     293      136552 :                 pfree(DatumGetPointer(ttc->ttc_values[i]));
     294             :         }
     295             :     }
     296             : 
     297             :     /*
     298             :      * Delete external values from the old tuple
     299             :      */
     300      125968 :     if ((ttc->ttc_flags & TOAST_NEEDS_DELETE_OLD) != 0)
     301             :     {
     302             :         int         i;
     303             : 
     304        9268 :         for (i = 0; i < numAttrs; i++)
     305             :         {
     306        8890 :             ToastAttrInfo *attr = &ttc->ttc_attr[i];
     307             : 
     308        8890 :             if ((attr->tai_colflags & TOASTCOL_NEEDS_DELETE_OLD) != 0)
     309         438 :                 toast_delete_datum(ttc->ttc_rel, ttc->ttc_oldvalues[i], false);
     310             :         }
     311             :     }
     312      125968 : }
     313             : 
     314             : /*
     315             :  * Check for external stored attributes and delete them from the secondary
     316             :  * relation.
     317             :  */
     318             : void
     319         474 : toast_delete_external(Relation rel, Datum *values, bool *isnull,
     320             :                       bool is_speculative)
     321             : {
     322         474 :     TupleDesc   tupleDesc = rel->rd_att;
     323         474 :     int         numAttrs = tupleDesc->natts;
     324             :     int         i;
     325             : 
     326        3266 :     for (i = 0; i < numAttrs; i++)
     327             :     {
     328        2792 :         if (TupleDescAttr(tupleDesc, i)->attlen == -1)
     329             :         {
     330        1010 :             Datum       value = values[i];
     331             : 
     332        1010 :             if (isnull[i])
     333         300 :                 continue;
     334         710 :             else if (VARATT_IS_EXTERNAL_ONDISK(value))
     335         494 :                 toast_delete_datum(rel, value, is_speculative);
     336             :         }
     337             :     }
     338         474 : }