Line data    Source code

       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * toast_helper.c
       4             :  *    Helper functions for table AMs implementing compressed or
       5             :  *    out-of-line storage of varlena attributes.
       6             :  *
       7             :  * Copyright (c) 2000-2025, PostgreSQL Global Development Group
       8             :  *
       9             :  * IDENTIFICATION
      10             :  *    src/backend/access/table/toast_helper.c
      11             :  *
      12             :  *-------------------------------------------------------------------------
      13             :  */
      14             : 
      15             : #include "postgres.h"
      16             : 
      17             : #include "access/detoast.h"
      18             : #include "access/toast_helper.h"
      19             : #include "access/toast_internals.h"
      20             : #include "catalog/pg_type_d.h"
      21             : #include "varatt.h"
      22             : 
      23             : 
      24             : /*
      25             :  * Prepare to TOAST a tuple.
      26             :  *
      27             :  * tupleDesc, toast_values, and toast_isnull are required parameters; they
      28             :  * provide the necessary details about the tuple to be toasted.
      29             :  *
      30             :  * toast_oldvalues and toast_oldisnull should be NULL for a newly-inserted
      31             :  * tuple; for an update, they should describe the existing tuple.
      32             :  *
      33             :  * All of these arrays should have a length equal to tupleDesc->natts.
      34             :  *
      35             :  * On return, toast_flags and toast_attr will have been initialized.
      36             :  * toast_flags is just a single uint8, but toast_attr is a caller-provided
      37             :  * array with a length equal to tupleDesc->natts.  The caller need not
      38             :  * perform any initialization of the array before calling this function.
      39             :  */
      40             : void
      41       37036 : toast_tuple_init(ToastTupleContext *ttc)
      42             : {
      43       37036 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
      44       37036 :     int         numAttrs = tupleDesc->natts;
      45             :     int         i;
      46             : 
      47       37036 :     ttc->ttc_flags = 0;
      48             : 
      49      349310 :     for (i = 0; i < numAttrs; i++)
      50             :     {
      51      312274 :         Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
      52             :         struct varlena *old_value;
      53             :         struct varlena *new_value;
      54             : 
      55      312274 :         ttc->ttc_attr[i].tai_colflags = 0;
      56      312274 :         ttc->ttc_attr[i].tai_oldexternal = NULL;
      57      312274 :         ttc->ttc_attr[i].tai_compression = att->attcompression;
      58             : 
      59      312274 :         if (ttc->ttc_oldvalues != NULL)
      60             :         {
      61             :             /*
      62             :              * For UPDATE get the old and new values of this attribute
      63             :              */
      64             :             old_value =
      65       66646 :                 (struct varlena *) DatumGetPointer(ttc->ttc_oldvalues[i]);
      66             :             new_value =
      67       66646 :                 (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
      68             : 
      69             :             /*
      70             :              * If the old value is stored on disk, check if it has changed so
      71             :              * we have to delete it later.
      72             :              */
      73       66646 :             if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
      74        7776 :                 VARATT_IS_EXTERNAL_ONDISK(old_value))
      75             :             {
      76         656 :                 if (ttc->ttc_isnull[i] ||
      77         638 :                     !VARATT_IS_EXTERNAL_ONDISK(new_value) ||
      78          96 :                     memcmp((char *) old_value, (char *) new_value,
      79          96 :                            VARSIZE_EXTERNAL(old_value)) != 0)
      80             :                 {
      81             :                     /*
      82             :                      * The old external stored value isn't needed any more
      83             :                      * after the update
      84             :                      */
      85         562 :                     ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_DELETE_OLD;
      86         562 :                     ttc->ttc_flags |= TOAST_NEEDS_DELETE_OLD;
      87             :                 }
      88             :                 else
      89             :                 {
      90             :                     /*
      91             :                      * This attribute isn't changed by this update so we reuse
      92             :                      * the original reference to the old value in the new
      93             :                      * tuple.
      94             :                      */
      95          94 :                     ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
      96          94 :                     continue;
      97             :                 }
      98             :             }
      99             :         }
     100             :         else
     101             :         {
     102             :             /*
     103             :              * For INSERT simply get the new value
     104             :              */
     105      245628 :             new_value = (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
     106             :         }
     107             : 
     108             :         /*
     109             :          * Handle NULL attributes
     110             :          */
     111      312180 :         if (ttc->ttc_isnull[i])
     112             :         {
     113       31208 :             ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     114       31208 :             ttc->ttc_flags |= TOAST_HAS_NULLS;
     115       31208 :             continue;
     116             :         }
     117             : 
     118             :         /*
     119             :          * Now look at varlena attributes
     120             :          */
     121      280972 :         if (att->attlen == -1)
     122             :         {
     123             :             /*
     124             :              * If the table's attribute says PLAIN always, force it so.
     125             :              */
     126       67268 :             if (att->attstorage == TYPSTORAGE_PLAIN)
     127        2200 :                 ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     128             : 
     129             :             /*
     130             :              * We took care of UPDATE above, so any external value we find
     131             :              * still in the tuple must be someone else's that we cannot reuse
     132             :              * (this includes the case of an out-of-line in-memory datum).
     133             :              * Fetch it back (without decompression, unless we are forcing
     134             :              * PLAIN storage).  If necessary, we'll push it out as a new
     135             :              * external value below.
     136             :              */
     137       67268 :             if (VARATT_IS_EXTERNAL(new_value))
     138             :             {
     139         938 :                 ttc->ttc_attr[i].tai_oldexternal = new_value;
     140         938 :                 if (att->attstorage == TYPSTORAGE_PLAIN)
     141           0 :                     new_value = detoast_attr(new_value);
     142             :                 else
     143         938 :                     new_value = detoast_external_attr(new_value);
     144         938 :                 ttc->ttc_values[i] = PointerGetDatum(new_value);
     145         938 :                 ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_FREE;
     146         938 :                 ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     147             :             }
     148             : 
     149             :             /*
     150             :              * Remember the size of this attribute
     151             :              */
     152       67268 :             ttc->ttc_attr[i].tai_size = VARSIZE_ANY(new_value);
     153             :         }
     154             :         else
     155             :         {
     156             :             /*
     157             :              * Not a varlena attribute, plain storage always
     158             :              */
     159      213704 :             ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     160             :         }
     161             :     }
     162       37036 : }
     163             : 
     164             : /*
     165             :  * Find the largest varlena attribute that satisfies certain criteria.
     166             :  *
     167             :  * The relevant column must not be marked TOASTCOL_IGNORE, and if the
     168             :  * for_compression flag is passed as true, it must also not be marked
     169             :  * TOASTCOL_INCOMPRESSIBLE.
     170             :  *
     171             :  * The column must have attstorage EXTERNAL or EXTENDED if check_main is
     172             :  * false, and must have attstorage MAIN if check_main is true.
     173             :  *
     174             :  * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
     175             :  * if not, no benefit is to be expected by compressing it.
     176             :  *
     177             :  * The return value is the index of the biggest suitable column, or
     178             :  * -1 if there is none.
     179             :  */
     180             : int
     181       42652 : toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
     182             :                                    bool for_compression, bool check_main)
     183             : {
     184       42652 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
     185       42652 :     int         numAttrs = tupleDesc->natts;
     186       42652 :     int         biggest_attno = -1;
     187       42652 :     int32       biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
     188       42652 :     int32       skip_colflags = TOASTCOL_IGNORE;
     189             :     int         i;
     190             : 
     191       42652 :     if (for_compression)
     192       41232 :         skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
     193             : 
     194      501600 :     for (i = 0; i < numAttrs; i++)
     195             :     {
     196      458948 :         Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
     197             : 
     198      458948 :         if ((ttc->ttc_attr[i].tai_colflags & skip_colflags) != 0)
     199      374112 :             continue;
     200       84836 :         if (VARATT_IS_EXTERNAL(DatumGetPointer(ttc->ttc_values[i])))
     201           0 :             continue;           /* can't happen, toast_action would be PLAIN */
     202       84836 :         if (for_compression &&
     203       80712 :             VARATT_IS_COMPRESSED(DatumGetPointer(ttc->ttc_values[i])))
     204        6666 :             continue;
     205       78170 :         if (check_main && att->attstorage != TYPSTORAGE_MAIN)
     206           0 :             continue;
     207       78170 :         if (!check_main && att->attstorage != TYPSTORAGE_EXTENDED &&
     208        5922 :             att->attstorage != TYPSTORAGE_EXTERNAL)
     209         112 :             continue;
     210             : 
     211       78058 :         if (ttc->ttc_attr[i].tai_size > biggest_size)
     212             :         {
     213       52072 :             biggest_attno = i;
     214       52072 :             biggest_size = ttc->ttc_attr[i].tai_size;
     215             :         }
     216             :     }
     217             : 
     218       42652 :     return biggest_attno;
     219             : }
     220             : 
     221             : /*
     222             :  * Try compression for an attribute.
     223             :  *
     224             :  * If we find that the attribute is not compressible, mark it so.
     225             :  */
     226             : void
     227       34020 : toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
     228             : {
     229       34020 :     Datum      *value = &ttc->ttc_values[attribute];
     230             :     Datum       new_value;
     231       34020 :     ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
     232             : 
     233       34020 :     new_value = toast_compress_datum(*value, attr->tai_compression);
     234             : 
     235       34020 :     if (DatumGetPointer(new_value) != NULL)
     236             :     {
     237             :         /* successful compression */
     238       32574 :         if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     239         122 :             pfree(DatumGetPointer(*value));
     240       32574 :         *value = new_value;
     241       32574 :         attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
     242       32574 :         attr->tai_size = VARSIZE(DatumGetPointer(*value));
     243       32574 :         ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     244             :     }
     245             :     else
     246             :     {
     247             :         /* incompressible, ignore on subsequent compression passes */
     248        1446 :         attr->tai_colflags |= TOASTCOL_INCOMPRESSIBLE;
     249             :     }
     250       34020 : }
     251             : 
     252             : /*
     253             :  * Move an attribute to external storage.
     254             :  */
     255             : void
     256       15310 : toast_tuple_externalize(ToastTupleContext *ttc, int attribute, int options)
     257             : {
     258       15310 :     Datum      *value = &ttc->ttc_values[attribute];
     259       15310 :     Datum       old_value = *value;
     260       15310 :     ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
     261             : 
     262       15310 :     attr->tai_colflags |= TOASTCOL_IGNORE;
     263       15310 :     *value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal,
     264             :                               options);
     265       15310 :     if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     266        9416 :         pfree(DatumGetPointer(old_value));
     267       15310 :     attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
     268       15310 :     ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     269       15310 : }
     270             : 
     271             : /*
     272             :  * Perform appropriate cleanup after one tuple has been subjected to TOAST.
     273             :  */
     274             : void
     275       37036 : toast_tuple_cleanup(ToastTupleContext *ttc)
     276             : {
     277       37036 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
     278       37036 :     int         numAttrs = tupleDesc->natts;
     279             : 
     280             :     /*
     281             :      * Free allocated temp values
     282             :      */
     283       37036 :     if ((ttc->ttc_flags & TOAST_NEEDS_FREE) != 0)
     284             :     {
     285             :         int         i;
     286             : 
     287      347614 :         for (i = 0; i < numAttrs; i++)
     288             :         {
     289      310720 :             ToastAttrInfo *attr = &ttc->ttc_attr[i];
     290             : 
     291      310720 :             if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     292       39284 :                 pfree(DatumGetPointer(ttc->ttc_values[i]));
     293             :         }
     294             :     }
     295             : 
     296             :     /*
     297             :      * Delete external values from the old tuple
     298             :      */
     299       37036 :     if ((ttc->ttc_flags & TOAST_NEEDS_DELETE_OLD) != 0)
     300             :     {
     301             :         int         i;
     302             : 
     303       12518 :         for (i = 0; i < numAttrs; i++)
     304             :         {
     305       12018 :             ToastAttrInfo *attr = &ttc->ttc_attr[i];
     306             : 
     307       12018 :             if ((attr->tai_colflags & TOASTCOL_NEEDS_DELETE_OLD) != 0)
     308         562 :                 toast_delete_datum(ttc->ttc_rel, ttc->ttc_oldvalues[i], false);
     309             :         }
     310             :     }
     311       37036 : }
     312             : 
     313             : /*
     314             :  * Check for external stored attributes and delete them from the secondary
     315             :  * relation.
     316             :  */
     317             : void
     318         580 : toast_delete_external(Relation rel, const Datum *values, const bool *isnull,
     319             :                       bool is_speculative)
     320             : {
     321         580 :     TupleDesc   tupleDesc = rel->rd_att;
     322         580 :     int         numAttrs = tupleDesc->natts;
     323             :     int         i;
     324             : 
     325        4742 :     for (i = 0; i < numAttrs; i++)
     326             :     {
     327        4162 :         if (TupleDescCompactAttr(tupleDesc, i)->attlen == -1)
     328             :         {
     329        1400 :             Datum       value = values[i];
     330             : 
     331        1400 :             if (isnull[i])
     332         394 :                 continue;
     333        1006 :             else if (VARATT_IS_EXTERNAL_ONDISK(value))
     334         600 :                 toast_delete_datum(rel, value, is_speculative);
     335             :         }
     336             :     }
     337         580 : }