Line data    Source code

       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * toast_helper.c
       4             :  *    Helper functions for table AMs implementing compressed or
       5             :  *    out-of-line storage of varlena attributes.
       6             :  *
       7             :  * Copyright (c) 2000-2025, PostgreSQL Global Development Group
       8             :  *
       9             :  * IDENTIFICATION
      10             :  *    src/backend/access/table/toast_helper.c
      11             :  *
      12             :  *-------------------------------------------------------------------------
      13             :  */
      14             : 
      15             : #include "postgres.h"
      16             : 
      17             : #include "access/detoast.h"
      18             : #include "access/toast_helper.h"
      19             : #include "access/toast_internals.h"
      20             : #include "catalog/pg_type_d.h"
      21             : #include "varatt.h"
      22             : 
      23             : 
      24             : /*
      25             :  * Prepare to TOAST a tuple.
      26             :  *
      27             :  * tupleDesc, toast_values, and toast_isnull are required parameters; they
      28             :  * provide the necessary details about the tuple to be toasted.
      29             :  *
      30             :  * toast_oldvalues and toast_oldisnull should be NULL for a newly-inserted
      31             :  * tuple; for an update, they should describe the existing tuple.
      32             :  *
      33             :  * All of these arrays should have a length equal to tupleDesc->natts.
      34             :  *
      35             :  * On return, toast_flags and toast_attr will have been initialized.
      36             :  * toast_flags is just a single uint8, but toast_attr is a caller-provided
      37             :  * array with a length equal to tupleDesc->natts.  The caller need not
      38             :  * perform any initialization of the array before calling this function.
      39             :  */
      40             : void
      41       39970 : toast_tuple_init(ToastTupleContext *ttc)
      42             : {
      43       39970 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
      44       39970 :     int         numAttrs = tupleDesc->natts;
      45             :     int         i;
      46             : 
      47       39970 :     ttc->ttc_flags = 0;
      48             : 
      49      401810 :     for (i = 0; i < numAttrs; i++)
      50             :     {
      51      361840 :         Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
      52             :         struct varlena *old_value;
      53             :         struct varlena *new_value;
      54             : 
      55      361840 :         ttc->ttc_attr[i].tai_colflags = 0;
      56      361840 :         ttc->ttc_attr[i].tai_oldexternal = NULL;
      57      361840 :         ttc->ttc_attr[i].tai_compression = att->attcompression;
      58             : 
      59      361840 :         if (ttc->ttc_oldvalues != NULL)
      60             :         {
      61             :             /*
      62             :              * For UPDATE get the old and new values of this attribute
      63             :              */
      64             :             old_value =
      65       86416 :                 (struct varlena *) DatumGetPointer(ttc->ttc_oldvalues[i]);
      66             :             new_value =
      67       86416 :                 (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
      68             : 
      69             :             /*
      70             :              * If the old value is stored on disk, check if it has changed so
      71             :              * we have to delete it later.
      72             :              */
      73       86416 :             if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
      74        9990 :                 VARATT_IS_EXTERNAL_ONDISK(old_value))
      75             :             {
      76         860 :                 if (ttc->ttc_isnull[i] ||
      77         842 :                     !VARATT_IS_EXTERNAL_ONDISK(new_value) ||
      78          96 :                     memcmp(old_value, new_value,
      79          96 :                            VARSIZE_EXTERNAL(old_value)) != 0)
      80             :                 {
      81             :                     /*
      82             :                      * The old external stored value isn't needed any more
      83             :                      * after the update
      84             :                      */
      85         766 :                     ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_DELETE_OLD;
      86         766 :                     ttc->ttc_flags |= TOAST_NEEDS_DELETE_OLD;
      87             :                 }
      88             :                 else
      89             :                 {
      90             :                     /*
      91             :                      * This attribute isn't changed by this update so we reuse
      92             :                      * the original reference to the old value in the new
      93             :                      * tuple.
      94             :                      */
      95          94 :                     ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
      96          94 :                     continue;
      97             :                 }
      98             :             }
      99             :         }
     100             :         else
     101             :         {
     102             :             /*
     103             :              * For INSERT simply get the new value
     104             :              */
     105      275424 :             new_value = (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
     106             :         }
     107             : 
     108             :         /*
     109             :          * Handle NULL attributes
     110             :          */
     111      361746 :         if (ttc->ttc_isnull[i])
     112             :         {
     113       39228 :             ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     114       39228 :             ttc->ttc_flags |= TOAST_HAS_NULLS;
     115       39228 :             continue;
     116             :         }
     117             : 
     118             :         /*
     119             :          * Now look at varlena attributes
     120             :          */
     121      322518 :         if (att->attlen == -1)
     122             :         {
     123             :             /*
     124             :              * If the table's attribute says PLAIN always, force it so.
     125             :              */
     126       74616 :             if (att->attstorage == TYPSTORAGE_PLAIN)
     127        2278 :                 ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     128             : 
     129             :             /*
     130             :              * We took care of UPDATE above, so any external value we find
     131             :              * still in the tuple must be someone else's that we cannot reuse
     132             :              * (this includes the case of an out-of-line in-memory datum).
     133             :              * Fetch it back (without decompression, unless we are forcing
     134             :              * PLAIN storage).  If necessary, we'll push it out as a new
     135             :              * external value below.
     136             :              */
     137       74616 :             if (VARATT_IS_EXTERNAL(new_value))
     138             :             {
     139         940 :                 ttc->ttc_attr[i].tai_oldexternal = new_value;
     140         940 :                 if (att->attstorage == TYPSTORAGE_PLAIN)
     141           0 :                     new_value = detoast_attr(new_value);
     142             :                 else
     143         940 :                     new_value = detoast_external_attr(new_value);
     144         940 :                 ttc->ttc_values[i] = PointerGetDatum(new_value);
     145         940 :                 ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_FREE;
     146         940 :                 ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     147             :             }
     148             : 
     149             :             /*
     150             :              * Remember the size of this attribute
     151             :              */
     152       74616 :             ttc->ttc_attr[i].tai_size = VARSIZE_ANY(new_value);
     153             :         }
     154             :         else
     155             :         {
     156             :             /*
     157             :              * Not a varlena attribute, plain storage always
     158             :              */
     159      247902 :             ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
     160             :         }
     161             :     }
     162       39970 : }
     163             : 
     164             : /*
     165             :  * Find the largest varlena attribute that satisfies certain criteria.
     166             :  *
     167             :  * The relevant column must not be marked TOASTCOL_IGNORE, and if the
     168             :  * for_compression flag is passed as true, it must also not be marked
     169             :  * TOASTCOL_INCOMPRESSIBLE.
     170             :  *
     171             :  * The column must have attstorage EXTERNAL or EXTENDED if check_main is
     172             :  * false, and must have attstorage MAIN if check_main is true.
     173             :  *
     174             :  * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
     175             :  * if not, no benefit is to be expected by compressing it.
     176             :  *
     177             :  * The return value is the index of the biggest suitable column, or
     178             :  * -1 if there is none.
     179             :  */
     180             : int
     181       46958 : toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
     182             :                                    bool for_compression, bool check_main)
     183             : {
     184       46958 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
     185       46958 :     int         numAttrs = tupleDesc->natts;
     186       46958 :     int         biggest_attno = -1;
     187       46958 :     int32       biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
     188       46958 :     int32       skip_colflags = TOASTCOL_IGNORE;
     189             :     int         i;
     190             : 
     191       46958 :     if (for_compression)
     192       45322 :         skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
     193             : 
     194      596310 :     for (i = 0; i < numAttrs; i++)
     195             :     {
     196      549352 :         Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
     197             : 
     198      549352 :         if ((ttc->ttc_attr[i].tai_colflags & skip_colflags) != 0)
     199      451916 :             continue;
     200       97436 :         if (VARATT_IS_EXTERNAL(DatumGetPointer(ttc->ttc_values[i])))
     201           0 :             continue;           /* can't happen, toast_action would be PLAIN */
     202       97436 :         if (for_compression &&
     203       92506 :             VARATT_IS_COMPRESSED(DatumGetPointer(ttc->ttc_values[i])))
     204        8508 :             continue;
     205       88928 :         if (check_main && att->attstorage != TYPSTORAGE_MAIN)
     206           0 :             continue;
     207       88928 :         if (!check_main && att->attstorage != TYPSTORAGE_EXTENDED &&
     208        5922 :             att->attstorage != TYPSTORAGE_EXTERNAL)
     209         112 :             continue;
     210             : 
     211       88816 :         if (ttc->ttc_attr[i].tai_size > biggest_size)
     212             :         {
     213       59000 :             biggest_attno = i;
     214       59000 :             biggest_size = ttc->ttc_attr[i].tai_size;
     215             :         }
     216             :     }
     217             : 
     218       46958 :     return biggest_attno;
     219             : }
     220             : 
     221             : /*
     222             :  * Try compression for an attribute.
     223             :  *
     224             :  * If we find that the attribute is not compressible, mark it so.
     225             :  */
     226             : void
     227       37894 : toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
     228             : {
     229       37894 :     Datum      *value = &ttc->ttc_values[attribute];
     230             :     Datum       new_value;
     231       37894 :     ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
     232             : 
     233       37894 :     new_value = toast_compress_datum(*value, attr->tai_compression);
     234             : 
     235       37894 :     if (DatumGetPointer(new_value) != NULL)
     236             :     {
     237             :         /* successful compression */
     238       35982 :         if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     239         122 :             pfree(DatumGetPointer(*value));
     240       35982 :         *value = new_value;
     241       35982 :         attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
     242       35982 :         attr->tai_size = VARSIZE(DatumGetPointer(*value));
     243       35982 :         ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     244             :     }
     245             :     else
     246             :     {
     247             :         /* incompressible, ignore on subsequent compression passes */
     248        1912 :         attr->tai_colflags |= TOASTCOL_INCOMPRESSIBLE;
     249             :     }
     250       37894 : }
     251             : 
     252             : /*
     253             :  * Move an attribute to external storage.
     254             :  */
     255             : void
     256       16530 : toast_tuple_externalize(ToastTupleContext *ttc, int attribute, int options)
     257             : {
     258       16530 :     Datum      *value = &ttc->ttc_values[attribute];
     259       16530 :     Datum       old_value = *value;
     260       16530 :     ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
     261             : 
     262       16530 :     attr->tai_colflags |= TOASTCOL_IGNORE;
     263       16530 :     *value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal,
     264             :                               options);
     265       16530 :     if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     266       10432 :         pfree(DatumGetPointer(old_value));
     267       16530 :     attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
     268       16530 :     ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
     269       16530 : }
     270             : 
     271             : /*
     272             :  * Perform appropriate cleanup after one tuple has been subjected to TOAST.
     273             :  */
     274             : void
     275       39970 : toast_tuple_cleanup(ToastTupleContext *ttc)
     276             : {
     277       39970 :     TupleDesc   tupleDesc = ttc->ttc_rel->rd_att;
     278       39970 :     int         numAttrs = tupleDesc->natts;
     279             : 
     280             :     /*
     281             :      * Free allocated temp values
     282             :      */
     283       39970 :     if ((ttc->ttc_flags & TOAST_NEEDS_FREE) != 0)
     284             :     {
     285             :         int         i;
     286             : 
     287      400114 :         for (i = 0; i < numAttrs; i++)
     288             :         {
     289      360286 :             ToastAttrInfo *attr = &ttc->ttc_attr[i];
     290             : 
     291      360286 :             if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
     292       42898 :                 pfree(DatumGetPointer(ttc->ttc_values[i]));
     293             :         }
     294             :     }
     295             : 
     296             :     /*
     297             :      * Delete external values from the old tuple
     298             :      */
     299       39970 :     if ((ttc->ttc_flags & TOAST_NEEDS_DELETE_OLD) != 0)
     300             :     {
     301             :         int         i;
     302             : 
     303       18278 :         for (i = 0; i < numAttrs; i++)
     304             :         {
     305       17598 :             ToastAttrInfo *attr = &ttc->ttc_attr[i];
     306             : 
     307       17598 :             if ((attr->tai_colflags & TOASTCOL_NEEDS_DELETE_OLD) != 0)
     308         766 :                 toast_delete_datum(ttc->ttc_rel, ttc->ttc_oldvalues[i], false);
     309             :         }
     310             :     }
     311       39970 : }
     312             : 
     313             : /*
     314             :  * Check for external stored attributes and delete them from the secondary
     315             :  * relation.
     316             :  */
     317             : void
     318         588 : toast_delete_external(Relation rel, const Datum *values, const bool *isnull,
     319             :                       bool is_speculative)
     320             : {
     321         588 :     TupleDesc   tupleDesc = rel->rd_att;
     322         588 :     int         numAttrs = tupleDesc->natts;
     323             :     int         i;
     324             : 
     325        4814 :     for (i = 0; i < numAttrs; i++)
     326             :     {
     327        4226 :         if (TupleDescCompactAttr(tupleDesc, i)->attlen == -1)
     328             :         {
     329        1416 :             Datum       value = values[i];
     330             : 
     331        1416 :             if (isnull[i])
     332         394 :                 continue;
     333        1022 :             else if (VARATT_IS_EXTERNAL_ONDISK(value))
     334         608 :                 toast_delete_datum(rel, value, is_speculative);
     335             :         }
     336             :     }
     337         588 : }