LCOV - code coverage report
Current view: top level - src/backend/access/common - toast_compression.c (source / functions) Hit Total Coverage
Test: PostgreSQL 16beta1 Lines: 80 90 88.9 %
Date: 2023-06-02 19:12:20 Functions: 9 9 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * toast_compression.c
       4             :  *    Functions for toast compression.
       5             :  *
       6             :  * Copyright (c) 2021-2023, PostgreSQL Global Development Group
       7             :  *
       8             :  *
       9             :  * IDENTIFICATION
      10             :  *    src/backend/access/common/toast_compression.c
      11             :  *
      12             :  *-------------------------------------------------------------------------
      13             :  */
      14             : #include "postgres.h"
      15             : 
      16             : #ifdef USE_LZ4
      17             : #include <lz4.h>
      18             : #endif
      19             : 
      20             : #include "access/detoast.h"
      21             : #include "access/toast_compression.h"
      22             : #include "common/pg_lzcompress.h"
      23             : #include "fmgr.h"
      24             : #include "utils/builtins.h"
      25             : #include "varatt.h"
      26             : 
      27             : /* GUC */
      28             : int         default_toast_compression = TOAST_PGLZ_COMPRESSION;
      29             : 
      30             : #define NO_LZ4_SUPPORT() \
      31             :     ereport(ERROR, \
      32             :             (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), \
      33             :              errmsg("compression method lz4 not supported"), \
      34             :              errdetail("This functionality requires the server to be built with lz4 support.")))
      35             : 
      36             : /*
      37             :  * Compress a varlena using PGLZ.
      38             :  *
      39             :  * Returns the compressed varlena, or NULL if compression fails.
      40             :  */
      41             : struct varlena *
      42      135936 : pglz_compress_datum(const struct varlena *value)
      43             : {
      44             :     int32       valsize,
      45             :                 len;
      46      135936 :     struct varlena *tmp = NULL;
      47             : 
      48      135936 :     valsize = VARSIZE_ANY_EXHDR(value);
      49             : 
      50             :     /*
      51             :      * No point in wasting a palloc cycle if value size is outside the allowed
      52             :      * range for compression.
      53             :      */
      54      135936 :     if (valsize < PGLZ_strategy_default->min_input_size ||
      55      133944 :         valsize > PGLZ_strategy_default->max_input_size)
      56        1992 :         return NULL;
      57             : 
      58             :     /*
      59             :      * Figure out the maximum possible size of the pglz output, add the bytes
      60             :      * that will be needed for varlena overhead, and allocate that amount.
      61             :      */
      62      133944 :     tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) +
      63             :                                     VARHDRSZ_COMPRESSED);
      64             : 
      65      133944 :     len = pglz_compress(VARDATA_ANY(value),
      66             :                         valsize,
      67             :                         (char *) tmp + VARHDRSZ_COMPRESSED,
      68             :                         NULL);
      69      133944 :     if (len < 0)
      70             :     {
      71        3772 :         pfree(tmp);
      72        3772 :         return NULL;
      73             :     }
      74             : 
      75      130172 :     SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
      76             : 
      77      130172 :     return tmp;
      78             : }
      79             : 
      80             : /*
      81             :  * Decompress a varlena that was compressed using PGLZ.
      82             :  */
      83             : struct varlena *
      84      169792 : pglz_decompress_datum(const struct varlena *value)
      85             : {
      86             :     struct varlena *result;
      87             :     int32       rawsize;
      88             : 
      89             :     /* allocate memory for the uncompressed data */
      90      169792 :     result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
      91             : 
      92             :     /* decompress the data */
      93      169792 :     rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
      94      169792 :                               VARSIZE(value) - VARHDRSZ_COMPRESSED,
      95      169792 :                               VARDATA(result),
      96      169792 :                               VARDATA_COMPRESSED_GET_EXTSIZE(value), true);
      97      169792 :     if (rawsize < 0)
      98           0 :         ereport(ERROR,
      99             :                 (errcode(ERRCODE_DATA_CORRUPTED),
     100             :                  errmsg_internal("compressed pglz data is corrupt")));
     101             : 
     102      169792 :     SET_VARSIZE(result, rawsize + VARHDRSZ);
     103             : 
     104      169792 :     return result;
     105             : }
     106             : 
     107             : /*
     108             :  * Decompress part of a varlena that was compressed using PGLZ.
     109             :  */
     110             : struct varlena *
     111          66 : pglz_decompress_datum_slice(const struct varlena *value,
     112             :                             int32 slicelength)
     113             : {
     114             :     struct varlena *result;
     115             :     int32       rawsize;
     116             : 
     117             :     /* allocate memory for the uncompressed data */
     118          66 :     result = (struct varlena *) palloc(slicelength + VARHDRSZ);
     119             : 
     120             :     /* decompress the data */
     121          66 :     rawsize = pglz_decompress((char *) value + VARHDRSZ_COMPRESSED,
     122          66 :                               VARSIZE(value) - VARHDRSZ_COMPRESSED,
     123          66 :                               VARDATA(result),
     124             :                               slicelength, false);
     125          66 :     if (rawsize < 0)
     126           0 :         ereport(ERROR,
     127             :                 (errcode(ERRCODE_DATA_CORRUPTED),
     128             :                  errmsg_internal("compressed pglz data is corrupt")));
     129             : 
     130          66 :     SET_VARSIZE(result, rawsize + VARHDRSZ);
     131             : 
     132          66 :     return result;
     133             : }
     134             : 
     135             : /*
     136             :  * Compress a varlena using LZ4.
     137             :  *
     138             :  * Returns the compressed varlena, or NULL if compression fails.
     139             :  */
     140             : struct varlena *
     141          36 : lz4_compress_datum(const struct varlena *value)
     142             : {
     143             : #ifndef USE_LZ4
     144             :     NO_LZ4_SUPPORT();
     145             :     return NULL;                /* keep compiler quiet */
     146             : #else
     147             :     int32       valsize;
     148             :     int32       len;
     149             :     int32       max_size;
     150          36 :     struct varlena *tmp = NULL;
     151             : 
     152          36 :     valsize = VARSIZE_ANY_EXHDR(value);
     153             : 
     154             :     /*
     155             :      * Figure out the maximum possible size of the LZ4 output, add the bytes
     156             :      * that will be needed for varlena overhead, and allocate that amount.
     157             :      */
     158          36 :     max_size = LZ4_compressBound(valsize);
     159          36 :     tmp = (struct varlena *) palloc(max_size + VARHDRSZ_COMPRESSED);
     160             : 
     161          36 :     len = LZ4_compress_default(VARDATA_ANY(value),
     162             :                                (char *) tmp + VARHDRSZ_COMPRESSED,
     163             :                                valsize, max_size);
     164          36 :     if (len <= 0)
     165           0 :         elog(ERROR, "lz4 compression failed");
     166             : 
     167             :     /* data is incompressible so just free the memory and return NULL */
     168          36 :     if (len > valsize)
     169             :     {
     170           0 :         pfree(tmp);
     171           0 :         return NULL;
     172             :     }
     173             : 
     174          36 :     SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_COMPRESSED);
     175             : 
     176          36 :     return tmp;
     177             : #endif
     178             : }
     179             : 
     180             : /*
     181             :  * Decompress a varlena that was compressed using LZ4.
     182             :  */
     183             : struct varlena *
     184         104 : lz4_decompress_datum(const struct varlena *value)
     185             : {
     186             : #ifndef USE_LZ4
     187             :     NO_LZ4_SUPPORT();
     188             :     return NULL;                /* keep compiler quiet */
     189             : #else
     190             :     int32       rawsize;
     191             :     struct varlena *result;
     192             : 
     193             :     /* allocate memory for the uncompressed data */
     194         104 :     result = (struct varlena *) palloc(VARDATA_COMPRESSED_GET_EXTSIZE(value) + VARHDRSZ);
     195             : 
     196             :     /* decompress the data */
     197         104 :     rawsize = LZ4_decompress_safe((char *) value + VARHDRSZ_COMPRESSED,
     198         104 :                                   VARDATA(result),
     199         104 :                                   VARSIZE(value) - VARHDRSZ_COMPRESSED,
     200         104 :                                   VARDATA_COMPRESSED_GET_EXTSIZE(value));
     201         104 :     if (rawsize < 0)
     202           0 :         ereport(ERROR,
     203             :                 (errcode(ERRCODE_DATA_CORRUPTED),
     204             :                  errmsg_internal("compressed lz4 data is corrupt")));
     205             : 
     206             : 
     207         104 :     SET_VARSIZE(result, rawsize + VARHDRSZ);
     208             : 
     209         104 :     return result;
     210             : #endif
     211             : }
     212             : 
     213             : /*
     214             :  * Decompress part of a varlena that was compressed using LZ4.
     215             :  */
     216             : struct varlena *
     217          18 : lz4_decompress_datum_slice(const struct varlena *value, int32 slicelength)
     218             : {
     219             : #ifndef USE_LZ4
     220             :     NO_LZ4_SUPPORT();
     221             :     return NULL;                /* keep compiler quiet */
     222             : #else
     223             :     int32       rawsize;
     224             :     struct varlena *result;
     225             : 
     226             :     /* slice decompression not supported prior to 1.8.3 */
     227          18 :     if (LZ4_versionNumber() < 10803)
     228           0 :         return lz4_decompress_datum(value);
     229             : 
     230             :     /* allocate memory for the uncompressed data */
     231          18 :     result = (struct varlena *) palloc(slicelength + VARHDRSZ);
     232             : 
     233             :     /* decompress the data */
     234          18 :     rawsize = LZ4_decompress_safe_partial((char *) value + VARHDRSZ_COMPRESSED,
     235          18 :                                           VARDATA(result),
     236          18 :                                           VARSIZE(value) - VARHDRSZ_COMPRESSED,
     237             :                                           slicelength,
     238             :                                           slicelength);
     239          18 :     if (rawsize < 0)
     240           0 :         ereport(ERROR,
     241             :                 (errcode(ERRCODE_DATA_CORRUPTED),
     242             :                  errmsg_internal("compressed lz4 data is corrupt")));
     243             : 
     244          18 :     SET_VARSIZE(result, rawsize + VARHDRSZ);
     245             : 
     246          18 :     return result;
     247             : #endif
     248             : }
     249             : 
     250             : /*
     251             :  * Extract compression ID from a varlena.
     252             :  *
     253             :  * Returns TOAST_INVALID_COMPRESSION_ID if the varlena is not compressed.
     254             :  */
     255             : ToastCompressionId
     256         162 : toast_get_compression_id(struct varlena *attr)
     257             : {
     258         162 :     ToastCompressionId cmid = TOAST_INVALID_COMPRESSION_ID;
     259             : 
     260             :     /*
     261             :      * If it is stored externally then fetch the compression method id from
     262             :      * the external toast pointer.  If compressed inline, fetch it from the
     263             :      * toast compression header.
     264             :      */
     265         162 :     if (VARATT_IS_EXTERNAL_ONDISK(attr))
     266          24 :     {
     267             :         struct varatt_external toast_pointer;
     268             : 
     269          24 :         VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
     270             : 
     271          24 :         if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
     272          24 :             cmid = VARATT_EXTERNAL_GET_COMPRESS_METHOD(toast_pointer);
     273             :     }
     274         138 :     else if (VARATT_IS_COMPRESSED(attr))
     275         132 :         cmid = VARDATA_COMPRESSED_GET_COMPRESS_METHOD(attr);
     276             : 
     277         162 :     return cmid;
     278             : }
     279             : 
     280             : /*
     281             :  * CompressionNameToMethod - Get compression method from compression name
     282             :  *
     283             :  * Search in the available built-in methods.  If the compression not found
     284             :  * in the built-in methods then return InvalidCompressionMethod.
     285             :  */
     286             : char
     287         136 : CompressionNameToMethod(const char *compression)
     288             : {
     289         136 :     if (strcmp(compression, "pglz") == 0)
     290          60 :         return TOAST_PGLZ_COMPRESSION;
     291          76 :     else if (strcmp(compression, "lz4") == 0)
     292             :     {
     293             : #ifndef USE_LZ4
     294             :         NO_LZ4_SUPPORT();
     295             : #endif
     296          64 :         return TOAST_LZ4_COMPRESSION;
     297             :     }
     298             : 
     299          12 :     return InvalidCompressionMethod;
     300             : }
     301             : 
     302             : /*
     303             :  * GetCompressionMethodName - Get compression method name
     304             :  */
     305             : const char *
     306          30 : GetCompressionMethodName(char method)
     307             : {
     308          30 :     switch (method)
     309             :     {
     310          12 :         case TOAST_PGLZ_COMPRESSION:
     311          12 :             return "pglz";
     312          18 :         case TOAST_LZ4_COMPRESSION:
     313          18 :             return "lz4";
     314           0 :         default:
     315           0 :             elog(ERROR, "invalid compression method %c", method);
     316             :             return NULL;        /* keep compiler quiet */
     317             :     }
     318             : }

Generated by: LCOV version 1.14