LCOV - code coverage report
Current view: top level - src/backend/backup - backup_manifest.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17beta1 Lines: 107 120 89.2 %
Date: 2024-05-29 11:10:47 Functions: 7 7 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * backup_manifest.c
       4             :  *    code for generating and sending a backup manifest
       5             :  *
       6             :  * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/backup/backup_manifest.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : #include "postgres.h"
      14             : 
      15             : #include "access/timeline.h"
      16             : #include "access/xlog.h"
      17             : #include "backup/backup_manifest.h"
      18             : #include "backup/basebackup_sink.h"
      19             : #include "mb/pg_wchar.h"
      20             : #include "utils/builtins.h"
      21             : #include "utils/json.h"
      22             : 
      23             : static void AppendStringToManifest(backup_manifest_info *manifest, const char *s);
      24             : 
      25             : /*
      26             :  * Does the user want a backup manifest?
      27             :  *
      28             :  * It's simplest to always have a manifest_info object, so that we don't need
      29             :  * checks for NULL pointers in too many places. However, if the user doesn't
      30             :  * want a manifest, we set manifest->buffile to NULL.
      31             :  */
      32             : static inline bool
      33      270304 : IsManifestEnabled(backup_manifest_info *manifest)
      34             : {
      35      270304 :     return (manifest->buffile != NULL);
      36             : }
      37             : 
      38             : /*
      39             :  * Convenience macro for appending data to the backup manifest.
      40             :  */
      41             : #define AppendToManifest(manifest, ...) \
      42             :     { \
      43             :         char *_manifest_s = psprintf(__VA_ARGS__);  \
      44             :         AppendStringToManifest(manifest, _manifest_s);  \
      45             :         pfree(_manifest_s); \
      46             :     }
      47             : 
      48             : /*
      49             :  * Initialize state so that we can construct a backup manifest.
      50             :  *
      51             :  * NB: Although the checksum type for the data files is configurable, the
      52             :  * checksum for the manifest itself always uses SHA-256. See comments in
      53             :  * SendBackupManifest.
      54             :  */
      55             : void
      56         280 : InitializeBackupManifest(backup_manifest_info *manifest,
      57             :                          backup_manifest_option want_manifest,
      58             :                          pg_checksum_type manifest_checksum_type)
      59             : {
      60         280 :     memset(manifest, 0, sizeof(backup_manifest_info));
      61         280 :     manifest->checksum_type = manifest_checksum_type;
      62             : 
      63         280 :     if (want_manifest == MANIFEST_OPTION_NO)
      64           2 :         manifest->buffile = NULL;
      65             :     else
      66             :     {
      67         278 :         manifest->buffile = BufFileCreateTemp(false);
      68         278 :         manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
      69         278 :         if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
      70           0 :             elog(ERROR, "failed to initialize checksum of backup manifest: %s",
      71             :                  pg_cryptohash_error(manifest->manifest_ctx));
      72             :     }
      73             : 
      74         280 :     manifest->manifest_size = UINT64CONST(0);
      75         280 :     manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
      76         280 :     manifest->first_file = true;
      77         280 :     manifest->still_checksumming = true;
      78             : 
      79         280 :     if (want_manifest != MANIFEST_OPTION_NO)
      80         278 :         AppendToManifest(manifest,
      81             :                          "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
      82             :                          "\"System-Identifier\": " UINT64_FORMAT ",\n"
      83             :                          "\"Files\": [",
      84             :                          GetSystemIdentifier());
      85         280 : }
      86             : 
      87             : /*
      88             :  * Free resources assigned to a backup manifest constructed.
      89             :  */
      90             : void
      91         262 : FreeBackupManifest(backup_manifest_info *manifest)
      92             : {
      93         262 :     pg_cryptohash_free(manifest->manifest_ctx);
      94         262 :     manifest->manifest_ctx = NULL;
      95         262 : }
      96             : 
      97             : /*
      98             :  * Add an entry to the backup manifest for a file.
      99             :  */
     100             : void
     101      269768 : AddFileToBackupManifest(backup_manifest_info *manifest, Oid spcoid,
     102             :                         const char *pathname, size_t size, pg_time_t mtime,
     103             :                         pg_checksum_context *checksum_ctx)
     104             : {
     105             :     char        pathbuf[MAXPGPATH];
     106             :     int         pathlen;
     107             :     StringInfoData buf;
     108             : 
     109      269768 :     if (!IsManifestEnabled(manifest))
     110        1934 :         return;
     111             : 
     112             :     /*
     113             :      * If this file is part of a tablespace, the pathname passed to this
     114             :      * function will be relative to the tar file that contains it. We want the
     115             :      * pathname relative to the data directory (ignoring the intermediate
     116             :      * symlink traversal).
     117             :      */
     118      267834 :     if (OidIsValid(spcoid))
     119             :     {
     120         680 :         snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%u/%s", spcoid,
     121             :                  pathname);
     122         680 :         pathname = pathbuf;
     123             :     }
     124             : 
     125             :     /*
     126             :      * Each file's entry needs to be separated from any entry that follows by
     127             :      * a comma, but there's no comma before the first one or after the last
     128             :      * one. To make that work, adding a file to the manifest starts by
     129             :      * terminating the most recently added line, with a comma if appropriate,
     130             :      * but does not terminate the line inserted for this file.
     131             :      */
     132      267834 :     initStringInfo(&buf);
     133      267834 :     if (manifest->first_file)
     134             :     {
     135         278 :         appendStringInfoChar(&buf, '\n');
     136         278 :         manifest->first_file = false;
     137             :     }
     138             :     else
     139      267556 :         appendStringInfoString(&buf, ",\n");
     140             : 
     141             :     /*
     142             :      * Write the relative pathname to this file out to the manifest. The
     143             :      * manifest is always stored in UTF-8, so we have to encode paths that are
     144             :      * not valid in that encoding.
     145             :      */
     146      267834 :     pathlen = strlen(pathname);
     147      533736 :     if (!manifest->force_encode &&
     148      265902 :         pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
     149             :     {
     150      265902 :         appendStringInfoString(&buf, "{ \"Path\": ");
     151      265902 :         escape_json(&buf, pathname);
     152      265902 :         appendStringInfoString(&buf, ", ");
     153             :     }
     154             :     else
     155             :     {
     156        1932 :         appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
     157        1932 :         enlargeStringInfo(&buf, 2 * pathlen);
     158        3864 :         buf.len += hex_encode(pathname, pathlen,
     159        1932 :                               &buf.data[buf.len]);
     160        1932 :         appendStringInfoString(&buf, "\", ");
     161             :     }
     162             : 
     163      267834 :     appendStringInfo(&buf, "\"Size\": %zu, ", size);
     164             : 
     165             :     /*
     166             :      * Convert last modification time to a string and append it to the
     167             :      * manifest. Since it's not clear what time zone to use and since time
     168             :      * zone definitions can change, possibly causing confusion, use GMT
     169             :      * always.
     170             :      */
     171      267834 :     appendStringInfoString(&buf, "\"Last-Modified\": \"");
     172      267834 :     enlargeStringInfo(&buf, 128);
     173      267834 :     buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
     174      267834 :                            pg_gmtime(&mtime));
     175      267834 :     appendStringInfoChar(&buf, '"');
     176             : 
     177             :     /* Add checksum information. */
     178      267834 :     if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
     179             :     {
     180             :         uint8       checksumbuf[PG_CHECKSUM_MAX_LENGTH];
     181             :         int         checksumlen;
     182             : 
     183      265902 :         checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
     184      265902 :         if (checksumlen < 0)
     185           0 :             elog(ERROR, "could not finalize checksum of file \"%s\"",
     186             :                  pathname);
     187             : 
     188      265902 :         appendStringInfo(&buf,
     189             :                          ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
     190             :                          pg_checksum_type_name(checksum_ctx->type));
     191      265902 :         enlargeStringInfo(&buf, 2 * checksumlen);
     192      531804 :         buf.len += hex_encode((char *) checksumbuf, checksumlen,
     193      265902 :                               &buf.data[buf.len]);
     194      265902 :         appendStringInfoChar(&buf, '"');
     195             :     }
     196             : 
     197             :     /* Close out the object. */
     198      267834 :     appendStringInfoString(&buf, " }");
     199             : 
     200             :     /* OK, add it to the manifest. */
     201      267834 :     AppendStringToManifest(manifest, buf.data);
     202             : 
     203             :     /* Avoid leaking memory. */
     204      267834 :     pfree(buf.data);
     205             : }
     206             : 
     207             : /*
     208             :  * Add information about the WAL that will need to be replayed when restoring
     209             :  * this backup to the manifest.
     210             :  */
     211             : void
     212         268 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
     213             :                            TimeLineID starttli, XLogRecPtr endptr,
     214             :                            TimeLineID endtli)
     215             : {
     216             :     List       *timelines;
     217             :     ListCell   *lc;
     218         268 :     bool        first_wal_range = true;
     219         268 :     bool        found_start_timeline = false;
     220             : 
     221         268 :     if (!IsManifestEnabled(manifest))
     222           2 :         return;
     223             : 
     224             :     /* Terminate the list of files. */
     225         266 :     AppendStringToManifest(manifest, "\n],\n");
     226             : 
     227             :     /* Read the timeline history for the ending timeline. */
     228         266 :     timelines = readTimeLineHistory(endtli);
     229             : 
     230             :     /* Start a list of LSN ranges. */
     231         266 :     AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
     232             : 
     233         266 :     foreach(lc, timelines)
     234             :     {
     235         266 :         TimeLineHistoryEntry *entry = lfirst(lc);
     236             :         XLogRecPtr  tl_beginptr;
     237             : 
     238             :         /*
     239             :          * We only care about timelines that were active during the backup.
     240             :          * Skip any that ended before the backup started. (Note that if
     241             :          * entry->end is InvalidXLogRecPtr, it means that the timeline has not
     242             :          * yet ended.)
     243             :          */
     244         266 :         if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
     245           0 :             continue;
     246             : 
     247             :         /*
     248             :          * Because the timeline history file lists newer timelines before
     249             :          * older ones, the first timeline we encounter that is new enough to
     250             :          * matter ought to match the ending timeline of the backup.
     251             :          */
     252         266 :         if (first_wal_range && endtli != entry->tli)
     253           0 :             ereport(ERROR,
     254             :                     errmsg("expected end timeline %u but found timeline %u",
     255             :                            starttli, entry->tli));
     256             : 
     257             :         /*
     258             :          * If this timeline entry matches with the timeline on which the
     259             :          * backup started, WAL needs to be checked from the start LSN of the
     260             :          * backup.  If this entry refers to a newer timeline, WAL needs to be
     261             :          * checked since the beginning of this timeline, so use the LSN where
     262             :          * the timeline began.
     263             :          */
     264         266 :         if (starttli == entry->tli)
     265         266 :             tl_beginptr = startptr;
     266             :         else
     267             :         {
     268           0 :             tl_beginptr = entry->begin;
     269             : 
     270             :             /*
     271             :              * If we reach a TLI that has no valid beginning LSN, there can't
     272             :              * be any more timelines in the history after this point, so we'd
     273             :              * better have arrived at the expected starting TLI. If not,
     274             :              * something's gone horribly wrong.
     275             :              */
     276           0 :             if (XLogRecPtrIsInvalid(entry->begin))
     277           0 :                 ereport(ERROR,
     278             :                         errmsg("expected start timeline %u but found timeline %u",
     279             :                                starttli, entry->tli));
     280             :         }
     281             : 
     282         266 :         AppendToManifest(manifest,
     283             :                          "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
     284             :                          first_wal_range ? "" : ",\n",
     285             :                          entry->tli,
     286             :                          LSN_FORMAT_ARGS(tl_beginptr),
     287             :                          LSN_FORMAT_ARGS(endptr));
     288             : 
     289         266 :         if (starttli == entry->tli)
     290             :         {
     291         266 :             found_start_timeline = true;
     292         266 :             break;
     293             :         }
     294             : 
     295           0 :         endptr = entry->begin;
     296           0 :         first_wal_range = false;
     297             :     }
     298             : 
     299             :     /*
     300             :      * The last entry in the timeline history for the ending timeline should
     301             :      * be the ending timeline itself. Verify that this is what we observed.
     302             :      */
     303         266 :     if (!found_start_timeline)
     304           0 :         ereport(ERROR,
     305             :                 errmsg("start timeline %u not found in history of timeline %u",
     306             :                        starttli, endtli));
     307             : 
     308             :     /* Terminate the list of WAL ranges. */
     309         266 :     AppendStringToManifest(manifest, "\n],\n");
     310             : }
     311             : 
     312             : /*
     313             :  * Finalize the backup manifest, and send it to the client.
     314             :  */
     315             : void
     316         268 : SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
     317             : {
     318             :     uint8       checksumbuf[PG_SHA256_DIGEST_LENGTH];
     319             :     char        checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
     320         268 :     size_t      manifest_bytes_done = 0;
     321             : 
     322         268 :     if (!IsManifestEnabled(manifest))
     323           2 :         return;
     324             : 
     325             :     /*
     326             :      * Append manifest checksum, so that the problems with the manifest itself
     327             :      * can be detected.
     328             :      *
     329             :      * We always use SHA-256 for this, regardless of what algorithm is chosen
     330             :      * for checksumming the files.  If we ever want to make the checksum
     331             :      * algorithm used for the manifest file variable, the client will need a
     332             :      * way to figure out which algorithm to use as close to the beginning of
     333             :      * the manifest file as possible, to avoid having to read the whole thing
     334             :      * twice.
     335             :      */
     336         266 :     manifest->still_checksumming = false;
     337         266 :     if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
     338             :                             sizeof(checksumbuf)) < 0)
     339           0 :         elog(ERROR, "failed to finalize checksum of backup manifest: %s",
     340             :              pg_cryptohash_error(manifest->manifest_ctx));
     341         266 :     AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
     342             : 
     343         266 :     hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
     344         266 :     checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
     345             : 
     346         266 :     AppendStringToManifest(manifest, checksumstringbuf);
     347         266 :     AppendStringToManifest(manifest, "\"}\n");
     348             : 
     349             :     /*
     350             :      * We've written all the data to the manifest file.  Rewind the file so
     351             :      * that we can read it all back.
     352             :      */
     353         266 :     if (BufFileSeek(manifest->buffile, 0, 0, SEEK_SET))
     354           0 :         ereport(ERROR,
     355             :                 (errcode_for_file_access(),
     356             :                  errmsg("could not rewind temporary file")));
     357             : 
     358             : 
     359             :     /*
     360             :      * Send the backup manifest.
     361             :      */
     362         266 :     bbsink_begin_manifest(sink);
     363        1638 :     while (manifest_bytes_done < manifest->manifest_size)
     364             :     {
     365             :         size_t      bytes_to_read;
     366             : 
     367        1372 :         bytes_to_read = Min(sink->bbs_buffer_length,
     368             :                             manifest->manifest_size - manifest_bytes_done);
     369        1372 :         BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
     370        1372 :         bbsink_manifest_contents(sink, bytes_to_read);
     371        1372 :         manifest_bytes_done += bytes_to_read;
     372             :     }
     373         266 :     bbsink_end_manifest(sink);
     374             : 
     375             :     /* Release resources */
     376         266 :     BufFileClose(manifest->buffile);
     377             : }
     378             : 
     379             : /*
     380             :  * Append a cstring to the manifest.
     381             :  */
     382             : static void
     383      269974 : AppendStringToManifest(backup_manifest_info *manifest, const char *s)
     384             : {
     385      269974 :     int         len = strlen(s);
     386             : 
     387             :     Assert(manifest != NULL);
     388      269974 :     if (manifest->still_checksumming)
     389             :     {
     390      269176 :         if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
     391           0 :             elog(ERROR, "failed to update checksum of backup manifest: %s",
     392             :                  pg_cryptohash_error(manifest->manifest_ctx));
     393             :     }
     394      269974 :     BufFileWrite(manifest->buffile, s, len);
     395      269974 :     manifest->manifest_size += len;
     396      269974 : }

Generated by: LCOV version 1.14