LCOV - code coverage report
Current view: top level - src/backend/replication - backup_manifest.c (source / functions) Hit Total Coverage
Test: PostgreSQL 14devel Lines: 105 114 92.1 %
Date: 2020-07-11 21:06:30 Functions: 6 6 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * backup_manifest.c
       4             :  *    code for generating and sending a backup manifest
       5             :  *
       6             :  * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/replication/backup_manifest.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : #include "postgres.h"
      14             : 
      15             : #include "access/timeline.h"
      16             : #include "libpq/libpq.h"
      17             : #include "libpq/pqformat.h"
      18             : #include "mb/pg_wchar.h"
      19             : #include "replication/backup_manifest.h"
      20             : #include "utils/builtins.h"
      21             : #include "utils/json.h"
      22             : 
      23             : static void AppendStringToManifest(backup_manifest_info *manifest, char *s);
      24             : 
      25             : /*
      26             :  * Does the user want a backup manifest?
      27             :  *
      28             :  * It's simplest to always have a manifest_info object, so that we don't need
      29             :  * checks for NULL pointers in too many places. However, if the user doesn't
      30             :  * want a manifest, we set manifest->buffile to NULL.
      31             :  */
      32             : static inline bool
      33      130686 : IsManifestEnabled(backup_manifest_info *manifest)
      34             : {
      35      130686 :     return (manifest->buffile != NULL);
      36             : }
      37             : 
      38             : /*
      39             :  * Convenience macro for appending data to the backup manifest.
      40             :  */
      41             : #define AppendToManifest(manifest, ...) \
      42             :     { \
      43             :         char *_manifest_s = psprintf(__VA_ARGS__);  \
      44             :         AppendStringToManifest(manifest, _manifest_s);  \
      45             :         pfree(_manifest_s); \
      46             :     }
      47             : 
      48             : /*
      49             :  * Initialize state so that we can construct a backup manifest.
      50             :  *
      51             :  * NB: Although the checksum type for the data files is configurable, the
      52             :  * checksum for the manifest itself always uses SHA-256. See comments in
      53             :  * SendBackupManifest.
      54             :  */
      55             : void
      56         144 : InitializeBackupManifest(backup_manifest_info *manifest,
      57             :                          backup_manifest_option want_manifest,
      58             :                          pg_checksum_type manifest_checksum_type)
      59             : {
      60         144 :     if (want_manifest == MANIFEST_OPTION_NO)
      61           2 :         manifest->buffile = NULL;
      62             :     else
      63         142 :         manifest->buffile = BufFileCreateTemp(false);
      64         144 :     manifest->checksum_type = manifest_checksum_type;
      65         144 :     pg_sha256_init(&manifest->manifest_ctx);
      66         144 :     manifest->manifest_size = UINT64CONST(0);
      67         144 :     manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
      68         144 :     manifest->first_file = true;
      69         144 :     manifest->still_checksumming = true;
      70             : 
      71         144 :     if (want_manifest != MANIFEST_OPTION_NO)
      72         142 :         AppendToManifest(manifest,
      73             :                          "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
      74             :                          "\"Files\": [");
      75         144 : }
      76             : 
      77             : /*
      78             :  * Add an entry to the backup manifest for a file.
      79             :  */
      80             : void
      81      130410 : AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid,
      82             :                         const char *pathname, size_t size, pg_time_t mtime,
      83             :                         pg_checksum_context *checksum_ctx)
      84             : {
      85             :     char        pathbuf[MAXPGPATH];
      86             :     int         pathlen;
      87             :     StringInfoData buf;
      88             : 
      89      130410 :     if (!IsManifestEnabled(manifest))
      90        1870 :         return;
      91             : 
      92             :     /*
      93             :      * If this file is part of a tablespace, the pathname passed to this
      94             :      * function will be relative to the tar file that contains it. We want the
      95             :      * pathname relative to the data directory (ignoring the intermediate
      96             :      * symlink traversal).
      97             :      */
      98      128540 :     if (spcoid != NULL)
      99             :     {
     100          32 :         snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
     101             :                  pathname);
     102          32 :         pathname = pathbuf;
     103             :     }
     104             : 
     105             :     /*
     106             :      * Each file's entry needs to be separated from any entry that follows by
     107             :      * a comma, but there's no comma before the first one or after the last
     108             :      * one. To make that work, adding a file to the manifest starts by
     109             :      * terminating the most recently added line, with a comma if appropriate,
     110             :      * but does not terminate the line inserted for this file.
     111             :      */
     112      128540 :     initStringInfo(&buf);
     113      128540 :     if (manifest->first_file)
     114             :     {
     115         142 :         appendStringInfoString(&buf, "\n");
     116         142 :         manifest->first_file = false;
     117             :     }
     118             :     else
     119      128398 :         appendStringInfoString(&buf, ",\n");
     120             : 
     121             :     /*
     122             :      * Write the relative pathname to this file out to the manifest. The
     123             :      * manifest is always stored in UTF-8, so we have to encode paths that are
     124             :      * not valid in that encoding.
     125             :      */
     126      128540 :     pathlen = strlen(pathname);
     127      255212 :     if (!manifest->force_encode &&
     128      126672 :         pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
     129             :     {
     130      126672 :         appendStringInfoString(&buf, "{ \"Path\": ");
     131      126672 :         escape_json(&buf, pathname);
     132      126672 :         appendStringInfoString(&buf, ", ");
     133             :     }
     134             :     else
     135             :     {
     136        1868 :         appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
     137        1868 :         enlargeStringInfo(&buf, 2 * pathlen);
     138        3736 :         buf.len += hex_encode(pathname, pathlen,
     139        1868 :                               &buf.data[buf.len]);
     140        1868 :         appendStringInfoString(&buf, "\", ");
     141             :     }
     142             : 
     143      128540 :     appendStringInfo(&buf, "\"Size\": %zu, ", size);
     144             : 
     145             :     /*
     146             :      * Convert last modification time to a string and append it to the
     147             :      * manifest. Since it's not clear what time zone to use and since time
     148             :      * zone definitions can change, possibly causing confusion, use GMT
     149             :      * always.
     150             :      */
     151      128540 :     appendStringInfoString(&buf, "\"Last-Modified\": \"");
     152      128540 :     enlargeStringInfo(&buf, 128);
     153      128540 :     buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
     154      128540 :                            pg_gmtime(&mtime));
     155      128540 :     appendStringInfoString(&buf, "\"");
     156             : 
     157             :     /* Add checksum information. */
     158      128540 :     if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
     159             :     {
     160             :         uint8       checksumbuf[PG_CHECKSUM_MAX_LENGTH];
     161             :         int         checksumlen;
     162             : 
     163      126672 :         checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
     164             : 
     165      126672 :         appendStringInfo(&buf,
     166             :                          ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
     167             :                          pg_checksum_type_name(checksum_ctx->type));
     168      126672 :         enlargeStringInfo(&buf, 2 * checksumlen);
     169      253344 :         buf.len += hex_encode((char *) checksumbuf, checksumlen,
     170      126672 :                               &buf.data[buf.len]);
     171      126672 :         appendStringInfoString(&buf, "\"");
     172             :     }
     173             : 
     174             :     /* Close out the object. */
     175      128540 :     appendStringInfoString(&buf, " }");
     176             : 
     177             :     /* OK, add it to the manifest. */
     178      128540 :     AppendStringToManifest(manifest, buf.data);
     179             : 
     180             :     /* Avoid leaking memory. */
     181      128540 :     pfree(buf.data);
     182             : }
     183             : 
     184             : /*
     185             :  * Add information about the WAL that will need to be replayed when restoring
     186             :  * this backup to the manifest.
     187             :  */
     188             : void
     189         138 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
     190             :                            TimeLineID starttli, XLogRecPtr endptr,
     191             :                            TimeLineID endtli)
     192             : {
     193             :     List       *timelines;
     194             :     ListCell   *lc;
     195         138 :     bool        first_wal_range = true;
     196         138 :     bool        found_start_timeline = false;
     197             : 
     198         138 :     if (!IsManifestEnabled(manifest))
     199           2 :         return;
     200             : 
     201             :     /* Terminate the list of files. */
     202         136 :     AppendStringToManifest(manifest, "\n],\n");
     203             : 
     204             :     /* Read the timeline history for the ending timeline. */
     205         136 :     timelines = readTimeLineHistory(endtli);
     206             : 
     207             :     /* Start a list of LSN ranges. */
     208         136 :     AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
     209             : 
     210         136 :     foreach(lc, timelines)
     211             :     {
     212         136 :         TimeLineHistoryEntry *entry = lfirst(lc);
     213             :         XLogRecPtr  tl_beginptr;
     214             : 
     215             :         /*
     216             :          * We only care about timelines that were active during the backup.
     217             :          * Skip any that ended before the backup started. (Note that if
     218             :          * entry->end is InvalidXLogRecPtr, it means that the timeline has not
     219             :          * yet ended.)
     220             :          */
     221         136 :         if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
     222           0 :             continue;
     223             : 
     224             :         /*
     225             :          * Because the timeline history file lists newer timelines before
     226             :          * older ones, the first timeline we encounter that is new enough to
     227             :          * matter ought to match the ending timeline of the backup.
     228             :          */
     229         136 :         if (first_wal_range && endtli != entry->tli)
     230           0 :             ereport(ERROR,
     231             :                     errmsg("expected end timeline %u but found timeline %u",
     232             :                            starttli, entry->tli));
     233             : 
     234         136 :         if (!XLogRecPtrIsInvalid(entry->begin))
     235           0 :             tl_beginptr = entry->begin;
     236             :         else
     237             :         {
     238         136 :             tl_beginptr = startptr;
     239             : 
     240             :             /*
     241             :              * If we reach a TLI that has no valid beginning LSN, there can't
     242             :              * be any more timelines in the history after this point, so we'd
     243             :              * better have arrived at the expected starting TLI. If not,
     244             :              * something's gone horribly wrong.
     245             :              */
     246         136 :             if (starttli != entry->tli)
     247           0 :                 ereport(ERROR,
     248             :                         errmsg("expected start timeline %u but found timeline %u",
     249             :                                starttli, entry->tli));
     250             :         }
     251             : 
     252         136 :         AppendToManifest(manifest,
     253             :                          "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
     254             :                          first_wal_range ? "" : ",\n",
     255             :                          entry->tli,
     256             :                          (uint32) (tl_beginptr >> 32), (uint32) tl_beginptr,
     257             :                          (uint32) (endptr >> 32), (uint32) endptr);
     258             : 
     259         136 :         if (starttli == entry->tli)
     260             :         {
     261         136 :             found_start_timeline = true;
     262         136 :             break;
     263             :         }
     264             : 
     265           0 :         endptr = entry->begin;
     266           0 :         first_wal_range = false;
     267             :     }
     268             : 
     269             :     /*
     270             :      * The last entry in the timeline history for the ending timeline should
     271             :      * be the ending timeline itself. Verify that this is what we observed.
     272             :      */
     273         136 :     if (!found_start_timeline)
     274           0 :         ereport(ERROR,
     275             :                 errmsg("start timeline %u not found history of timeline %u",
     276             :                        starttli, endtli));
     277             : 
     278             :     /* Terminate the list of WAL ranges. */
     279         136 :     AppendStringToManifest(manifest, "\n],\n");
     280             : }
     281             : 
     282             : /*
     283             :  * Finalize the backup manifest, and send it to the client.
     284             :  */
     285             : void
     286         138 : SendBackupManifest(backup_manifest_info *manifest)
     287             : {
     288             :     StringInfoData protobuf;
     289             :     uint8       checksumbuf[PG_SHA256_DIGEST_LENGTH];
     290             :     char        checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
     291         138 :     size_t      manifest_bytes_done = 0;
     292             : 
     293         138 :     if (!IsManifestEnabled(manifest))
     294           2 :         return;
     295             : 
     296             :     /*
     297             :      * Append manifest checksum, so that the problems with the manifest itself
     298             :      * can be detected.
     299             :      *
     300             :      * We always use SHA-256 for this, regardless of what algorithm is chosen
     301             :      * for checksumming the files.  If we ever want to make the checksum
     302             :      * algorithm used for the manifest file variable, the client will need a
     303             :      * way to figure out which algorithm to use as close to the beginning of
     304             :      * the manifest file as possible, to avoid having to read the whole thing
     305             :      * twice.
     306             :      */
     307         136 :     manifest->still_checksumming = false;
     308         136 :     pg_sha256_final(&manifest->manifest_ctx, checksumbuf);
     309         136 :     AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
     310         136 :     hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
     311         136 :     checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
     312         136 :     AppendStringToManifest(manifest, checksumstringbuf);
     313         136 :     AppendStringToManifest(manifest, "\"}\n");
     314             : 
     315             :     /*
     316             :      * We've written all the data to the manifest file.  Rewind the file so
     317             :      * that we can read it all back.
     318             :      */
     319         136 :     if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
     320           0 :         ereport(ERROR,
     321             :                 (errcode_for_file_access(),
     322             :                  errmsg("could not rewind temporary file")));
     323             : 
     324             :     /* Send CopyOutResponse message */
     325         136 :     pq_beginmessage(&protobuf, 'H');
     326         136 :     pq_sendbyte(&protobuf, 0);  /* overall format */
     327         136 :     pq_sendint16(&protobuf, 0); /* natts */
     328         136 :     pq_endmessage(&protobuf);
     329             : 
     330             :     /*
     331             :      * Send CopyData messages.
     332             :      *
     333             :      * We choose to read back the data from the temporary file in chunks of
     334             :      * size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
     335             :      * size, so it seems to make sense to match that value here.
     336             :      */
     337        2532 :     while (manifest_bytes_done < manifest->manifest_size)
     338             :     {
     339             :         char        manifestbuf[BLCKSZ];
     340             :         size_t      bytes_to_read;
     341             :         size_t      rc;
     342             : 
     343        2396 :         bytes_to_read = Min(sizeof(manifestbuf),
     344             :                             manifest->manifest_size - manifest_bytes_done);
     345        2396 :         rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
     346        2396 :         if (rc != bytes_to_read)
     347           0 :             ereport(ERROR,
     348             :                     (errcode_for_file_access(),
     349             :                      errmsg("could not read from temporary file: %m")));
     350        2396 :         pq_putmessage('d', manifestbuf, bytes_to_read);
     351        2396 :         manifest_bytes_done += bytes_to_read;
     352             :     }
     353             : 
     354             :     /* No more data, so send CopyDone message */
     355         136 :     pq_putemptymessage('c');
     356             : 
     357             :     /* Release resources */
     358         136 :     BufFileClose(manifest->buffile);
     359             : }
     360             : 
     361             : /*
     362             :  * Append a cstring to the manifest.
     363             :  */
     364             : static void
     365      129634 : AppendStringToManifest(backup_manifest_info *manifest, char *s)
     366             : {
     367      129634 :     int         len = strlen(s);
     368             : 
     369             :     Assert(manifest != NULL);
     370      129634 :     if (manifest->still_checksumming)
     371      129226 :         pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len);
     372      129634 :     BufFileWrite(manifest->buffile, s, len);
     373      129634 :     manifest->manifest_size += len;
     374      129634 : }

Generated by: LCOV version 1.13