LCOV - code coverage report
Current view: top level - src/backend/replication - backup_manifest.c (source / functions) Hit Total Coverage
Test: PostgreSQL 14devel Lines: 106 115 92.2 %
Date: 2020-11-27 12:05:55 Functions: 6 6 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * backup_manifest.c
       4             :  *    code for generating and sending a backup manifest
       5             :  *
       6             :  * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/replication/backup_manifest.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : #include "postgres.h"
      14             : 
      15             : #include "access/timeline.h"
      16             : #include "libpq/libpq.h"
      17             : #include "libpq/pqformat.h"
      18             : #include "mb/pg_wchar.h"
      19             : #include "replication/backup_manifest.h"
      20             : #include "utils/builtins.h"
      21             : #include "utils/json.h"
      22             : 
      23             : static void AppendStringToManifest(backup_manifest_info *manifest, char *s);
      24             : 
      25             : /*
      26             :  * Does the user want a backup manifest?
      27             :  *
      28             :  * It's simplest to always have a manifest_info object, so that we don't need
      29             :  * checks for NULL pointers in too many places. However, if the user doesn't
      30             :  * want a manifest, we set manifest->buffile to NULL.
      31             :  */
      32             : static inline bool
      33      136342 : IsManifestEnabled(backup_manifest_info *manifest)
      34             : {
      35      136342 :     return (manifest->buffile != NULL);
      36             : }
      37             : 
      38             : /*
      39             :  * Convenience macro for appending data to the backup manifest.
      40             :  */
      41             : #define AppendToManifest(manifest, ...) \
      42             :     { \
      43             :         char *_manifest_s = psprintf(__VA_ARGS__);  \
      44             :         AppendStringToManifest(manifest, _manifest_s);  \
      45             :         pfree(_manifest_s); \
      46             :     }
      47             : 
      48             : /*
      49             :  * Initialize state so that we can construct a backup manifest.
      50             :  *
      51             :  * NB: Although the checksum type for the data files is configurable, the
      52             :  * checksum for the manifest itself always uses SHA-256. See comments in
      53             :  * SendBackupManifest.
      54             :  */
      55             : void
      56         150 : InitializeBackupManifest(backup_manifest_info *manifest,
      57             :                          backup_manifest_option want_manifest,
      58             :                          pg_checksum_type manifest_checksum_type)
      59             : {
      60         150 :     memset(manifest, 0, sizeof(backup_manifest_info));
      61         150 :     manifest->checksum_type = manifest_checksum_type;
      62             : 
      63         150 :     if (want_manifest == MANIFEST_OPTION_NO)
      64           2 :         manifest->buffile = NULL;
      65             :     else
      66             :     {
      67         148 :         manifest->buffile = BufFileCreateTemp(false);
      68         148 :         pg_sha256_init(&manifest->manifest_ctx);
      69             :     }
      70             : 
      71         150 :     manifest->manifest_size = UINT64CONST(0);
      72         150 :     manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
      73         150 :     manifest->first_file = true;
      74         150 :     manifest->still_checksumming = true;
      75             : 
      76         150 :     if (want_manifest != MANIFEST_OPTION_NO)
      77         148 :         AppendToManifest(manifest,
      78             :                          "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
      79             :                          "\"Files\": [");
      80         150 : }
      81             : 
      82             : /*
      83             :  * Add an entry to the backup manifest for a file.
      84             :  */
      85             : void
      86      136054 : AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid,
      87             :                         const char *pathname, size_t size, pg_time_t mtime,
      88             :                         pg_checksum_context *checksum_ctx)
      89             : {
      90             :     char        pathbuf[MAXPGPATH];
      91             :     int         pathlen;
      92             :     StringInfoData buf;
      93             : 
      94      136054 :     if (!IsManifestEnabled(manifest))
      95        1870 :         return;
      96             : 
      97             :     /*
      98             :      * If this file is part of a tablespace, the pathname passed to this
      99             :      * function will be relative to the tar file that contains it. We want the
     100             :      * pathname relative to the data directory (ignoring the intermediate
     101             :      * symlink traversal).
     102             :      */
     103      134184 :     if (spcoid != NULL)
     104             :     {
     105          32 :         snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
     106             :                  pathname);
     107          32 :         pathname = pathbuf;
     108             :     }
     109             : 
     110             :     /*
     111             :      * Each file's entry needs to be separated from any entry that follows by
     112             :      * a comma, but there's no comma before the first one or after the last
     113             :      * one. To make that work, adding a file to the manifest starts by
     114             :      * terminating the most recently added line, with a comma if appropriate,
     115             :      * but does not terminate the line inserted for this file.
     116             :      */
     117      134184 :     initStringInfo(&buf);
     118      134184 :     if (manifest->first_file)
     119             :     {
     120         148 :         appendStringInfoChar(&buf, '\n');
     121         148 :         manifest->first_file = false;
     122             :     }
     123             :     else
     124      134036 :         appendStringInfoString(&buf, ",\n");
     125             : 
     126             :     /*
     127             :      * Write the relative pathname to this file out to the manifest. The
     128             :      * manifest is always stored in UTF-8, so we have to encode paths that are
     129             :      * not valid in that encoding.
     130             :      */
     131      134184 :     pathlen = strlen(pathname);
     132      266500 :     if (!manifest->force_encode &&
     133      132316 :         pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
     134             :     {
     135      132316 :         appendStringInfoString(&buf, "{ \"Path\": ");
     136      132316 :         escape_json(&buf, pathname);
     137      132316 :         appendStringInfoString(&buf, ", ");
     138             :     }
     139             :     else
     140             :     {
     141        1868 :         appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
     142        1868 :         enlargeStringInfo(&buf, 2 * pathlen);
     143        3736 :         buf.len += hex_encode(pathname, pathlen,
     144        1868 :                               &buf.data[buf.len]);
     145        1868 :         appendStringInfoString(&buf, "\", ");
     146             :     }
     147             : 
     148      134184 :     appendStringInfo(&buf, "\"Size\": %zu, ", size);
     149             : 
     150             :     /*
     151             :      * Convert last modification time to a string and append it to the
     152             :      * manifest. Since it's not clear what time zone to use and since time
     153             :      * zone definitions can change, possibly causing confusion, use GMT
     154             :      * always.
     155             :      */
     156      134184 :     appendStringInfoString(&buf, "\"Last-Modified\": \"");
     157      134184 :     enlargeStringInfo(&buf, 128);
     158      134184 :     buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
     159      134184 :                            pg_gmtime(&mtime));
     160      134184 :     appendStringInfoChar(&buf, '"');
     161             : 
     162             :     /* Add checksum information. */
     163      134184 :     if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
     164             :     {
     165             :         uint8       checksumbuf[PG_CHECKSUM_MAX_LENGTH];
     166             :         int         checksumlen;
     167             : 
     168      132316 :         checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
     169             : 
     170      132316 :         appendStringInfo(&buf,
     171             :                          ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
     172             :                          pg_checksum_type_name(checksum_ctx->type));
     173      132316 :         enlargeStringInfo(&buf, 2 * checksumlen);
     174      264632 :         buf.len += hex_encode((char *) checksumbuf, checksumlen,
     175      132316 :                               &buf.data[buf.len]);
     176      132316 :         appendStringInfoChar(&buf, '"');
     177             :     }
     178             : 
     179             :     /* Close out the object. */
     180      134184 :     appendStringInfoString(&buf, " }");
     181             : 
     182             :     /* OK, add it to the manifest. */
     183      134184 :     AppendStringToManifest(manifest, buf.data);
     184             : 
     185             :     /* Avoid leaking memory. */
     186      134184 :     pfree(buf.data);
     187             : }
     188             : 
     189             : /*
     190             :  * Add information about the WAL that will need to be replayed when restoring
     191             :  * this backup to the manifest.
     192             :  */
     193             : void
     194         144 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
     195             :                            TimeLineID starttli, XLogRecPtr endptr,
     196             :                            TimeLineID endtli)
     197             : {
     198             :     List       *timelines;
     199             :     ListCell   *lc;
     200         144 :     bool        first_wal_range = true;
     201         144 :     bool        found_start_timeline = false;
     202             : 
     203         144 :     if (!IsManifestEnabled(manifest))
     204           2 :         return;
     205             : 
     206             :     /* Terminate the list of files. */
     207         142 :     AppendStringToManifest(manifest, "\n],\n");
     208             : 
     209             :     /* Read the timeline history for the ending timeline. */
     210         142 :     timelines = readTimeLineHistory(endtli);
     211             : 
     212             :     /* Start a list of LSN ranges. */
     213         142 :     AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
     214             : 
     215         142 :     foreach(lc, timelines)
     216             :     {
     217         142 :         TimeLineHistoryEntry *entry = lfirst(lc);
     218             :         XLogRecPtr  tl_beginptr;
     219             : 
     220             :         /*
     221             :          * We only care about timelines that were active during the backup.
     222             :          * Skip any that ended before the backup started. (Note that if
     223             :          * entry->end is InvalidXLogRecPtr, it means that the timeline has not
     224             :          * yet ended.)
     225             :          */
     226         142 :         if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
     227           0 :             continue;
     228             : 
     229             :         /*
     230             :          * Because the timeline history file lists newer timelines before
     231             :          * older ones, the first timeline we encounter that is new enough to
     232             :          * matter ought to match the ending timeline of the backup.
     233             :          */
     234         142 :         if (first_wal_range && endtli != entry->tli)
     235           0 :             ereport(ERROR,
     236             :                     errmsg("expected end timeline %u but found timeline %u",
     237             :                            starttli, entry->tli));
     238             : 
     239         142 :         if (!XLogRecPtrIsInvalid(entry->begin))
     240           0 :             tl_beginptr = entry->begin;
     241             :         else
     242             :         {
     243         142 :             tl_beginptr = startptr;
     244             : 
     245             :             /*
     246             :              * If we reach a TLI that has no valid beginning LSN, there can't
     247             :              * be any more timelines in the history after this point, so we'd
     248             :              * better have arrived at the expected starting TLI. If not,
     249             :              * something's gone horribly wrong.
     250             :              */
     251         142 :             if (starttli != entry->tli)
     252           0 :                 ereport(ERROR,
     253             :                         errmsg("expected start timeline %u but found timeline %u",
     254             :                                starttli, entry->tli));
     255             :         }
     256             : 
     257         142 :         AppendToManifest(manifest,
     258             :                          "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
     259             :                          first_wal_range ? "" : ",\n",
     260             :                          entry->tli,
     261             :                          (uint32) (tl_beginptr >> 32), (uint32) tl_beginptr,
     262             :                          (uint32) (endptr >> 32), (uint32) endptr);
     263             : 
     264         142 :         if (starttli == entry->tli)
     265             :         {
     266         142 :             found_start_timeline = true;
     267         142 :             break;
     268             :         }
     269             : 
     270           0 :         endptr = entry->begin;
     271           0 :         first_wal_range = false;
     272             :     }
     273             : 
     274             :     /*
     275             :      * The last entry in the timeline history for the ending timeline should
     276             :      * be the ending timeline itself. Verify that this is what we observed.
     277             :      */
     278         142 :     if (!found_start_timeline)
     279           0 :         ereport(ERROR,
     280             :                 errmsg("start timeline %u not found in history of timeline %u",
     281             :                        starttli, endtli));
     282             : 
     283             :     /* Terminate the list of WAL ranges. */
     284         142 :     AppendStringToManifest(manifest, "\n],\n");
     285             : }
     286             : 
     287             : /*
     288             :  * Finalize the backup manifest, and send it to the client.
     289             :  */
     290             : void
     291         144 : SendBackupManifest(backup_manifest_info *manifest)
     292             : {
     293             :     StringInfoData protobuf;
     294             :     uint8       checksumbuf[PG_SHA256_DIGEST_LENGTH];
     295             :     char        checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
     296         144 :     size_t      manifest_bytes_done = 0;
     297             : 
     298         144 :     if (!IsManifestEnabled(manifest))
     299           2 :         return;
     300             : 
     301             :     /*
     302             :      * Append manifest checksum, so that the problems with the manifest itself
     303             :      * can be detected.
     304             :      *
     305             :      * We always use SHA-256 for this, regardless of what algorithm is chosen
     306             :      * for checksumming the files.  If we ever want to make the checksum
     307             :      * algorithm used for the manifest file variable, the client will need a
     308             :      * way to figure out which algorithm to use as close to the beginning of
     309             :      * the manifest file as possible, to avoid having to read the whole thing
     310             :      * twice.
     311             :      */
     312         142 :     manifest->still_checksumming = false;
     313         142 :     pg_sha256_final(&manifest->manifest_ctx, checksumbuf);
     314         142 :     AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
     315         142 :     hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
     316         142 :     checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
     317         142 :     AppendStringToManifest(manifest, checksumstringbuf);
     318         142 :     AppendStringToManifest(manifest, "\"}\n");
     319             : 
     320             :     /*
     321             :      * We've written all the data to the manifest file.  Rewind the file so
     322             :      * that we can read it all back.
     323             :      */
     324         142 :     if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
     325           0 :         ereport(ERROR,
     326             :                 (errcode_for_file_access(),
     327             :                  errmsg("could not rewind temporary file")));
     328             : 
     329             :     /* Send CopyOutResponse message */
     330         142 :     pq_beginmessage(&protobuf, 'H');
     331         142 :     pq_sendbyte(&protobuf, 0);  /* overall format */
     332         142 :     pq_sendint16(&protobuf, 0); /* natts */
     333         142 :     pq_endmessage(&protobuf);
     334             : 
     335             :     /*
     336             :      * Send CopyData messages.
     337             :      *
     338             :      * We choose to read back the data from the temporary file in chunks of
     339             :      * size BLCKSZ; this isn't necessary, but buffile.c uses that as the I/O
     340             :      * size, so it seems to make sense to match that value here.
     341             :      */
     342        2640 :     while (manifest_bytes_done < manifest->manifest_size)
     343             :     {
     344             :         char        manifestbuf[BLCKSZ];
     345             :         size_t      bytes_to_read;
     346             :         size_t      rc;
     347             : 
     348        2498 :         bytes_to_read = Min(sizeof(manifestbuf),
     349             :                             manifest->manifest_size - manifest_bytes_done);
     350        2498 :         rc = BufFileRead(manifest->buffile, manifestbuf, bytes_to_read);
     351        2498 :         if (rc != bytes_to_read)
     352           0 :             ereport(ERROR,
     353             :                     (errcode_for_file_access(),
     354             :                      errmsg("could not read from temporary file: %m")));
     355        2498 :         pq_putmessage('d', manifestbuf, bytes_to_read);
     356        2498 :         manifest_bytes_done += bytes_to_read;
     357             :     }
     358             : 
     359             :     /* No more data, so send CopyDone message */
     360         142 :     pq_putemptymessage('c');
     361             : 
     362             :     /* Release resources */
     363         142 :     BufFileClose(manifest->buffile);
     364             : }
     365             : 
     366             : /*
     367             :  * Append a cstring to the manifest.
     368             :  */
     369             : static void
     370      135326 : AppendStringToManifest(backup_manifest_info *manifest, char *s)
     371             : {
     372      135326 :     int         len = strlen(s);
     373             : 
     374             :     Assert(manifest != NULL);
     375      135326 :     if (manifest->still_checksumming)
     376      134900 :         pg_sha256_update(&manifest->manifest_ctx, (uint8 *) s, len);
     377      135326 :     BufFileWrite(manifest->buffile, s, len);
     378      135326 :     manifest->manifest_size += len;
     379      135326 : }

Generated by: LCOV version 1.13