Line data    Source code

       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * Write a new backup manifest.
       4             :  *
       5             :  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
       6             :  * Portions Copyright (c) 1994, Regents of the University of California
       7             :  *
       8             :  * src/bin/pg_combinebackup/write_manifest.c
       9             :  *
      10             :  *-------------------------------------------------------------------------
      11             :  */
      12             : 
      13             : #include "postgres_fe.h"
      14             : 
      15             : #include <fcntl.h>
      16             : #include <time.h>
      17             : #include <unistd.h>
      18             : 
      19             : #include "common/checksum_helper.h"
      20             : #include "common/file_perm.h"
      21             : #include "common/logging.h"
      22             : #include "lib/stringinfo.h"
      23             : #include "load_manifest.h"
      24             : #include "mb/pg_wchar.h"
      25             : #include "write_manifest.h"
      26             : 
      27             : struct manifest_writer
      28             : {
      29             :     char        pathname[MAXPGPATH];
      30             :     int         fd;
      31             :     StringInfoData buf;
      32             :     bool        first_file;
      33             :     bool        still_checksumming;
      34             :     pg_checksum_context manifest_ctx;
      35             : };
      36             : 
      37             : static void escape_json(StringInfo buf, const char *str);
      38             : static void flush_manifest(manifest_writer *mwriter);
      39             : static size_t hex_encode(const uint8 *src, size_t len, char *dst);
      40             : 
      41             : /*
      42             :  * Create a new backup manifest writer.
      43             :  *
      44             :  * The backup manifest will be written into a file named backup_manifest
      45             :  * in the specified directory.
      46             :  */
      47             : manifest_writer *
      48          22 : create_manifest_writer(char *directory, uint64 system_identifier)
      49             : {
      50          22 :     manifest_writer *mwriter = pg_malloc(sizeof(manifest_writer));
      51             : 
      52          22 :     snprintf(mwriter->pathname, MAXPGPATH, "%s/backup_manifest", directory);
      53          22 :     mwriter->fd = -1;
      54          22 :     initStringInfo(&mwriter->buf);
      55          22 :     mwriter->first_file = true;
      56          22 :     mwriter->still_checksumming = true;
      57          22 :     pg_checksum_init(&mwriter->manifest_ctx, CHECKSUM_TYPE_SHA256);
      58             : 
      59          22 :     appendStringInfo(&mwriter->buf,
      60             :                      "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
      61             :                      "\"System-Identifier\": " UINT64_FORMAT ",\n"
      62             :                      "\"Files\": [",
      63             :                      system_identifier);
      64             : 
      65          22 :     return mwriter;
      66             : }
      67             : 
      68             : /*
      69             :  * Add an entry for a file to a backup manifest.
      70             :  *
      71             :  * This is very similar to the backend's AddFileToBackupManifest, but
      72             :  * various adjustments are required due to frontend/backend differences
      73             :  * and other details.
      74             :  */
      75             : void
      76       22052 : add_file_to_manifest(manifest_writer *mwriter, const char *manifest_path,
      77             :                      uint64 size, time_t mtime,
      78             :                      pg_checksum_type checksum_type,
      79             :                      int checksum_length,
      80             :                      uint8 *checksum_payload)
      81             : {
      82       22052 :     int         pathlen = strlen(manifest_path);
      83             : 
      84       22052 :     if (mwriter->first_file)
      85             :     {
      86          22 :         appendStringInfoChar(&mwriter->buf, '\n');
      87          22 :         mwriter->first_file = false;
      88             :     }
      89             :     else
      90       22030 :         appendStringInfoString(&mwriter->buf, ",\n");
      91             : 
      92       22052 :     if (pg_encoding_verifymbstr(PG_UTF8, manifest_path, pathlen) == pathlen)
      93             :     {
      94       22052 :         appendStringInfoString(&mwriter->buf, "{ \"Path\": ");
      95       22052 :         escape_json(&mwriter->buf, manifest_path);
      96       22052 :         appendStringInfoString(&mwriter->buf, ", ");
      97             :     }
      98             :     else
      99             :     {
     100           0 :         appendStringInfoString(&mwriter->buf, "{ \"Encoded-Path\": \"");
     101           0 :         enlargeStringInfo(&mwriter->buf, 2 * pathlen);
     102           0 :         mwriter->buf.len += hex_encode((const uint8 *) manifest_path, pathlen,
     103           0 :                                        &mwriter->buf.data[mwriter->buf.len]);
     104           0 :         appendStringInfoString(&mwriter->buf, "\", ");
     105             :     }
     106             : 
     107       22052 :     appendStringInfo(&mwriter->buf, "\"Size\": %llu, ",
     108             :                      (unsigned long long) size);
     109             : 
     110       22052 :     appendStringInfoString(&mwriter->buf, "\"Last-Modified\": \"");
     111       22052 :     enlargeStringInfo(&mwriter->buf, 128);
     112       22052 :     mwriter->buf.len += strftime(&mwriter->buf.data[mwriter->buf.len], 128,
     113             :                                  "%Y-%m-%d %H:%M:%S %Z",
     114       22052 :                                  gmtime(&mtime));
     115       22052 :     appendStringInfoChar(&mwriter->buf, '"');
     116             : 
     117       22052 :     if (mwriter->buf.len > 128 * 1024)
     118          14 :         flush_manifest(mwriter);
     119             : 
     120       22052 :     if (checksum_length > 0)
     121             :     {
     122       20086 :         appendStringInfo(&mwriter->buf,
     123             :                          ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
     124             :                          pg_checksum_type_name(checksum_type));
     125             : 
     126       20086 :         enlargeStringInfo(&mwriter->buf, 2 * checksum_length);
     127       40172 :         mwriter->buf.len += hex_encode(checksum_payload, checksum_length,
     128       20086 :                                        &mwriter->buf.data[mwriter->buf.len]);
     129             : 
     130       20086 :         appendStringInfoChar(&mwriter->buf, '"');
     131             :     }
     132             : 
     133       22052 :     appendStringInfoString(&mwriter->buf, " }");
     134             : 
     135       22052 :     if (mwriter->buf.len > 128 * 1024)
     136           4 :         flush_manifest(mwriter);
     137       22052 : }
     138             : 
     139             : /*
     140             :  * Finalize the backup_manifest.
     141             :  */
     142             : void
     143          20 : finalize_manifest(manifest_writer *mwriter,
     144             :                   manifest_wal_range *first_wal_range)
     145             : {
     146             :     uint8       checksumbuf[PG_SHA256_DIGEST_LENGTH];
     147             :     int         len;
     148             :     manifest_wal_range *wal_range;
     149             : 
     150             :     /* Terminate the list of files. */
     151          20 :     appendStringInfoString(&mwriter->buf, "\n],\n");
     152             : 
     153             :     /* Start a list of LSN ranges. */
     154          20 :     appendStringInfoString(&mwriter->buf, "\"WAL-Ranges\": [\n");
     155             : 
     156          40 :     for (wal_range = first_wal_range; wal_range != NULL;
     157          20 :          wal_range = wal_range->next)
     158          20 :         appendStringInfo(&mwriter->buf,
     159             :                          "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
     160             :                          wal_range == first_wal_range ? "" : ",\n",
     161             :                          wal_range->tli,
     162          20 :                          LSN_FORMAT_ARGS(wal_range->start_lsn),
     163          20 :                          LSN_FORMAT_ARGS(wal_range->end_lsn));
     164             : 
     165             :     /* Terminate the list of WAL ranges. */
     166          20 :     appendStringInfoString(&mwriter->buf, "\n],\n");
     167             : 
     168             :     /* Flush accumulated data and update checksum calculation. */
     169          20 :     flush_manifest(mwriter);
     170             : 
     171             :     /* Checksum only includes data up to this point. */
     172          20 :     mwriter->still_checksumming = false;
     173             : 
     174             :     /* Compute and insert manifest checksum. */
     175          20 :     appendStringInfoString(&mwriter->buf, "\"Manifest-Checksum\": \"");
     176          20 :     enlargeStringInfo(&mwriter->buf, 2 * PG_SHA256_DIGEST_STRING_LENGTH);
     177          20 :     len = pg_checksum_final(&mwriter->manifest_ctx, checksumbuf);
     178             :     Assert(len == PG_SHA256_DIGEST_LENGTH);
     179          20 :     mwriter->buf.len +=
     180          20 :         hex_encode(checksumbuf, len, &mwriter->buf.data[mwriter->buf.len]);
     181          20 :     appendStringInfoString(&mwriter->buf, "\"}\n");
     182             : 
     183             :     /* Flush the last manifest checksum itself. */
     184          20 :     flush_manifest(mwriter);
     185             : 
     186             :     /* Close the file. */
     187          20 :     if (close(mwriter->fd) != 0)
     188           0 :         pg_fatal("could not close file \"%s\": %m", mwriter->pathname);
     189          20 :     mwriter->fd = -1;
     190          20 : }
     191             : 
     192             : /*
     193             :  * Produce a JSON string literal, properly escaping characters in the text.
     194             :  */
     195             : static void
     196       22052 : escape_json(StringInfo buf, const char *str)
     197             : {
     198             :     const char *p;
     199             : 
     200       22052 :     appendStringInfoCharMacro(buf, '"');
     201      293290 :     for (p = str; *p; p++)
     202             :     {
     203      271238 :         switch (*p)
     204             :         {
     205           0 :             case '\b':
     206           0 :                 appendStringInfoString(buf, "\\b");
     207           0 :                 break;
     208           0 :             case '\f':
     209           0 :                 appendStringInfoString(buf, "\\f");
     210           0 :                 break;
     211           0 :             case '\n':
     212           0 :                 appendStringInfoString(buf, "\\n");
     213           0 :                 break;
     214           0 :             case '\r':
     215           0 :                 appendStringInfoString(buf, "\\r");
     216           0 :                 break;
     217           0 :             case '\t':
     218           0 :                 appendStringInfoString(buf, "\\t");
     219           0 :                 break;
     220           0 :             case '"':
     221           0 :                 appendStringInfoString(buf, "\\\"");
     222           0 :                 break;
     223           0 :             case '\\':
     224           0 :                 appendStringInfoString(buf, "\\\\");
     225           0 :                 break;
     226      271238 :             default:
     227      271238 :                 if ((unsigned char) *p < ' ')
     228           0 :                     appendStringInfo(buf, "\\u%04x", (int) *p);
     229             :                 else
     230      271238 :                     appendStringInfoCharMacro(buf, *p);
     231      271238 :                 break;
     232             :         }
     233             :     }
     234       22052 :     appendStringInfoCharMacro(buf, '"');
     235       22052 : }
     236             : 
     237             : /*
     238             :  * Flush whatever portion of the backup manifest we have generated and
     239             :  * buffered in memory out to a file on disk.
     240             :  *
     241             :  * The first call to this function will create the file. After that, we
     242             :  * keep it open and just append more data.
     243             :  */
     244             : static void
     245          58 : flush_manifest(manifest_writer *mwriter)
     246             : {
     247          58 :     if (mwriter->fd == -1 &&
     248          20 :         (mwriter->fd = open(mwriter->pathname,
     249             :                             O_WRONLY | O_CREAT | O_EXCL | PG_BINARY,
     250             :                             pg_file_create_mode)) < 0)
     251           0 :         pg_fatal("could not open file \"%s\": %m", mwriter->pathname);
     252             : 
     253          58 :     if (mwriter->buf.len > 0)
     254             :     {
     255             :         ssize_t     wb;
     256             : 
     257          58 :         wb = write(mwriter->fd, mwriter->buf.data, mwriter->buf.len);
     258          58 :         if (wb != mwriter->buf.len)
     259             :         {
     260           0 :             if (wb < 0)
     261           0 :                 pg_fatal("could not write file \"%s\": %m", mwriter->pathname);
     262             :             else
     263           0 :                 pg_fatal("could not write file \"%s\": wrote %d of %d",
     264             :                          mwriter->pathname, (int) wb, mwriter->buf.len);
     265             :         }
     266             : 
     267          96 :         if (mwriter->still_checksumming &&
     268          38 :             pg_checksum_update(&mwriter->manifest_ctx,
     269          38 :                                (uint8 *) mwriter->buf.data,
     270          38 :                                mwriter->buf.len) < 0)
     271           0 :             pg_fatal("could not update checksum of file \"%s\"",
     272             :                      mwriter->pathname);
     273          58 :         resetStringInfo(&mwriter->buf);
     274             :     }
     275          58 : }
     276             : 
     277             : /*
     278             :  * Encode bytes using two hexadecimal digits for each one.
     279             :  */
     280             : static size_t
     281       20106 : hex_encode(const uint8 *src, size_t len, char *dst)
     282             : {
     283       20106 :     const uint8 *end = src + len;
     284             : 
     285      147650 :     while (src < end)
     286             :     {
     287      127544 :         unsigned    n1 = (*src >> 4) & 0xF;
     288      127544 :         unsigned    n2 = *src & 0xF;
     289             : 
     290      127544 :         *dst++ = n1 < 10 ? '0' + n1 : 'a' + n1 - 10;
     291      127544 :         *dst++ = n2 < 10 ? '0' + n2 : 'a' + n2 - 10;
     292      127544 :         ++src;
     293             :     }
     294             : 
     295       20106 :     return len * 2;
     296             : }