LCOV - code coverage report
Current view: top level - src/bin/pg_verifybackup - pg_verifybackup.c (source / functions) Hit Total Coverage
Test: PostgreSQL 18devel Lines: 423 477 88.7 %
Date: 2024-12-03 07:14:52 Functions: 23 23 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_verifybackup.c
       4             :  *    Verify a backup against a backup manifest.
       5             :  *
       6             :  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
       7             :  * Portions Copyright (c) 1994, Regents of the University of California
       8             :  *
       9             :  * src/bin/pg_verifybackup/pg_verifybackup.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres_fe.h"
      15             : 
      16             : #include <dirent.h>
      17             : #include <fcntl.h>
      18             : #include <sys/stat.h>
      19             : #include <time.h>
      20             : 
      21             : #include "common/logging.h"
      22             : #include "common/parse_manifest.h"
      23             : #include "fe_utils/simple_list.h"
      24             : #include "getopt_long.h"
      25             : #include "limits.h"
      26             : #include "pg_verifybackup.h"
      27             : #include "pgtime.h"
      28             : 
      29             : /*
      30             :  * For efficiency, we'd like our hash table containing information about the
      31             :  * manifest to start out with approximately the correct number of entries.
      32             :  * There's no way to know the exact number of entries without reading the whole
      33             :  * file, but we can get an estimate by dividing the file size by the estimated
      34             :  * number of bytes per line.
      35             :  *
      36             :  * This could be off by about a factor of two in either direction, because the
      37             :  * checksum algorithm has a big impact on the line lengths; e.g. a SHA512
      38             :  * checksum is 128 hex bytes, whereas a CRC-32C value is only 8, and there
      39             :  * might be no checksum at all.
      40             :  */
      41             : #define ESTIMATED_BYTES_PER_MANIFEST_LINE   100
      42             : 
      43             : /*
      44             :  * How many bytes should we try to read from a file at once?
      45             :  */
      46             : #define READ_CHUNK_SIZE             (128 * 1024)
      47             : 
      48             : /*
      49             :  * Tar file information needed for content verification.
      50             :  */
      51             : typedef struct tar_file
      52             : {
      53             :     char       *relpath;
      54             :     Oid         tblspc_oid;
      55             :     pg_compress_algorithm compress_algorithm;
      56             : } tar_file;
      57             : 
      58             : static manifest_data *parse_manifest_file(char *manifest_path);
      59             : static void verifybackup_version_cb(JsonManifestParseContext *context,
      60             :                                     int manifest_version);
      61             : static void verifybackup_system_identifier(JsonManifestParseContext *context,
      62             :                                            uint64 manifest_system_identifier);
      63             : static void verifybackup_per_file_cb(JsonManifestParseContext *context,
      64             :                                      const char *pathname, uint64 size,
      65             :                                      pg_checksum_type checksum_type,
      66             :                                      int checksum_length,
      67             :                                      uint8 *checksum_payload);
      68             : static void verifybackup_per_wal_range_cb(JsonManifestParseContext *context,
      69             :                                           TimeLineID tli,
      70             :                                           XLogRecPtr start_lsn,
      71             :                                           XLogRecPtr end_lsn);
      72             : static void report_manifest_error(JsonManifestParseContext *context,
      73             :                                   const char *fmt,...)
      74             :             pg_attribute_printf(2, 3) pg_attribute_noreturn();
      75             : 
      76             : static void verify_tar_backup(verifier_context *context, DIR *dir);
      77             : static void verify_plain_backup_directory(verifier_context *context,
      78             :                                           char *relpath, char *fullpath,
      79             :                                           DIR *dir);
      80             : static void verify_plain_backup_file(verifier_context *context, char *relpath,
      81             :                                      char *fullpath);
      82             : static void verify_control_file(const char *controlpath,
      83             :                                 uint64 manifest_system_identifier);
      84             : static void precheck_tar_backup_file(verifier_context *context, char *relpath,
      85             :                                      char *fullpath, SimplePtrList *tarfiles);
      86             : static void verify_tar_file(verifier_context *context, char *relpath,
      87             :                             char *fullpath, astreamer *streamer);
      88             : static void report_extra_backup_files(verifier_context *context);
      89             : static void verify_backup_checksums(verifier_context *context);
      90             : static void verify_file_checksum(verifier_context *context,
      91             :                                  manifest_file *m, char *fullpath,
      92             :                                  uint8 *buffer);
      93             : static void parse_required_wal(verifier_context *context,
      94             :                                char *pg_waldump_path,
      95             :                                char *wal_directory);
      96             : static astreamer *create_archive_verifier(verifier_context *context,
      97             :                                           char *archive_name,
      98             :                                           Oid tblspc_oid,
      99             :                                           pg_compress_algorithm compress_algo);
     100             : 
     101             : static void progress_report(bool finished);
     102             : static void usage(void);
     103             : 
     104             : static const char *progname;
     105             : 
     106             : /* is progress reporting enabled? */
     107             : static bool show_progress = false;
     108             : 
     109             : /* Progress indicators */
     110             : static uint64 total_size = 0;
     111             : static uint64 done_size = 0;
     112             : 
     113             : /*
     114             :  * Main entry point.
     115             :  */
     116             : int
     117         238 : main(int argc, char **argv)
     118             : {
     119             :     static struct option long_options[] = {
     120             :         {"exit-on-error", no_argument, NULL, 'e'},
     121             :         {"ignore", required_argument, NULL, 'i'},
     122             :         {"manifest-path", required_argument, NULL, 'm'},
     123             :         {"format", required_argument, NULL, 'F'},
     124             :         {"no-parse-wal", no_argument, NULL, 'n'},
     125             :         {"progress", no_argument, NULL, 'P'},
     126             :         {"quiet", no_argument, NULL, 'q'},
     127             :         {"skip-checksums", no_argument, NULL, 's'},
     128             :         {"wal-directory", required_argument, NULL, 'w'},
     129             :         {NULL, 0, NULL, 0}
     130             :     };
     131             : 
     132             :     int         c;
     133             :     verifier_context context;
     134         238 :     char       *manifest_path = NULL;
     135         238 :     bool        no_parse_wal = false;
     136         238 :     bool        quiet = false;
     137         238 :     char       *wal_directory = NULL;
     138         238 :     char       *pg_waldump_path = NULL;
     139             :     DIR        *dir;
     140             : 
     141         238 :     pg_logging_init(argv[0]);
     142         238 :     set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
     143         238 :     progname = get_progname(argv[0]);
     144             : 
     145         238 :     memset(&context, 0, sizeof(context));
     146             : 
     147         238 :     if (argc > 1)
     148             :     {
     149         236 :         if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
     150             :         {
     151           2 :             usage();
     152           2 :             exit(0);
     153             :         }
     154         234 :         if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
     155             :         {
     156           2 :             puts("pg_verifybackup (PostgreSQL) " PG_VERSION);
     157           2 :             exit(0);
     158             :         }
     159             :     }
     160             : 
     161             :     /*
     162             :      * Skip certain files in the toplevel directory.
     163             :      *
     164             :      * Ignore the backup_manifest file, because it's not included in the
     165             :      * backup manifest.
     166             :      *
     167             :      * Ignore the pg_wal directory, because those files are not included in
     168             :      * the backup manifest either, since they are fetched separately from the
     169             :      * backup itself, and verified via a separate mechanism.
     170             :      *
     171             :      * Ignore postgresql.auto.conf, recovery.signal, and standby.signal,
     172             :      * because we expect that those files may sometimes be created or changed
     173             :      * as part of the backup process. For example, pg_basebackup -R will
     174             :      * modify postgresql.auto.conf and create standby.signal.
     175             :      */
     176         234 :     simple_string_list_append(&context.ignore_list, "backup_manifest");
     177         234 :     simple_string_list_append(&context.ignore_list, "pg_wal");
     178         234 :     simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
     179         234 :     simple_string_list_append(&context.ignore_list, "recovery.signal");
     180         234 :     simple_string_list_append(&context.ignore_list, "standby.signal");
     181             : 
     182         368 :     while ((c = getopt_long(argc, argv, "eF:i:m:nPqsw:", long_options, NULL)) != -1)
     183             :     {
     184         138 :         switch (c)
     185             :         {
     186          48 :             case 'e':
     187          48 :                 context.exit_on_error = true;
     188          48 :                 break;
     189           8 :             case 'i':
     190             :                 {
     191           8 :                     char       *arg = pstrdup(optarg);
     192             : 
     193           8 :                     canonicalize_path(arg);
     194           8 :                     simple_string_list_append(&context.ignore_list, arg);
     195           8 :                     break;
     196             :                 }
     197           8 :             case 'm':
     198           8 :                 manifest_path = pstrdup(optarg);
     199           8 :                 canonicalize_path(manifest_path);
     200           8 :                 break;
     201           6 :             case 'F':
     202           6 :                 if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
     203           2 :                     context.format = 'p';
     204           4 :                 else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
     205           2 :                     context.format = 't';
     206             :                 else
     207           2 :                     pg_fatal("invalid backup format \"%s\", must be \"plain\" or \"tar\"",
     208             :                              optarg);
     209           4 :                 break;
     210          50 :             case 'n':
     211          50 :                 no_parse_wal = true;
     212          50 :                 break;
     213           4 :             case 'P':
     214           4 :                 show_progress = true;
     215           4 :                 break;
     216           6 :             case 'q':
     217           6 :                 quiet = true;
     218           6 :                 break;
     219           4 :             case 's':
     220           4 :                 context.skip_checksums = true;
     221           4 :                 break;
     222           2 :             case 'w':
     223           2 :                 wal_directory = pstrdup(optarg);
     224           2 :                 canonicalize_path(wal_directory);
     225           2 :                 break;
     226           2 :             default:
     227             :                 /* getopt_long already emitted a complaint */
     228           2 :                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
     229           2 :                 exit(1);
     230             :         }
     231             :     }
     232             : 
     233             :     /* Get backup directory name */
     234         230 :     if (optind >= argc)
     235             :     {
     236           2 :         pg_log_error("no backup directory specified");
     237           2 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
     238           2 :         exit(1);
     239             :     }
     240         228 :     context.backup_directory = pstrdup(argv[optind++]);
     241         228 :     canonicalize_path(context.backup_directory);
     242             : 
     243             :     /* Complain if any arguments remain */
     244         228 :     if (optind < argc)
     245             :     {
     246           2 :         pg_log_error("too many command-line arguments (first is \"%s\")",
     247             :                      argv[optind]);
     248           2 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
     249           2 :         exit(1);
     250             :     }
     251             : 
     252             :     /* Complain if the specified arguments conflict */
     253         226 :     if (show_progress && quiet)
     254           2 :         pg_fatal("cannot specify both %s and %s",
     255             :                  "-P/--progress", "-q/--quiet");
     256             : 
     257             :     /* Unless --no-parse-wal was specified, we will need pg_waldump. */
     258         224 :     if (!no_parse_wal)
     259             :     {
     260             :         int         ret;
     261             : 
     262         174 :         pg_waldump_path = pg_malloc(MAXPGPATH);
     263         174 :         ret = find_other_exec(argv[0], "pg_waldump",
     264             :                               "pg_waldump (PostgreSQL) " PG_VERSION "\n",
     265             :                               pg_waldump_path);
     266         174 :         if (ret < 0)
     267             :         {
     268             :             char        full_path[MAXPGPATH];
     269             : 
     270           0 :             if (find_my_exec(argv[0], full_path) < 0)
     271           0 :                 strlcpy(full_path, progname, sizeof(full_path));
     272             : 
     273           0 :             if (ret == -1)
     274           0 :                 pg_fatal("program \"%s\" is needed by %s but was not found in the same directory as \"%s\"",
     275             :                          "pg_waldump", "pg_verifybackup", full_path);
     276             :             else
     277           0 :                 pg_fatal("program \"%s\" was found by \"%s\" but was not the same version as %s",
     278             :                          "pg_waldump", full_path, "pg_verifybackup");
     279             :         }
     280             :     }
     281             : 
     282             :     /* By default, look for the manifest in the backup directory. */
     283         224 :     if (manifest_path == NULL)
     284         216 :         manifest_path = psprintf("%s/backup_manifest",
     285             :                                  context.backup_directory);
     286             : 
     287             :     /* By default, look for the WAL in the backup directory, too. */
     288         224 :     if (wal_directory == NULL)
     289         222 :         wal_directory = psprintf("%s/pg_wal", context.backup_directory);
     290             : 
     291             :     /*
     292             :      * Try to read the manifest. We treat any errors encountered while parsing
     293             :      * the manifest as fatal; there doesn't seem to be much point in trying to
     294             :      * verify the backup directory against a corrupted manifest.
     295             :      */
     296         224 :     context.manifest = parse_manifest_file(manifest_path);
     297             : 
     298             :     /*
     299             :      * If the backup directory cannot be found, treat this as a fatal error.
     300             :      */
     301         154 :     dir = opendir(context.backup_directory);
     302         154 :     if (dir == NULL)
     303           2 :         report_fatal_error("could not open directory \"%s\": %m",
     304             :                            context.backup_directory);
     305             : 
     306             :     /*
     307             :      * At this point, we know that the backup directory exists, so it's now
     308             :      * reasonable to check for files immediately inside it. Thus, before going
     309             :      * further, if the user did not specify the backup format, check for
     310             :      * PG_VERSION to distinguish between tar and plain format.
     311             :      */
     312         152 :     if (context.format == '\0')
     313             :     {
     314             :         struct stat sb;
     315             :         char       *path;
     316             : 
     317         148 :         path = psprintf("%s/%s", context.backup_directory, "PG_VERSION");
     318         148 :         if (stat(path, &sb) == 0)
     319         108 :             context.format = 'p';
     320          40 :         else if (errno != ENOENT)
     321             :         {
     322           0 :             pg_log_error("could not stat file \"%s\": %m", path);
     323           0 :             exit(1);
     324             :         }
     325             :         else
     326             :         {
     327             :             /* No PG_VERSION, so assume tar format. */
     328          40 :             context.format = 't';
     329             :         }
     330         148 :         pfree(path);
     331             :     }
     332             : 
     333             :     /*
     334             :      * XXX: In the future, we should consider enhancing pg_waldump to read WAL
     335             :      * files from an archive.
     336             :      */
     337         152 :     if (!no_parse_wal && context.format == 't')
     338             :     {
     339           0 :         pg_log_error("pg_waldump cannot read tar files");
     340           0 :         pg_log_error_hint("You must use -n or --no-parse-wal when verifying a tar-format backup.");
     341           0 :         exit(1);
     342             :     }
     343             : 
     344             :     /*
     345             :      * Perform the appropriate type of verification appropriate based on the
     346             :      * backup format. This will close 'dir'.
     347             :      */
     348         152 :     if (context.format == 'p')
     349         110 :         verify_plain_backup_directory(&context, NULL, context.backup_directory,
     350             :                                       dir);
     351             :     else
     352          42 :         verify_tar_backup(&context, dir);
     353             : 
     354             :     /*
     355             :      * The "matched" flag should now be set on every entry in the hash table.
     356             :      * Any entries for which the bit is not set are files mentioned in the
     357             :      * manifest that don't exist on disk (or in the relevant tar files).
     358             :      */
     359         148 :     report_extra_backup_files(&context);
     360             : 
     361             :     /*
     362             :      * If this is a tar-format backup, checksums were already verified above;
     363             :      * but if it's a plain-format backup, we postpone it until this point,
     364             :      * since the earlier checks can be performed just by knowing which files
     365             :      * are present, without needing to read all of them.
     366             :      */
     367         146 :     if (context.format == 'p' && !context.skip_checksums)
     368         102 :         verify_backup_checksums(&context);
     369             : 
     370             :     /*
     371             :      * Try to parse the required ranges of WAL records, unless we were told
     372             :      * not to do so.
     373             :      */
     374         146 :     if (!no_parse_wal)
     375         100 :         parse_required_wal(&context, pg_waldump_path, wal_directory);
     376             : 
     377             :     /*
     378             :      * If everything looks OK, tell the user this, unless we were asked to
     379             :      * work quietly.
     380             :      */
     381         146 :     if (!context.saw_any_error && !quiet)
     382          98 :         printf(_("backup successfully verified\n"));
     383             : 
     384         146 :     return context.saw_any_error ? 1 : 0;
     385             : }
     386             : 
     387             : /*
     388             :  * Parse a manifest file and return a data structure describing the contents.
     389             :  */
     390             : static manifest_data *
     391         224 : parse_manifest_file(char *manifest_path)
     392             : {
     393             :     int         fd;
     394             :     struct stat statbuf;
     395             :     off_t       estimate;
     396             :     uint32      initial_size;
     397             :     manifest_files_hash *ht;
     398             :     char       *buffer;
     399             :     int         rc;
     400             :     JsonManifestParseContext context;
     401             :     manifest_data *result;
     402             : 
     403         224 :     int         chunk_size = READ_CHUNK_SIZE;
     404             : 
     405             :     /* Open the manifest file. */
     406         224 :     if ((fd = open(manifest_path, O_RDONLY | PG_BINARY, 0)) < 0)
     407           6 :         report_fatal_error("could not open file \"%s\": %m", manifest_path);
     408             : 
     409             :     /* Figure out how big the manifest is. */
     410         218 :     if (fstat(fd, &statbuf) != 0)
     411           0 :         report_fatal_error("could not stat file \"%s\": %m", manifest_path);
     412             : 
     413             :     /* Guess how large to make the hash table based on the manifest size. */
     414         218 :     estimate = statbuf.st_size / ESTIMATED_BYTES_PER_MANIFEST_LINE;
     415         218 :     initial_size = Min(PG_UINT32_MAX, Max(estimate, 256));
     416             : 
     417             :     /* Create the hash table. */
     418         218 :     ht = manifest_files_create(initial_size, NULL);
     419             : 
     420         218 :     result = pg_malloc0(sizeof(manifest_data));
     421         218 :     result->files = ht;
     422         218 :     context.private_data = result;
     423         218 :     context.version_cb = verifybackup_version_cb;
     424         218 :     context.system_identifier_cb = verifybackup_system_identifier;
     425         218 :     context.per_file_cb = verifybackup_per_file_cb;
     426         218 :     context.per_wal_range_cb = verifybackup_per_wal_range_cb;
     427         218 :     context.error_cb = report_manifest_error;
     428             : 
     429             :     /*
     430             :      * Parse the file, in chunks if necessary.
     431             :      */
     432         218 :     if (statbuf.st_size <= chunk_size)
     433             :     {
     434          66 :         buffer = pg_malloc(statbuf.st_size);
     435          66 :         rc = read(fd, buffer, statbuf.st_size);
     436          66 :         if (rc != statbuf.st_size)
     437             :         {
     438           0 :             if (rc < 0)
     439           0 :                 pg_fatal("could not read file \"%s\": %m", manifest_path);
     440             :             else
     441           0 :                 pg_fatal("could not read file \"%s\": read %d of %lld",
     442             :                          manifest_path, rc, (long long int) statbuf.st_size);
     443             :         }
     444             : 
     445             :         /* Close the manifest file. */
     446          66 :         close(fd);
     447             : 
     448             :         /* Parse the manifest. */
     449          66 :         json_parse_manifest(&context, buffer, statbuf.st_size);
     450             :     }
     451             :     else
     452             :     {
     453         152 :         int         bytes_left = statbuf.st_size;
     454             :         JsonManifestParseIncrementalState *inc_state;
     455             : 
     456         152 :         inc_state = json_parse_manifest_incremental_init(&context);
     457             : 
     458         152 :         buffer = pg_malloc(chunk_size + 1);
     459             : 
     460         452 :         while (bytes_left > 0)
     461             :         {
     462         304 :             int         bytes_to_read = chunk_size;
     463             : 
     464             :             /*
     465             :              * Make sure that the last chunk is sufficiently large. (i.e. at
     466             :              * least half the chunk size) so that it will contain fully the
     467             :              * piece at the end with the checksum.
     468             :              */
     469         304 :             if (bytes_left < chunk_size)
     470         152 :                 bytes_to_read = bytes_left;
     471         152 :             else if (bytes_left < 2 * chunk_size)
     472         152 :                 bytes_to_read = bytes_left / 2;
     473         304 :             rc = read(fd, buffer, bytes_to_read);
     474         304 :             if (rc != bytes_to_read)
     475             :             {
     476           0 :                 if (rc < 0)
     477           0 :                     pg_fatal("could not read file \"%s\": %m", manifest_path);
     478             :                 else
     479           0 :                     pg_fatal("could not read file \"%s\": read %lld of %lld",
     480             :                              manifest_path,
     481             :                              (long long int) (statbuf.st_size + rc - bytes_left),
     482             :                              (long long int) statbuf.st_size);
     483             :             }
     484         304 :             bytes_left -= rc;
     485         304 :             json_parse_manifest_incremental_chunk(inc_state, buffer, rc,
     486             :                                                   bytes_left == 0);
     487             :         }
     488             : 
     489             :         /* Release the incremental state memory */
     490         148 :         json_parse_manifest_incremental_shutdown(inc_state);
     491             : 
     492         148 :         close(fd);
     493             :     }
     494             : 
     495             :     /* Done with the buffer. */
     496         154 :     pfree(buffer);
     497             : 
     498         154 :     return result;
     499             : }
     500             : 
     501             : /*
     502             :  * Report an error while parsing the manifest.
     503             :  *
     504             :  * We consider all such errors to be fatal errors. The manifest parser
     505             :  * expects this function not to return.
     506             :  */
     507             : static void
     508          62 : report_manifest_error(JsonManifestParseContext *context, const char *fmt,...)
     509             : {
     510             :     va_list     ap;
     511             : 
     512          62 :     va_start(ap, fmt);
     513          62 :     pg_log_generic_v(PG_LOG_ERROR, PG_LOG_PRIMARY, gettext(fmt), ap);
     514          62 :     va_end(ap);
     515             : 
     516          62 :     exit(1);
     517             : }
     518             : 
     519             : /*
     520             :  * Record details extracted from the backup manifest.
     521             :  */
     522             : static void
     523         206 : verifybackup_version_cb(JsonManifestParseContext *context,
     524             :                         int manifest_version)
     525             : {
     526         206 :     manifest_data *manifest = context->private_data;
     527             : 
     528             :     /* Validation will be at the later stage */
     529         206 :     manifest->version = manifest_version;
     530         206 : }
     531             : 
     532             : /*
     533             :  * Record details extracted from the backup manifest.
     534             :  */
     535             : static void
     536         158 : verifybackup_system_identifier(JsonManifestParseContext *context,
     537             :                                uint64 manifest_system_identifier)
     538             : {
     539         158 :     manifest_data *manifest = context->private_data;
     540             : 
     541             :     /* Validation will be at the later stage */
     542         158 :     manifest->system_identifier = manifest_system_identifier;
     543         158 : }
     544             : 
     545             : /*
     546             :  * Record details extracted from the backup manifest for one file.
     547             :  */
     548             : static void
     549      153468 : verifybackup_per_file_cb(JsonManifestParseContext *context,
     550             :                          const char *pathname, uint64 size,
     551             :                          pg_checksum_type checksum_type,
     552             :                          int checksum_length, uint8 *checksum_payload)
     553             : {
     554      153468 :     manifest_data *manifest = context->private_data;
     555      153468 :     manifest_files_hash *ht = manifest->files;
     556             :     manifest_file *m;
     557             :     bool        found;
     558             : 
     559             :     /* Make a new entry in the hash table for this file. */
     560      153468 :     m = manifest_files_insert(ht, pathname, &found);
     561      153468 :     if (found)
     562           2 :         report_fatal_error("duplicate path name in backup manifest: \"%s\"",
     563             :                            pathname);
     564             : 
     565             :     /* Initialize the entry. */
     566      153466 :     m->size = size;
     567      153466 :     m->checksum_type = checksum_type;
     568      153466 :     m->checksum_length = checksum_length;
     569      153466 :     m->checksum_payload = checksum_payload;
     570      153466 :     m->matched = false;
     571      153466 :     m->bad = false;
     572      153466 : }
     573             : 
     574             : /*
     575             :  * Record details extracted from the backup manifest for one WAL range.
     576             :  */
     577             : static void
     578         160 : verifybackup_per_wal_range_cb(JsonManifestParseContext *context,
     579             :                               TimeLineID tli,
     580             :                               XLogRecPtr start_lsn, XLogRecPtr end_lsn)
     581             : {
     582         160 :     manifest_data *manifest = context->private_data;
     583             :     manifest_wal_range *range;
     584             : 
     585             :     /* Allocate and initialize a struct describing this WAL range. */
     586         160 :     range = palloc(sizeof(manifest_wal_range));
     587         160 :     range->tli = tli;
     588         160 :     range->start_lsn = start_lsn;
     589         160 :     range->end_lsn = end_lsn;
     590         160 :     range->prev = manifest->last_wal_range;
     591         160 :     range->next = NULL;
     592             : 
     593             :     /* Add it to the end of the list. */
     594         160 :     if (manifest->first_wal_range == NULL)
     595         160 :         manifest->first_wal_range = range;
     596             :     else
     597           0 :         manifest->last_wal_range->next = range;
     598         160 :     manifest->last_wal_range = range;
     599         160 : }
     600             : 
     601             : /*
     602             :  * Verify one directory of a plain-format backup.
     603             :  *
     604             :  * 'relpath' is NULL if we are to verify the top-level backup directory,
     605             :  * and otherwise the relative path to the directory that is to be verified.
     606             :  *
     607             :  * 'fullpath' is the backup directory with 'relpath' appended; i.e. the actual
     608             :  * filesystem path at which it can be found.
     609             :  *
     610             :  * 'dir' is an open directory handle, or NULL if the caller wants us to
     611             :  * open it. If the caller chooses to pass a handle, we'll close it when
     612             :  * we're done with it.
     613             :  */
     614             : static void
     615        2730 : verify_plain_backup_directory(verifier_context *context, char *relpath,
     616             :                               char *fullpath, DIR *dir)
     617             : {
     618             :     struct dirent *dirent;
     619             : 
     620             :     /* Open the directory unless the caller did it. */
     621        2730 :     if (dir == NULL && ((dir = opendir(fullpath)) == NULL))
     622             :     {
     623           2 :         report_backup_error(context,
     624             :                             "could not open directory \"%s\": %m", fullpath);
     625           2 :         simple_string_list_append(&context->ignore_list, relpath);
     626             : 
     627           2 :         return;
     628             :     }
     629             : 
     630      114142 :     while (errno = 0, (dirent = readdir(dir)) != NULL)
     631             :     {
     632      111418 :         char       *filename = dirent->d_name;
     633      111418 :         char       *newfullpath = psprintf("%s/%s", fullpath, filename);
     634             :         char       *newrelpath;
     635             : 
     636             :         /* Skip "." and ".." */
     637      111418 :         if (filename[0] == '.' && (filename[1] == '\0'
     638        2724 :                                    || strcmp(filename, "..") == 0))
     639        5450 :             continue;
     640             : 
     641      105968 :         if (relpath == NULL)
     642        2604 :             newrelpath = pstrdup(filename);
     643             :         else
     644      103364 :             newrelpath = psprintf("%s/%s", relpath, filename);
     645             : 
     646      105968 :         if (!should_ignore_relpath(context, newrelpath))
     647      105648 :             verify_plain_backup_file(context, newrelpath, newfullpath);
     648             : 
     649      105964 :         pfree(newfullpath);
     650      105964 :         pfree(newrelpath);
     651             :     }
     652             : 
     653        2724 :     if (closedir(dir))
     654             :     {
     655           0 :         report_backup_error(context,
     656             :                             "could not close directory \"%s\": %m", fullpath);
     657           0 :         return;
     658             :     }
     659             : }
     660             : 
     661             : /*
     662             :  * Verify one file (which might actually be a directory or a symlink).
     663             :  *
     664             :  * The arguments to this function have the same meaning as the similarly named
     665             :  * arguments to verify_plain_backup_directory.
     666             :  */
     667             : static void
     668      105648 : verify_plain_backup_file(verifier_context *context, char *relpath,
     669             :                          char *fullpath)
     670             : {
     671             :     struct stat sb;
     672             :     manifest_file *m;
     673             : 
     674      105648 :     if (stat(fullpath, &sb) != 0)
     675             :     {
     676           6 :         report_backup_error(context,
     677             :                             "could not stat file or directory \"%s\": %m",
     678             :                             relpath);
     679             : 
     680             :         /*
     681             :          * Suppress further errors related to this path name and, if it's a
     682             :          * directory, anything underneath it.
     683             :          */
     684           6 :         simple_string_list_append(&context->ignore_list, relpath);
     685             : 
     686        2628 :         return;
     687             :     }
     688             : 
     689             :     /* If it's a directory, just recurse. */
     690      105642 :     if (S_ISDIR(sb.st_mode))
     691             :     {
     692        2620 :         verify_plain_backup_directory(context, relpath, fullpath, NULL);
     693        2618 :         return;
     694             :     }
     695             : 
     696             :     /* If it's not a directory, it should be a plain file. */
     697      103022 :     if (!S_ISREG(sb.st_mode))
     698             :     {
     699           0 :         report_backup_error(context,
     700             :                             "\"%s\" is not a file or directory",
     701             :                             relpath);
     702           0 :         return;
     703             :     }
     704             : 
     705             :     /* Check whether there's an entry in the manifest hash. */
     706      103022 :     m = manifest_files_lookup(context->manifest->files, relpath);
     707      103022 :     if (m == NULL)
     708             :     {
     709           4 :         report_backup_error(context,
     710             :                             "\"%s\" is present on disk but not in the manifest",
     711             :                             relpath);
     712           4 :         return;
     713             :     }
     714             : 
     715             :     /* Flag this entry as having been encountered in the filesystem. */
     716      103018 :     m->matched = true;
     717             : 
     718             :     /* Check that the size matches. */
     719      103018 :     if (m->size != sb.st_size)
     720             :     {
     721           4 :         report_backup_error(context,
     722             :                             "\"%s\" has size %llu on disk but size %llu in the manifest",
     723           4 :                             relpath, (unsigned long long) sb.st_size,
     724           4 :                             (unsigned long long) m->size);
     725           4 :         m->bad = true;
     726             :     }
     727             : 
     728             :     /*
     729             :      * Validate the manifest system identifier, not available in manifest
     730             :      * version 1.
     731             :      */
     732      103018 :     if (context->manifest->version != 1 &&
     733      103018 :         strcmp(relpath, "global/pg_control") == 0)
     734         110 :         verify_control_file(fullpath, context->manifest->system_identifier);
     735             : 
     736             :     /* Update statistics for progress report, if necessary */
     737      103016 :     if (show_progress && !context->skip_checksums &&
     738        1936 :         should_verify_checksum(m))
     739        1936 :         total_size += m->size;
     740             : 
     741             :     /*
     742             :      * We don't verify checksums at this stage. We first finish verifying that
     743             :      * we have the expected set of files with the expected sizes, and only
     744             :      * afterwards verify the checksums. That's because computing checksums may
     745             :      * take a while, and we'd like to report more obvious problems quickly.
     746             :      */
     747             : }
     748             : 
     749             : /*
     750             :  * Sanity check control file and validate system identifier against manifest
     751             :  * system identifier.
     752             :  */
     753             : static void
     754         110 : verify_control_file(const char *controlpath, uint64 manifest_system_identifier)
     755             : {
     756             :     ControlFileData *control_file;
     757             :     bool        crc_ok;
     758             : 
     759         110 :     pg_log_debug("reading \"%s\"", controlpath);
     760         110 :     control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
     761             : 
     762             :     /* Control file contents not meaningful if CRC is bad. */
     763         110 :     if (!crc_ok)
     764           0 :         report_fatal_error("%s: CRC is incorrect", controlpath);
     765             : 
     766             :     /* Can't interpret control file if not current version. */
     767         110 :     if (control_file->pg_control_version != PG_CONTROL_VERSION)
     768           0 :         report_fatal_error("%s: unexpected control file version",
     769             :                            controlpath);
     770             : 
     771             :     /* System identifiers should match. */
     772         110 :     if (manifest_system_identifier != control_file->system_identifier)
     773           2 :         report_fatal_error("%s: manifest system identifier is %llu, but control file has %llu",
     774             :                            controlpath,
     775             :                            (unsigned long long) manifest_system_identifier,
     776           2 :                            (unsigned long long) control_file->system_identifier);
     777             : 
     778             :     /* Release memory. */
     779         108 :     pfree(control_file);
     780         108 : }
     781             : 
     782             : /*
     783             :  * Verify tar backup.
     784             :  *
     785             :  * The caller should pass a handle to the target directory, which we will
     786             :  * close when we're done with it.
     787             :  */
     788             : static void
     789          42 : verify_tar_backup(verifier_context *context, DIR *dir)
     790             : {
     791             :     struct dirent *dirent;
     792          42 :     SimplePtrList tarfiles = {NULL, NULL};
     793             :     SimplePtrListCell *cell;
     794             : 
     795             :     Assert(context->format != 'p');
     796             : 
     797          42 :     progress_report(false);
     798             : 
     799             :     /* First pass: scan the directory for tar files. */
     800         302 :     while (errno = 0, (dirent = readdir(dir)) != NULL)
     801             :     {
     802         260 :         char       *filename = dirent->d_name;
     803             : 
     804             :         /* Skip "." and ".." */
     805         260 :         if (filename[0] == '.' && (filename[1] == '\0'
     806          42 :                                    || strcmp(filename, "..") == 0))
     807          84 :             continue;
     808             : 
     809             :         /*
     810             :          * Unless it's something we should ignore, perform prechecks and add
     811             :          * it to the list.
     812             :          */
     813         176 :         if (!should_ignore_relpath(context, filename))
     814             :         {
     815             :             char       *fullpath;
     816             : 
     817         130 :             fullpath = psprintf("%s/%s", context->backup_directory, filename);
     818         130 :             precheck_tar_backup_file(context, filename, fullpath, &tarfiles);
     819         130 :             pfree(fullpath);
     820             :         }
     821             :     }
     822             : 
     823          42 :     if (closedir(dir))
     824             :     {
     825           0 :         report_backup_error(context,
     826             :                             "could not close directory \"%s\": %m",
     827             :                             context->backup_directory);
     828           0 :         return;
     829             :     }
     830             : 
     831             :     /* Second pass: Perform the final verification of the tar contents. */
     832          98 :     for (cell = tarfiles.head; cell != NULL; cell = cell->next)
     833             :     {
     834          58 :         tar_file   *tar = (tar_file *) cell->ptr;
     835             :         astreamer  *streamer;
     836             :         char       *fullpath;
     837             : 
     838             :         /*
     839             :          * Prepares the archive streamer stack according to the tar
     840             :          * compression format.
     841             :          */
     842          58 :         streamer = create_archive_verifier(context,
     843             :                                            tar->relpath,
     844             :                                            tar->tblspc_oid,
     845             :                                            tar->compress_algorithm);
     846             : 
     847             :         /* Compute the full pathname to the target file. */
     848          58 :         fullpath = psprintf("%s/%s", context->backup_directory,
     849             :                             tar->relpath);
     850             : 
     851             :         /* Invoke the streamer for reading, decompressing, and verifying. */
     852          58 :         verify_tar_file(context, tar->relpath, fullpath, streamer);
     853             : 
     854             :         /* Cleanup. */
     855          56 :         pfree(tar->relpath);
     856          56 :         pfree(tar);
     857          56 :         pfree(fullpath);
     858             : 
     859          56 :         astreamer_finalize(streamer);
     860          56 :         astreamer_free(streamer);
     861             :     }
     862          40 :     simple_ptr_list_destroy(&tarfiles);
     863             : 
     864          40 :     progress_report(true);
     865             : }
     866             : 
     867             : /*
     868             :  * Preparatory steps for verifying files in tar format backups.
     869             :  *
     870             :  * Carries out basic validation of the tar format backup file, detects the
     871             :  * compression type, and appends that information to the tarfiles list. An
     872             :  * error will be reported if the tar file is inaccessible, or if the file type,
     873             :  * name, or compression type is not as expected.
     874             :  *
     875             :  * The arguments to this function are mostly the same as the
     876             :  * verify_plain_backup_file. The additional argument outputs a list of valid
     877             :  * tar files.
     878             :  */
     879             : static void
     880         130 : precheck_tar_backup_file(verifier_context *context, char *relpath,
     881             :                          char *fullpath, SimplePtrList *tarfiles)
     882             : {
     883             :     struct stat sb;
     884         130 :     Oid         tblspc_oid = InvalidOid;
     885             :     pg_compress_algorithm compress_algorithm;
     886             :     tar_file   *tar;
     887         130 :     char       *suffix = NULL;
     888             : 
     889             :     /* Should be tar format backup */
     890             :     Assert(context->format == 't');
     891             : 
     892             :     /* Get file information */
     893         130 :     if (stat(fullpath, &sb) != 0)
     894             :     {
     895           0 :         report_backup_error(context,
     896             :                             "could not stat file or directory \"%s\": %m",
     897             :                             relpath);
     898          70 :         return;
     899             :     }
     900             : 
     901             :     /* In a tar format backup, we expect only plain files. */
     902         130 :     if (!S_ISREG(sb.st_mode))
     903             :     {
     904          32 :         report_backup_error(context,
     905             :                             "\"%s\" is not a plain file",
     906             :                             relpath);
     907          32 :         return;
     908             :     }
     909             : 
     910             :     /*
     911             :      * We expect tar files for backing up the main directory, tablespace, and
     912             :      * pg_wal directory.
     913             :      *
     914             :      * pg_basebackup writes the main data directory to an archive file named
     915             :      * base.tar, the pg_wal directory to pg_wal.tar, and the tablespace
     916             :      * directory to <tablespaceoid>.tar, each followed by a compression type
     917             :      * extension such as .gz, .lz4, or .zst.
     918             :      */
     919          98 :     if (strncmp("base", relpath, 4) == 0)
     920          40 :         suffix = relpath + 4;
     921          58 :     else if (strncmp("pg_wal", relpath, 6) == 0)
     922          28 :         suffix = relpath + 6;
     923             :     else
     924             :     {
     925             :         /* Expected a <tablespaceoid>.tar file here. */
     926          30 :         uint64      num = strtoul(relpath, &suffix, 10);
     927             : 
     928             :         /*
     929             :          * Report an error if we didn't consume at least one character, if the
     930             :          * result is 0, or if the value is too large to be a valid OID.
     931             :          */
     932          30 :         if (suffix == NULL || num <= 0 || num > OID_MAX)
     933             :         {
     934          10 :             report_backup_error(context,
     935             :                                 "file \"%s\" is not expected in a tar format backup",
     936             :                                 relpath);
     937          10 :             return;
     938             :         }
     939          20 :         tblspc_oid = (Oid) num;
     940             :     }
     941             : 
     942             :     /* Now, check the compression type of the tar */
     943          88 :     if (strcmp(suffix, ".tar") == 0)
     944          76 :         compress_algorithm = PG_COMPRESSION_NONE;
     945          12 :     else if (strcmp(suffix, ".tgz") == 0)
     946           0 :         compress_algorithm = PG_COMPRESSION_GZIP;
     947          12 :     else if (strcmp(suffix, ".tar.gz") == 0)
     948           6 :         compress_algorithm = PG_COMPRESSION_GZIP;
     949           6 :     else if (strcmp(suffix, ".tar.lz4") == 0)
     950           6 :         compress_algorithm = PG_COMPRESSION_LZ4;
     951           0 :     else if (strcmp(suffix, ".tar.zst") == 0)
     952           0 :         compress_algorithm = PG_COMPRESSION_ZSTD;
     953             :     else
     954             :     {
     955           0 :         report_backup_error(context,
     956             :                             "file \"%s\" is not expected in a tar format backup",
     957             :                             relpath);
     958           0 :         return;
     959             :     }
     960             : 
     961             :     /*
     962             :      * Ignore WALs, as reading and verification will be handled through
     963             :      * pg_waldump.
     964             :      */
     965          88 :     if (strncmp("pg_wal", relpath, 6) == 0)
     966          28 :         return;
     967             : 
     968             :     /*
     969             :      * Append the information to the list for complete verification at a later
     970             :      * stage.
     971             :      */
     972          60 :     tar = pg_malloc(sizeof(tar_file));
     973          60 :     tar->relpath = pstrdup(relpath);
     974          60 :     tar->tblspc_oid = tblspc_oid;
     975          60 :     tar->compress_algorithm = compress_algorithm;
     976             : 
     977          60 :     simple_ptr_list_append(tarfiles, tar);
     978             : 
     979             :     /* Update statistics for progress report, if necessary */
     980          60 :     if (show_progress)
     981           0 :         total_size += sb.st_size;
     982             : }
     983             : 
     984             : /*
     985             :  * Verification of a single tar file content.
     986             :  *
     987             :  * It reads a given tar archive in predefined chunks and passes it to the
     988             :  * streamer, which initiates routines for decompression (if necessary) and then
     989             :  * verifies each member within the tar file.
     990             :  */
     991             : static void
     992          58 : verify_tar_file(verifier_context *context, char *relpath, char *fullpath,
     993             :                 astreamer *streamer)
     994             : {
     995             :     int         fd;
     996             :     int         rc;
     997             :     char       *buffer;
     998             : 
     999          58 :     pg_log_debug("reading \"%s\"", fullpath);
    1000             : 
    1001             :     /* Open the target file. */
    1002          58 :     if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
    1003             :     {
    1004           0 :         report_backup_error(context, "could not open file \"%s\": %m",
    1005             :                             relpath);
    1006           0 :         return;
    1007             :     }
    1008             : 
    1009          58 :     buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
    1010             : 
    1011             :     /* Perform the reads */
    1012        6308 :     while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
    1013             :     {
    1014        6252 :         astreamer_content(streamer, NULL, buffer, rc, ASTREAMER_UNKNOWN);
    1015             : 
    1016             :         /* Report progress */
    1017        6250 :         done_size += rc;
    1018        6250 :         progress_report(false);
    1019             :     }
    1020             : 
    1021          56 :     pg_free(buffer);
    1022             : 
    1023          56 :     if (rc < 0)
    1024           0 :         report_backup_error(context, "could not read file \"%s\": %m",
    1025             :                             relpath);
    1026             : 
    1027             :     /* Close the file. */
    1028          56 :     if (close(fd) != 0)
    1029           0 :         report_backup_error(context, "could not close file \"%s\": %m",
    1030             :                             relpath);
    1031             : }
    1032             : 
    1033             : /*
    1034             :  * Scan the hash table for entries where the 'matched' flag is not set; report
    1035             :  * that such files are present in the manifest but not on disk.
    1036             :  */
    1037             : static void
    1038         148 : report_extra_backup_files(verifier_context *context)
    1039             : {
    1040         148 :     manifest_data *manifest = context->manifest;
    1041             :     manifest_files_iterator it;
    1042             :     manifest_file *m;
    1043             : 
    1044         148 :     manifest_files_start_iterate(manifest->files, &it);
    1045      142722 :     while ((m = manifest_files_iterate(manifest->files, &it)) != NULL)
    1046      142576 :         if (!m->matched && !should_ignore_relpath(context, m->pathname))
    1047        1952 :             report_backup_error(context,
    1048             :                                 "\"%s\" is present in the manifest but not on disk",
    1049             :                                 m->pathname);
    1050         146 : }
    1051             : 
    1052             : /*
    1053             :  * Verify checksums for hash table entries that are otherwise unproblematic.
    1054             :  * If we've already reported some problem related to a hash table entry, or
    1055             :  * if it has no checksum, just skip it.
    1056             :  */
    1057             : static void
    1058         102 : verify_backup_checksums(verifier_context *context)
    1059             : {
    1060         102 :     manifest_data *manifest = context->manifest;
    1061             :     manifest_files_iterator it;
    1062             :     manifest_file *m;
    1063             :     uint8      *buffer;
    1064             : 
    1065         102 :     progress_report(false);
    1066             : 
    1067         102 :     buffer = pg_malloc(READ_CHUNK_SIZE * sizeof(uint8));
    1068             : 
    1069         102 :     manifest_files_start_iterate(manifest->files, &it);
    1070       99168 :     while ((m = manifest_files_iterate(manifest->files, &it)) != NULL)
    1071             :     {
    1072       99066 :         if (should_verify_checksum(m) &&
    1073       93252 :             !should_ignore_relpath(context, m->pathname))
    1074             :         {
    1075             :             char       *fullpath;
    1076             : 
    1077             :             /* Compute the full pathname to the target file. */
    1078       93252 :             fullpath = psprintf("%s/%s", context->backup_directory,
    1079             :                                 m->pathname);
    1080             : 
    1081             :             /* Do the actual checksum verification. */
    1082       93252 :             verify_file_checksum(context, m, fullpath, buffer);
    1083             : 
    1084             :             /* Avoid leaking memory. */
    1085       93252 :             pfree(fullpath);
    1086             :         }
    1087             :     }
    1088             : 
    1089         102 :     pfree(buffer);
    1090             : 
    1091         102 :     progress_report(true);
    1092         102 : }
    1093             : 
    1094             : /*
    1095             :  * Verify the checksum of a single file.
    1096             :  */
    1097             : static void
    1098       93252 : verify_file_checksum(verifier_context *context, manifest_file *m,
    1099             :                      char *fullpath, uint8 *buffer)
    1100             : {
    1101             :     pg_checksum_context checksum_ctx;
    1102       93252 :     const char *relpath = m->pathname;
    1103             :     int         fd;
    1104             :     int         rc;
    1105       93252 :     uint64      bytes_read = 0;
    1106             :     uint8       checksumbuf[PG_CHECKSUM_MAX_LENGTH];
    1107             :     int         checksumlen;
    1108             : 
    1109             :     /* Open the target file. */
    1110       93252 :     if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
    1111             :     {
    1112           2 :         report_backup_error(context, "could not open file \"%s\": %m",
    1113             :                             relpath);
    1114           2 :         return;
    1115             :     }
    1116             : 
    1117             :     /* Initialize checksum context. */
    1118       93250 :     if (pg_checksum_init(&checksum_ctx, m->checksum_type) < 0)
    1119             :     {
    1120           0 :         report_backup_error(context, "could not initialize checksum of file \"%s\"",
    1121             :                             relpath);
    1122           0 :         close(fd);
    1123           0 :         return;
    1124             :     }
    1125             : 
    1126             :     /* Read the file chunk by chunk, updating the checksum as we go. */
    1127      175068 :     while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
    1128             :     {
    1129       81818 :         bytes_read += rc;
    1130       81818 :         if (pg_checksum_update(&checksum_ctx, buffer, rc) < 0)
    1131             :         {
    1132           0 :             report_backup_error(context, "could not update checksum of file \"%s\"",
    1133             :                                 relpath);
    1134           0 :             close(fd);
    1135           0 :             return;
    1136             :         }
    1137             : 
    1138             :         /* Report progress */
    1139       81818 :         done_size += rc;
    1140       81818 :         progress_report(false);
    1141             :     }
    1142       93250 :     if (rc < 0)
    1143           0 :         report_backup_error(context, "could not read file \"%s\": %m",
    1144             :                             relpath);
    1145             : 
    1146             :     /* Close the file. */
    1147       93250 :     if (close(fd) != 0)
    1148             :     {
    1149           0 :         report_backup_error(context, "could not close file \"%s\": %m",
    1150             :                             relpath);
    1151           0 :         return;
    1152             :     }
    1153             : 
    1154             :     /* If we didn't manage to read the whole file, bail out now. */
    1155       93250 :     if (rc < 0)
    1156           0 :         return;
    1157             : 
    1158             :     /*
    1159             :      * Double-check that we read the expected number of bytes from the file.
    1160             :      * Normally, mismatches would be caught in verify_plain_backup_file and
    1161             :      * this check would never be reached, but this provides additional safety
    1162             :      * and clarity in the event of concurrent modifications or filesystem
    1163             :      * misbehavior.
    1164             :      */
    1165       93250 :     if (bytes_read != m->size)
    1166             :     {
    1167           0 :         report_backup_error(context,
    1168             :                             "file \"%s\" should contain %llu bytes, but read %llu bytes",
    1169           0 :                             relpath, (unsigned long long) m->size,
    1170             :                             (unsigned long long) bytes_read);
    1171           0 :         return;
    1172             :     }
    1173             : 
    1174             :     /* Get the final checksum. */
    1175       93250 :     checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
    1176       93250 :     if (checksumlen < 0)
    1177             :     {
    1178           0 :         report_backup_error(context,
    1179             :                             "could not finalize checksum of file \"%s\"",
    1180             :                             relpath);
    1181           0 :         return;
    1182             :     }
    1183             : 
    1184             :     /* And check it against the manifest. */
    1185       93250 :     if (checksumlen != m->checksum_length)
    1186           0 :         report_backup_error(context,
    1187             :                             "file \"%s\" has checksum of length %d, but expected %d",
    1188             :                             relpath, m->checksum_length, checksumlen);
    1189       93250 :     else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
    1190           6 :         report_backup_error(context,
    1191             :                             "checksum mismatch for file \"%s\"",
    1192             :                             relpath);
    1193             : }
    1194             : 
    1195             : /*
    1196             :  * Attempt to parse the WAL files required to restore from backup using
    1197             :  * pg_waldump.
    1198             :  */
    1199             : static void
    1200         100 : parse_required_wal(verifier_context *context, char *pg_waldump_path,
    1201             :                    char *wal_directory)
    1202             : {
    1203         100 :     manifest_data *manifest = context->manifest;
    1204         100 :     manifest_wal_range *this_wal_range = manifest->first_wal_range;
    1205             : 
    1206         200 :     while (this_wal_range != NULL)
    1207             :     {
    1208             :         char       *pg_waldump_cmd;
    1209             : 
    1210         100 :         pg_waldump_cmd = psprintf("\"%s\" --quiet --path=\"%s\" --timeline=%u --start=%X/%X --end=%X/%X\n",
    1211             :                                   pg_waldump_path, wal_directory, this_wal_range->tli,
    1212         100 :                                   LSN_FORMAT_ARGS(this_wal_range->start_lsn),
    1213         100 :                                   LSN_FORMAT_ARGS(this_wal_range->end_lsn));
    1214         100 :         fflush(NULL);
    1215         100 :         if (system(pg_waldump_cmd) != 0)
    1216           4 :             report_backup_error(context,
    1217             :                                 "WAL parsing failed for timeline %u",
    1218             :                                 this_wal_range->tli);
    1219             : 
    1220         100 :         this_wal_range = this_wal_range->next;
    1221             :     }
    1222         100 : }
    1223             : 
    1224             : /*
    1225             :  * Report a problem with the backup.
    1226             :  *
    1227             :  * Update the context to indicate that we saw an error, and exit if the
    1228             :  * context says we should.
    1229             :  */
    1230             : void
    1231        2042 : report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
    1232             : {
    1233             :     va_list     ap;
    1234             : 
    1235        2042 :     va_start(ap, fmt);
    1236        2042 :     pg_log_generic_v(PG_LOG_ERROR, PG_LOG_PRIMARY, gettext(fmt), ap);
    1237        2042 :     va_end(ap);
    1238             : 
    1239        2042 :     context->saw_any_error = true;
    1240        2042 :     if (context->exit_on_error)
    1241           2 :         exit(1);
    1242        2040 : }
    1243             : 
    1244             : /*
    1245             :  * Report a fatal error and exit
    1246             :  */
    1247             : void
    1248          14 : report_fatal_error(const char *pg_restrict fmt,...)
    1249             : {
    1250             :     va_list     ap;
    1251             : 
    1252          14 :     va_start(ap, fmt);
    1253          14 :     pg_log_generic_v(PG_LOG_ERROR, PG_LOG_PRIMARY, gettext(fmt), ap);
    1254          14 :     va_end(ap);
    1255             : 
    1256          14 :     exit(1);
    1257             : }
    1258             : 
    1259             : /*
    1260             :  * Is the specified relative path, or some prefix of it, listed in the set
    1261             :  * of paths to ignore?
    1262             :  *
    1263             :  * Note that by "prefix" we mean a parent directory; for this purpose,
    1264             :  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
    1265             :  */
    1266             : bool
    1267      240346 : should_ignore_relpath(verifier_context *context, const char *relpath)
    1268             : {
    1269             :     SimpleStringListCell *cell;
    1270             : 
    1271     1461542 :     for (cell = context->ignore_list.head; cell != NULL; cell = cell->next)
    1272             :     {
    1273     1223606 :         const char *r = relpath;
    1274     1223606 :         char       *v = cell->val;
    1275             : 
    1276     1713408 :         while (*v != '\0' && *r == *v)
    1277      489802 :             ++r, ++v;
    1278             : 
    1279     1223606 :         if (*v == '\0' && (*r == '\0' || *r == '/'))
    1280        2410 :             return true;
    1281             :     }
    1282             : 
    1283      237936 :     return false;
    1284             : }
    1285             : 
    1286             : /*
    1287             :  * Create a chain of archive streamers appropriate for verifying a given
    1288             :  * archive.
    1289             :  */
    1290             : static astreamer *
    1291          58 : create_archive_verifier(verifier_context *context, char *archive_name,
    1292             :                         Oid tblspc_oid, pg_compress_algorithm compress_algo)
    1293             : {
    1294          58 :     astreamer  *streamer = NULL;
    1295             : 
    1296             :     /* Should be here only for tar backup */
    1297             :     Assert(context->format == 't');
    1298             : 
    1299             :     /* Last step is the actual verification. */
    1300          58 :     streamer = astreamer_verify_content_new(streamer, context, archive_name,
    1301             :                                             tblspc_oid);
    1302             : 
    1303             :     /* Before that we must parse the tar file. */
    1304          58 :     streamer = astreamer_tar_parser_new(streamer);
    1305             : 
    1306             :     /* Before that we must decompress, if archive is compressed. */
    1307          58 :     if (compress_algo == PG_COMPRESSION_GZIP)
    1308           6 :         streamer = astreamer_gzip_decompressor_new(streamer);
    1309          52 :     else if (compress_algo == PG_COMPRESSION_LZ4)
    1310           6 :         streamer = astreamer_lz4_decompressor_new(streamer);
    1311          46 :     else if (compress_algo == PG_COMPRESSION_ZSTD)
    1312           0 :         streamer = astreamer_zstd_decompressor_new(streamer);
    1313             : 
    1314          58 :     return streamer;
    1315             : }
    1316             : 
    1317             : /*
    1318             :  * Print a progress report based on the global variables.
    1319             :  *
    1320             :  * Progress report is written at maximum once per second, unless the finished
    1321             :  * parameter is set to true.
    1322             :  *
    1323             :  * If finished is set to true, this is the last progress report. The cursor
    1324             :  * is moved to the next line.
    1325             :  */
    1326             : static void
    1327       88354 : progress_report(bool finished)
    1328             : {
    1329             :     static pg_time_t last_progress_report = 0;
    1330             :     pg_time_t   now;
    1331       88354 :     int         percent_size = 0;
    1332             :     char        totalsize_str[32];
    1333             :     char        donesize_str[32];
    1334             : 
    1335       88354 :     if (!show_progress)
    1336       88350 :         return;
    1337             : 
    1338        1708 :     now = time(NULL);
    1339        1708 :     if (now == last_progress_report && !finished)
    1340        1704 :         return;                 /* Max once per second */
    1341             : 
    1342           4 :     last_progress_report = now;
    1343           4 :     percent_size = total_size ? (int) ((done_size * 100 / total_size)) : 0;
    1344             : 
    1345           4 :     snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT,
    1346             :              total_size / 1024);
    1347           4 :     snprintf(donesize_str, sizeof(donesize_str), UINT64_FORMAT,
    1348             :              done_size / 1024);
    1349             : 
    1350           4 :     fprintf(stderr,
    1351           4 :             _("%*s/%s kB (%d%%) verified"),
    1352           4 :             (int) strlen(totalsize_str),
    1353             :             donesize_str, totalsize_str, percent_size);
    1354             : 
    1355             :     /*
    1356             :      * Stay on the same line if reporting to a terminal and we're not done
    1357             :      * yet.
    1358             :      */
    1359           4 :     fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
    1360             : }
    1361             : 
    1362             : /*
    1363             :  * Print out usage information and exit.
    1364             :  */
    1365             : static void
    1366           2 : usage(void)
    1367             : {
    1368           2 :     printf(_("%s verifies a backup against the backup manifest.\n\n"), progname);
    1369           2 :     printf(_("Usage:\n  %s [OPTION]... BACKUPDIR\n\n"), progname);
    1370           2 :     printf(_("Options:\n"));
    1371           2 :     printf(_("  -e, --exit-on-error         exit immediately on error\n"));
    1372           2 :     printf(_("  -F, --format=p|t            backup format (plain, tar)\n"));
    1373           2 :     printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
    1374           2 :     printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
    1375           2 :     printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
    1376           2 :     printf(_("  -P, --progress              show progress information\n"));
    1377           2 :     printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
    1378           2 :     printf(_("  -s, --skip-checksums        skip checksum verification\n"));
    1379           2 :     printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
    1380           2 :     printf(_("  -V, --version               output version information, then exit\n"));
    1381           2 :     printf(_("  -?, --help                  show this help, then exit\n"));
    1382           2 :     printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
    1383           2 :     printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
    1384           2 : }

Generated by: LCOV version 1.14