LCOV - code coverage report
Current view: top level - src/bin/pg_verifybackup - pg_verifybackup.c (source / functions) Hit Total Coverage
Test: PostgreSQL 15devel Lines: 259 290 89.3 %
Date: 2021-11-29 05:09:10 Functions: 16 16 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_verifybackup.c
       4             :  *    Verify a backup against a backup manifest.
       5             :  *
       6             :  * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
       7             :  * Portions Copyright (c) 1994, Regents of the University of California
       8             :  *
       9             :  * src/bin/pg_verifybackup/pg_verifybackup.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres_fe.h"
      15             : 
      16             : #include <dirent.h>
      17             : #include <fcntl.h>
      18             : #include <sys/stat.h>
      19             : 
      20             : #include "common/hashfn.h"
      21             : #include "common/logging.h"
      22             : #include "fe_utils/simple_list.h"
      23             : #include "getopt_long.h"
      24             : #include "parse_manifest.h"
      25             : 
      26             : /*
      27             :  * For efficiency, we'd like our hash table containing information about the
      28             :  * manifest to start out with approximately the correct number of entries.
      29             :  * There's no way to know the exact number of entries without reading the whole
      30             :  * file, but we can get an estimate by dividing the file size by the estimated
      31             :  * number of bytes per line.
      32             :  *
      33             :  * This could be off by about a factor of two in either direction, because the
      34             :  * checksum algorithm has a big impact on the line lengths; e.g. a SHA512
      35             :  * checksum is 128 hex bytes, whereas a CRC-32C value is only 8, and there
      36             :  * might be no checksum at all.
      37             :  */
      38             : #define ESTIMATED_BYTES_PER_MANIFEST_LINE   100
      39             : 
      40             : /*
      41             :  * How many bytes should we try to read from a file at once?
      42             :  */
      43             : #define READ_CHUNK_SIZE             4096
      44             : 
      45             : /*
      46             :  * Each file described by the manifest file is parsed to produce an object
      47             :  * like this.
      48             :  */
      49             : typedef struct manifest_file
      50             : {
      51             :     uint32      status;         /* hash status */
      52             :     char       *pathname;
      53             :     size_t      size;
      54             :     pg_checksum_type checksum_type;
      55             :     int         checksum_length;
      56             :     uint8      *checksum_payload;
      57             :     bool        matched;
      58             :     bool        bad;
      59             : } manifest_file;
      60             : 
      61             : /*
      62             :  * Define a hash table which we can use to store information about the files
      63             :  * mentioned in the backup manifest.
      64             :  */
      65             : static uint32 hash_string_pointer(char *s);
      66             : #define SH_PREFIX       manifest_files
      67             : #define SH_ELEMENT_TYPE manifest_file
      68             : #define SH_KEY_TYPE     char *
      69             : #define SH_KEY          pathname
      70             : #define SH_HASH_KEY(tb, key)    hash_string_pointer(key)
      71             : #define SH_EQUAL(tb, a, b)      (strcmp(a, b) == 0)
      72             : #define SH_SCOPE        static inline
      73             : #define SH_RAW_ALLOCATOR    pg_malloc0
      74             : #define SH_DECLARE
      75             : #define SH_DEFINE
      76             : #include "lib/simplehash.h"
      77             : 
      78             : /*
      79             :  * Each WAL range described by the manifest file is parsed to produce an
      80             :  * object like this.
      81             :  */
      82             : typedef struct manifest_wal_range
      83             : {
      84             :     TimeLineID  tli;
      85             :     XLogRecPtr  start_lsn;
      86             :     XLogRecPtr  end_lsn;
      87             :     struct manifest_wal_range *next;
      88             :     struct manifest_wal_range *prev;
      89             : } manifest_wal_range;
      90             : 
      91             : /*
      92             :  * Details we need in callbacks that occur while parsing a backup manifest.
      93             :  */
      94             : typedef struct parser_context
      95             : {
      96             :     manifest_files_hash *ht;
      97             :     manifest_wal_range *first_wal_range;
      98             :     manifest_wal_range *last_wal_range;
      99             : } parser_context;
     100             : 
     101             : /*
     102             :  * All of the context information we need while checking a backup manifest.
     103             :  */
     104             : typedef struct verifier_context
     105             : {
     106             :     manifest_files_hash *ht;
     107             :     char       *backup_directory;
     108             :     SimpleStringList ignore_list;
     109             :     bool        exit_on_error;
     110             :     bool        saw_any_error;
     111             : } verifier_context;
     112             : 
     113             : static void parse_manifest_file(char *manifest_path,
     114             :                                 manifest_files_hash **ht_p,
     115             :                                 manifest_wal_range **first_wal_range_p);
     116             : 
     117             : static void record_manifest_details_for_file(JsonManifestParseContext *context,
     118             :                                              char *pathname, size_t size,
     119             :                                              pg_checksum_type checksum_type,
     120             :                                              int checksum_length,
     121             :                                              uint8 *checksum_payload);
     122             : static void record_manifest_details_for_wal_range(JsonManifestParseContext *context,
     123             :                                                   TimeLineID tli,
     124             :                                                   XLogRecPtr start_lsn,
     125             :                                                   XLogRecPtr end_lsn);
     126             : static void report_manifest_error(JsonManifestParseContext *context,
     127             :                                   const char *fmt,...)
     128             :             pg_attribute_printf(2, 3) pg_attribute_noreturn();
     129             : 
     130             : static void verify_backup_directory(verifier_context *context,
     131             :                                     char *relpath, char *fullpath);
     132             : static void verify_backup_file(verifier_context *context,
     133             :                                char *relpath, char *fullpath);
     134             : static void report_extra_backup_files(verifier_context *context);
     135             : static void verify_backup_checksums(verifier_context *context);
     136             : static void verify_file_checksum(verifier_context *context,
     137             :                                  manifest_file *m, char *pathname);
     138             : static void parse_required_wal(verifier_context *context,
     139             :                                char *pg_waldump_path,
     140             :                                char *wal_directory,
     141             :                                manifest_wal_range *first_wal_range);
     142             : 
     143             : static void report_backup_error(verifier_context *context,
     144             :                                 const char *pg_restrict fmt,...)
     145             :             pg_attribute_printf(2, 3);
     146             : static void report_fatal_error(const char *pg_restrict fmt,...)
     147             :             pg_attribute_printf(1, 2) pg_attribute_noreturn();
     148             : static bool should_ignore_relpath(verifier_context *context, char *relpath);
     149             : 
     150             : static void usage(void);
     151             : 
     152             : static const char *progname;
     153             : 
     154             : /*
     155             :  * Main entry point.
     156             :  */
     157             : int
     158         158 : main(int argc, char **argv)
     159             : {
     160             :     static struct option long_options[] = {
     161             :         {"exit-on-error", no_argument, NULL, 'e'},
     162             :         {"ignore", required_argument, NULL, 'i'},
     163             :         {"manifest-path", required_argument, NULL, 'm'},
     164             :         {"no-parse-wal", no_argument, NULL, 'n'},
     165             :         {"quiet", no_argument, NULL, 'q'},
     166             :         {"skip-checksums", no_argument, NULL, 's'},
     167             :         {"wal-directory", required_argument, NULL, 'w'},
     168             :         {NULL, 0, NULL, 0}
     169             :     };
     170             : 
     171             :     int         c;
     172             :     verifier_context context;
     173             :     manifest_wal_range *first_wal_range;
     174         158 :     char       *manifest_path = NULL;
     175         158 :     bool        no_parse_wal = false;
     176         158 :     bool        quiet = false;
     177         158 :     bool        skip_checksums = false;
     178         158 :     char       *wal_directory = NULL;
     179         158 :     char       *pg_waldump_path = NULL;
     180             : 
     181         158 :     pg_logging_init(argv[0]);
     182         158 :     set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_verifybackup"));
     183         158 :     progname = get_progname(argv[0]);
     184             : 
     185         158 :     memset(&context, 0, sizeof(context));
     186             : 
     187         158 :     if (argc > 1)
     188             :     {
     189         156 :         if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
     190             :         {
     191           2 :             usage();
     192           2 :             exit(0);
     193             :         }
     194         154 :         if (strcmp(argv[1], "--version") == 0 || strcmp(argv[1], "-V") == 0)
     195             :         {
     196           2 :             puts("pg_verifybackup (PostgreSQL) " PG_VERSION);
     197           2 :             exit(0);
     198             :         }
     199             :     }
     200             : 
     201             :     /*
     202             :      * Skip certain files in the toplevel directory.
     203             :      *
     204             :      * Ignore the backup_manifest file, because it's not included in the
     205             :      * backup manifest.
     206             :      *
     207             :      * Ignore the pg_wal directory, because those files are not included in
     208             :      * the backup manifest either, since they are fetched separately from the
     209             :      * backup itself, and verified via a separate mechanism.
     210             :      *
     211             :      * Ignore postgresql.auto.conf, recovery.signal, and standby.signal,
     212             :      * because we expect that those files may sometimes be created or changed
     213             :      * as part of the backup process. For example, pg_basebackup -R will
     214             :      * modify postgresql.auto.conf and create standby.signal.
     215             :      */
     216         154 :     simple_string_list_append(&context.ignore_list, "backup_manifest");
     217         154 :     simple_string_list_append(&context.ignore_list, "pg_wal");
     218         154 :     simple_string_list_append(&context.ignore_list, "postgresql.auto.conf");
     219         154 :     simple_string_list_append(&context.ignore_list, "recovery.signal");
     220         154 :     simple_string_list_append(&context.ignore_list, "standby.signal");
     221             : 
     222         192 :     while ((c = getopt_long(argc, argv, "ei:m:nqsw:", long_options, NULL)) != -1)
     223             :     {
     224          40 :         switch (c)
     225             :         {
     226          14 :             case 'e':
     227          14 :                 context.exit_on_error = true;
     228          14 :                 break;
     229           8 :             case 'i':
     230             :                 {
     231           8 :                     char       *arg = pstrdup(optarg);
     232             : 
     233           8 :                     canonicalize_path(arg);
     234           8 :                     simple_string_list_append(&context.ignore_list, arg);
     235           8 :                     break;
     236             :                 }
     237           4 :             case 'm':
     238           4 :                 manifest_path = pstrdup(optarg);
     239           4 :                 canonicalize_path(manifest_path);
     240           4 :                 break;
     241           2 :             case 'n':
     242           2 :                 no_parse_wal = true;
     243           2 :                 break;
     244           4 :             case 'q':
     245           4 :                 quiet = true;
     246           4 :                 break;
     247           4 :             case 's':
     248           4 :                 skip_checksums = true;
     249           4 :                 break;
     250           2 :             case 'w':
     251           2 :                 wal_directory = pstrdup(optarg);
     252           2 :                 canonicalize_path(wal_directory);
     253           2 :                 break;
     254           2 :             default:
     255           2 :                 fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
     256             :                         progname);
     257           2 :                 exit(1);
     258             :         }
     259             :     }
     260             : 
     261             :     /* Get backup directory name */
     262         152 :     if (optind >= argc)
     263             :     {
     264           2 :         pg_log_fatal("no backup directory specified");
     265           2 :         fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
     266             :                 progname);
     267           2 :         exit(1);
     268             :     }
     269         150 :     context.backup_directory = pstrdup(argv[optind++]);
     270         150 :     canonicalize_path(context.backup_directory);
     271             : 
     272             :     /* Complain if any arguments remain */
     273         150 :     if (optind < argc)
     274             :     {
     275           2 :         pg_log_fatal("too many command-line arguments (first is \"%s\")",
     276             :                      argv[optind]);
     277           2 :         fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
     278             :                 progname);
     279           2 :         exit(1);
     280             :     }
     281             : 
     282             :     /* Unless --no-parse-wal was specified, we will need pg_waldump. */
     283         148 :     if (!no_parse_wal)
     284             :     {
     285             :         int         ret;
     286             : 
     287         146 :         pg_waldump_path = pg_malloc(MAXPGPATH);
     288         146 :         ret = find_other_exec(argv[0], "pg_waldump",
     289             :                               "pg_waldump (PostgreSQL) " PG_VERSION "\n",
     290             :                               pg_waldump_path);
     291         146 :         if (ret < 0)
     292             :         {
     293             :             char        full_path[MAXPGPATH];
     294             : 
     295           0 :             if (find_my_exec(argv[0], full_path) < 0)
     296           0 :                 strlcpy(full_path, progname, sizeof(full_path));
     297           0 :             if (ret == -1)
     298           0 :                 pg_log_fatal("The program \"%s\" is needed by %s but was not found in the\n"
     299             :                              "same directory as \"%s\".\n"
     300             :                              "Check your installation.",
     301             :                              "pg_waldump", "pg_verifybackup", full_path);
     302             :             else
     303           0 :                 pg_log_fatal("The program \"%s\" was found by \"%s\"\n"
     304             :                              "but was not the same version as %s.\n"
     305             :                              "Check your installation.",
     306             :                              "pg_waldump", full_path, "pg_verifybackup");
     307           0 :             exit(1);
     308             :         }
     309             :     }
     310             : 
     311             :     /* By default, look for the manifest in the backup directory. */
     312         148 :     if (manifest_path == NULL)
     313         144 :         manifest_path = psprintf("%s/backup_manifest",
     314             :                                  context.backup_directory);
     315             : 
     316             :     /* By default, look for the WAL in the backup directory, too. */
     317         148 :     if (wal_directory == NULL)
     318         146 :         wal_directory = psprintf("%s/pg_wal", context.backup_directory);
     319             : 
     320             :     /*
     321             :      * Try to read the manifest. We treat any errors encountered while parsing
     322             :      * the manifest as fatal; there doesn't seem to be much point in trying to
     323             :      * verify the backup directory against a corrupted manifest.
     324             :      */
     325         148 :     parse_manifest_file(manifest_path, &context.ht, &first_wal_range);
     326             : 
     327             :     /*
     328             :      * Now scan the files in the backup directory. At this stage, we verify
     329             :      * that every file on disk is present in the manifest and that the sizes
     330             :      * match. We also set the "matched" flag on every manifest entry that
     331             :      * corresponds to a file on disk.
     332             :      */
     333          84 :     verify_backup_directory(&context, NULL, context.backup_directory);
     334             : 
     335             :     /*
     336             :      * The "matched" flag should now be set on every entry in the hash table.
     337             :      * Any entries for which the bit is not set are files mentioned in the
     338             :      * manifest that don't exist on disk.
     339             :      */
     340          82 :     report_extra_backup_files(&context);
     341             : 
     342             :     /*
     343             :      * Now do the expensive work of verifying file checksums, unless we were
     344             :      * told to skip it.
     345             :      */
     346          80 :     if (!skip_checksums)
     347          76 :         verify_backup_checksums(&context);
     348             : 
     349             :     /*
     350             :      * Try to parse the required ranges of WAL records, unless we were told
     351             :      * not to do so.
     352             :      */
     353          80 :     if (!no_parse_wal)
     354          78 :         parse_required_wal(&context, pg_waldump_path,
     355             :                            wal_directory, first_wal_range);
     356             : 
     357             :     /*
     358             :      * If everything looks OK, tell the user this, unless we were asked to
     359             :      * work quietly.
     360             :      */
     361          80 :     if (!context.saw_any_error && !quiet)
     362          48 :         printf(_("backup successfully verified\n"));
     363             : 
     364          80 :     return context.saw_any_error ? 1 : 0;
     365             : }
     366             : 
     367             : /*
     368             :  * Parse a manifest file. Construct a hash table with information about
     369             :  * all the files it mentions, and a linked list of all the WAL ranges it
     370             :  * mentions.
     371             :  */
     372             : static void
     373         148 : parse_manifest_file(char *manifest_path, manifest_files_hash **ht_p,
     374             :                     manifest_wal_range **first_wal_range_p)
     375             : {
     376             :     int         fd;
     377             :     struct stat statbuf;
     378             :     off_t       estimate;
     379             :     uint32      initial_size;
     380             :     manifest_files_hash *ht;
     381             :     char       *buffer;
     382             :     int         rc;
     383             :     parser_context private_context;
     384             :     JsonManifestParseContext context;
     385             : 
     386             :     /* Open the manifest file. */
     387         148 :     if ((fd = open(manifest_path, O_RDONLY | PG_BINARY, 0)) < 0)
     388           4 :         report_fatal_error("could not open file \"%s\": %m", manifest_path);
     389             : 
     390             :     /* Figure out how big the manifest is. */
     391         144 :     if (fstat(fd, &statbuf) != 0)
     392           0 :         report_fatal_error("could not stat file \"%s\": %m", manifest_path);
     393             : 
     394             :     /* Guess how large to make the hash table based on the manifest size. */
     395         144 :     estimate = statbuf.st_size / ESTIMATED_BYTES_PER_MANIFEST_LINE;
     396         144 :     initial_size = Min(PG_UINT32_MAX, Max(estimate, 256));
     397             : 
     398             :     /* Create the hash table. */
     399         144 :     ht = manifest_files_create(initial_size, NULL);
     400             : 
     401             :     /*
     402             :      * Slurp in the whole file.
     403             :      *
     404             :      * This is not ideal, but there's currently no easy way to get
     405             :      * pg_parse_json() to perform incremental parsing.
     406             :      */
     407         144 :     buffer = pg_malloc(statbuf.st_size);
     408         144 :     rc = read(fd, buffer, statbuf.st_size);
     409         144 :     if (rc != statbuf.st_size)
     410             :     {
     411           0 :         if (rc < 0)
     412           0 :             report_fatal_error("could not read file \"%s\": %m",
     413             :                                manifest_path);
     414             :         else
     415           0 :             report_fatal_error("could not read file \"%s\": read %d of %lld",
     416           0 :                                manifest_path, rc, (long long int) statbuf.st_size);
     417             :     }
     418             : 
     419             :     /* Close the manifest file. */
     420         144 :     close(fd);
     421             : 
     422             :     /* Parse the manifest. */
     423         144 :     private_context.ht = ht;
     424         144 :     private_context.first_wal_range = NULL;
     425         144 :     private_context.last_wal_range = NULL;
     426         144 :     context.private_data = &private_context;
     427         144 :     context.perfile_cb = record_manifest_details_for_file;
     428         144 :     context.perwalrange_cb = record_manifest_details_for_wal_range;
     429         144 :     context.error_cb = report_manifest_error;
     430         144 :     json_parse_manifest(&context, buffer, statbuf.st_size);
     431             : 
     432             :     /* Done with the buffer. */
     433          84 :     pfree(buffer);
     434             : 
     435             :     /* Return the file hash table and WAL range list we constructed. */
     436          84 :     *ht_p = ht;
     437          84 :     *first_wal_range_p = private_context.first_wal_range;
     438          84 : }
     439             : 
     440             : /*
     441             :  * Report an error while parsing the manifest.
     442             :  *
     443             :  * We consider all such errors to be fatal errors. The manifest parser
     444             :  * expects this function not to return.
     445             :  */
     446             : static void
     447          58 : report_manifest_error(JsonManifestParseContext *context, const char *fmt,...)
     448             : {
     449             :     va_list     ap;
     450             : 
     451          58 :     va_start(ap, fmt);
     452          58 :     pg_log_generic_v(PG_LOG_FATAL, gettext(fmt), ap);
     453          58 :     va_end(ap);
     454             : 
     455          58 :     exit(1);
     456             : }
     457             : 
     458             : /*
     459             :  * Record details extracted from the backup manifest for one file.
     460             :  */
     461             : static void
     462       81792 : record_manifest_details_for_file(JsonManifestParseContext *context,
     463             :                                  char *pathname, size_t size,
     464             :                                  pg_checksum_type checksum_type,
     465             :                                  int checksum_length, uint8 *checksum_payload)
     466             : {
     467       81792 :     parser_context *pcxt = context->private_data;
     468       81792 :     manifest_files_hash *ht = pcxt->ht;
     469             :     manifest_file *m;
     470             :     bool        found;
     471             : 
     472             :     /* Make a new entry in the hash table for this file. */
     473       81792 :     m = manifest_files_insert(ht, pathname, &found);
     474       81792 :     if (found)
     475           2 :         report_fatal_error("duplicate path name in backup manifest: \"%s\"",
     476             :                            pathname);
     477             : 
     478             :     /* Initialize the entry. */
     479       81790 :     m->size = size;
     480       81790 :     m->checksum_type = checksum_type;
     481       81790 :     m->checksum_length = checksum_length;
     482       81790 :     m->checksum_payload = checksum_payload;
     483       81790 :     m->matched = false;
     484       81790 :     m->bad = false;
     485       81790 : }
     486             : 
     487             : /*
     488             :  * Record details extracted from the backup manifest for one WAL range.
     489             :  */
     490             : static void
     491          88 : record_manifest_details_for_wal_range(JsonManifestParseContext *context,
     492             :                                       TimeLineID tli,
     493             :                                       XLogRecPtr start_lsn, XLogRecPtr end_lsn)
     494             : {
     495          88 :     parser_context *pcxt = context->private_data;
     496             :     manifest_wal_range *range;
     497             : 
     498             :     /* Allocate and initialize a struct describing this WAL range. */
     499          88 :     range = palloc(sizeof(manifest_wal_range));
     500          88 :     range->tli = tli;
     501          88 :     range->start_lsn = start_lsn;
     502          88 :     range->end_lsn = end_lsn;
     503          88 :     range->prev = pcxt->last_wal_range;
     504          88 :     range->next = NULL;
     505             : 
     506             :     /* Add it to the end of the list. */
     507          88 :     if (pcxt->first_wal_range == NULL)
     508          88 :         pcxt->first_wal_range = range;
     509             :     else
     510           0 :         pcxt->last_wal_range->next = range;
     511          88 :     pcxt->last_wal_range = range;
     512          88 : }
     513             : 
     514             : /*
     515             :  * Verify one directory.
     516             :  *
     517             :  * 'relpath' is NULL if we are to verify the top-level backup directory,
     518             :  * and otherwise the relative path to the directory that is to be verified.
     519             :  *
     520             :  * 'fullpath' is the backup directory with 'relpath' appended; i.e. the actual
     521             :  * filesystem path at which it can be found.
     522             :  */
     523             : static void
     524        2076 : verify_backup_directory(verifier_context *context, char *relpath,
     525             :                         char *fullpath)
     526             : {
     527             :     DIR        *dir;
     528             :     struct dirent *dirent;
     529             : 
     530        2076 :     dir = opendir(fullpath);
     531        2076 :     if (dir == NULL)
     532             :     {
     533             :         /*
     534             :          * If even the toplevel backup directory cannot be found, treat this
     535             :          * as a fatal error.
     536             :          */
     537           4 :         if (relpath == NULL)
     538           2 :             report_fatal_error("could not open directory \"%s\": %m", fullpath);
     539             : 
     540             :         /*
     541             :          * Otherwise, treat this as a non-fatal error, but ignore any further
     542             :          * errors related to this path and anything beneath it.
     543             :          */
     544           2 :         report_backup_error(context,
     545             :                             "could not open directory \"%s\": %m", fullpath);
     546           2 :         simple_string_list_append(&context->ignore_list, relpath);
     547             : 
     548           2 :         return;
     549             :     }
     550             : 
     551       84576 :     while (errno = 0, (dirent = readdir(dir)) != NULL)
     552             :     {
     553       82504 :         char       *filename = dirent->d_name;
     554       82504 :         char       *newfullpath = psprintf("%s/%s", fullpath, filename);
     555             :         char       *newrelpath;
     556             : 
     557             :         /* Skip "." and ".." */
     558       82504 :         if (filename[0] == '.' && (filename[1] == '\0'
     559        2072 :                                    || strcmp(filename, "..") == 0))
     560        4144 :             continue;
     561             : 
     562       78360 :         if (relpath == NULL)
     563        1956 :             newrelpath = pstrdup(filename);
     564             :         else
     565       76404 :             newrelpath = psprintf("%s/%s", relpath, filename);
     566             : 
     567       78360 :         if (!should_ignore_relpath(context, newrelpath))
     568       78114 :             verify_backup_file(context, newrelpath, newfullpath);
     569             : 
     570       78360 :         pfree(newfullpath);
     571       78360 :         pfree(newrelpath);
     572             :     }
     573             : 
     574        2072 :     if (closedir(dir))
     575             :     {
     576           0 :         report_backup_error(context,
     577             :                             "could not close directory \"%s\": %m", fullpath);
     578           0 :         return;
     579             :     }
     580             : }
     581             : 
     582             : /*
     583             :  * Verify one file (which might actually be a directory or a symlink).
     584             :  *
     585             :  * The arguments to this function have the same meaning as the arguments to
     586             :  * verify_backup_directory.
     587             :  */
     588             : static void
     589       78114 : verify_backup_file(verifier_context *context, char *relpath, char *fullpath)
     590             : {
     591             :     struct stat sb;
     592             :     manifest_file *m;
     593             : 
     594       78114 :     if (stat(fullpath, &sb) != 0)
     595             :     {
     596           6 :         report_backup_error(context,
     597             :                             "could not stat file or directory \"%s\": %m",
     598             :                             relpath);
     599             : 
     600             :         /*
     601             :          * Suppress further errors related to this path name and, if it's a
     602             :          * directory, anything underneath it.
     603             :          */
     604           6 :         simple_string_list_append(&context->ignore_list, relpath);
     605             : 
     606        2002 :         return;
     607             :     }
     608             : 
     609             :     /* If it's a directory, just recurse. */
     610       78108 :     if (S_ISDIR(sb.st_mode))
     611             :     {
     612        1992 :         verify_backup_directory(context, relpath, fullpath);
     613        1992 :         return;
     614             :     }
     615             : 
     616             :     /* If it's not a directory, it should be a plain file. */
     617       76116 :     if (!S_ISREG(sb.st_mode))
     618             :     {
     619           0 :         report_backup_error(context,
     620             :                             "\"%s\" is not a file or directory",
     621             :                             relpath);
     622           0 :         return;
     623             :     }
     624             : 
     625             :     /* Check whether there's an entry in the manifest hash. */
     626       76116 :     m = manifest_files_lookup(context->ht, relpath);
     627       76116 :     if (m == NULL)
     628             :     {
     629           4 :         report_backup_error(context,
     630             :                             "\"%s\" is present on disk but not in the manifest",
     631             :                             relpath);
     632           4 :         return;
     633             :     }
     634             : 
     635             :     /* Flag this entry as having been encountered in the filesystem. */
     636       76112 :     m->matched = true;
     637             : 
     638             :     /* Check that the size matches. */
     639       76112 :     if (m->size != sb.st_size)
     640             :     {
     641           4 :         report_backup_error(context,
     642             :                             "\"%s\" has size %lld on disk but size %zu in the manifest",
     643           4 :                             relpath, (long long int) sb.st_size, m->size);
     644           4 :         m->bad = true;
     645             :     }
     646             : 
     647             :     /*
     648             :      * We don't verify checksums at this stage. We first finish verifying that
     649             :      * we have the expected set of files with the expected sizes, and only
     650             :      * afterwards verify the checksums. That's because computing checksums may
     651             :      * take a while, and we'd like to report more obvious problems quickly.
     652             :      */
     653             : }
     654             : 
     655             : /*
     656             :  * Scan the hash table for entries where the 'matched' flag is not set; report
     657             :  * that such files are present in the manifest but not on disk.
     658             :  */
     659             : static void
     660          82 : report_extra_backup_files(verifier_context *context)
     661             : {
     662             :     manifest_files_iterator it;
     663             :     manifest_file *m;
     664             : 
     665          82 :     manifest_files_start_iterate(context->ht, &it);
     666       76262 :     while ((m = manifest_files_iterate(context->ht, &it)) != NULL)
     667       76182 :         if (!m->matched && !should_ignore_relpath(context, m->pathname))
     668          10 :             report_backup_error(context,
     669             :                                 "\"%s\" is present in the manifest but not on disk",
     670             :                                 m->pathname);
     671          80 : }
     672             : 
     673             : /*
     674             :  * Verify checksums for hash table entries that are otherwise unproblematic.
     675             :  * If we've already reported some problem related to a hash table entry, or
     676             :  * if it has no checksum, just skip it.
     677             :  */
     678             : static void
     679          76 : verify_backup_checksums(verifier_context *context)
     680             : {
     681             :     manifest_files_iterator it;
     682             :     manifest_file *m;
     683             : 
     684          76 :     manifest_files_start_iterate(context->ht, &it);
     685       72360 :     while ((m = manifest_files_iterate(context->ht, &it)) != NULL)
     686             :     {
     687       72284 :         if (m->matched && !m->bad && m->checksum_type != CHECKSUM_TYPE_NONE &&
     688       68518 :             !should_ignore_relpath(context, m->pathname))
     689             :         {
     690             :             char       *fullpath;
     691             : 
     692             :             /* Compute the full pathname to the target file. */
     693       68518 :             fullpath = psprintf("%s/%s", context->backup_directory,
     694             :                                 m->pathname);
     695             : 
     696             :             /* Do the actual checksum verification. */
     697       68518 :             verify_file_checksum(context, m, fullpath);
     698             : 
     699             :             /* Avoid leaking memory. */
     700       68518 :             pfree(fullpath);
     701             :         }
     702             :     }
     703          76 : }
     704             : 
     705             : /*
     706             :  * Verify the checksum of a single file.
     707             :  */
     708             : static void
     709       68518 : verify_file_checksum(verifier_context *context, manifest_file *m,
     710             :                      char *fullpath)
     711             : {
     712             :     pg_checksum_context checksum_ctx;
     713       68518 :     char       *relpath = m->pathname;
     714             :     int         fd;
     715             :     int         rc;
     716       68518 :     size_t      bytes_read = 0;
     717             :     uint8       buffer[READ_CHUNK_SIZE];
     718             :     uint8       checksumbuf[PG_CHECKSUM_MAX_LENGTH];
     719             :     int         checksumlen;
     720             : 
     721             :     /* Open the target file. */
     722       68518 :     if ((fd = open(fullpath, O_RDONLY | PG_BINARY, 0)) < 0)
     723             :     {
     724           2 :         report_backup_error(context, "could not open file \"%s\": %m",
     725             :                             relpath);
     726           2 :         return;
     727             :     }
     728             : 
     729             :     /* Initialize checksum context. */
     730       68516 :     if (pg_checksum_init(&checksum_ctx, m->checksum_type) < 0)
     731             :     {
     732           0 :         report_backup_error(context, "could not initialize checksum of file \"%s\"",
     733             :                             relpath);
     734           0 :         close(fd);
     735           0 :         return;
     736             :     }
     737             : 
     738             :     /* Read the file chunk by chunk, updating the checksum as we go. */
     739      457862 :     while ((rc = read(fd, buffer, READ_CHUNK_SIZE)) > 0)
     740             :     {
     741      389346 :         bytes_read += rc;
     742      389346 :         if (pg_checksum_update(&checksum_ctx, buffer, rc) < 0)
     743             :         {
     744           0 :             report_backup_error(context, "could not update checksum of file \"%s\"",
     745             :                                 relpath);
     746           0 :             close(fd);
     747           0 :             return;
     748             :         }
     749             : 
     750             : 
     751             :     }
     752       68516 :     if (rc < 0)
     753           0 :         report_backup_error(context, "could not read file \"%s\": %m",
     754             :                             relpath);
     755             : 
     756             :     /* Close the file. */
     757       68516 :     if (close(fd) != 0)
     758             :     {
     759           0 :         report_backup_error(context, "could not close file \"%s\": %m",
     760             :                             relpath);
     761           0 :         return;
     762             :     }
     763             : 
     764             :     /* If we didn't manage to read the whole file, bail out now. */
     765       68516 :     if (rc < 0)
     766           0 :         return;
     767             : 
     768             :     /*
     769             :      * Double-check that we read the expected number of bytes from the file.
     770             :      * Normally, a file size mismatch would be caught in verify_backup_file
     771             :      * and this check would never be reached, but this provides additional
     772             :      * safety and clarity in the event of concurrent modifications or
     773             :      * filesystem misbehavior.
     774             :      */
     775       68516 :     if (bytes_read != m->size)
     776             :     {
     777           0 :         report_backup_error(context,
     778             :                             "file \"%s\" should contain %zu bytes, but read %zu bytes",
     779             :                             relpath, m->size, bytes_read);
     780           0 :         return;
     781             :     }
     782             : 
     783             :     /* Get the final checksum. */
     784       68516 :     checksumlen = pg_checksum_final(&checksum_ctx, checksumbuf);
     785       68516 :     if (checksumlen < 0)
     786             :     {
     787           0 :         report_backup_error(context,
     788             :                             "could not finalize checksum of file \"%s\"",
     789             :                             relpath);
     790           0 :         return;
     791             :     }
     792             : 
     793             :     /* And check it against the manifest. */
     794       68516 :     if (checksumlen != m->checksum_length)
     795           0 :         report_backup_error(context,
     796             :                             "file \"%s\" has checksum of length %d, but expected %d",
     797             :                             relpath, m->checksum_length, checksumlen);
     798       68516 :     else if (memcmp(checksumbuf, m->checksum_payload, checksumlen) != 0)
     799           6 :         report_backup_error(context,
     800             :                             "checksum mismatch for file \"%s\"",
     801             :                             relpath);
     802             : }
     803             : 
     804             : /*
     805             :  * Attempt to parse the WAL files required to restore from backup using
     806             :  * pg_waldump.
     807             :  */
     808             : static void
     809          78 : parse_required_wal(verifier_context *context, char *pg_waldump_path,
     810             :                    char *wal_directory, manifest_wal_range *first_wal_range)
     811             : {
     812          78 :     manifest_wal_range *this_wal_range = first_wal_range;
     813             : 
     814         156 :     while (this_wal_range != NULL)
     815             :     {
     816             :         char       *pg_waldump_cmd;
     817             : 
     818          78 :         pg_waldump_cmd = psprintf("\"%s\" --quiet --path=\"%s\" --timeline=%u --start=%X/%X --end=%X/%X\n",
     819             :                                   pg_waldump_path, wal_directory, this_wal_range->tli,
     820          78 :                                   LSN_FORMAT_ARGS(this_wal_range->start_lsn),
     821          78 :                                   LSN_FORMAT_ARGS(this_wal_range->end_lsn));
     822          78 :         if (system(pg_waldump_cmd) != 0)
     823           4 :             report_backup_error(context,
     824             :                                 "WAL parsing failed for timeline %u",
     825             :                                 this_wal_range->tli);
     826             : 
     827          78 :         this_wal_range = this_wal_range->next;
     828             :     }
     829          78 : }
     830             : 
     831             : /*
     832             :  * Report a problem with the backup.
     833             :  *
     834             :  * Update the context to indicate that we saw an error, and exit if the
     835             :  * context says we should.
     836             :  */
     837             : static void
     838          38 : report_backup_error(verifier_context *context, const char *pg_restrict fmt,...)
     839             : {
     840             :     va_list     ap;
     841             : 
     842          38 :     va_start(ap, fmt);
     843          38 :     pg_log_generic_v(PG_LOG_ERROR, gettext(fmt), ap);
     844          38 :     va_end(ap);
     845             : 
     846          38 :     context->saw_any_error = true;
     847          38 :     if (context->exit_on_error)
     848           2 :         exit(1);
     849          36 : }
     850             : 
     851             : /*
     852             :  * Report a fatal error and exit
     853             :  */
     854             : static void
     855           8 : report_fatal_error(const char *pg_restrict fmt,...)
     856             : {
     857             :     va_list     ap;
     858             : 
     859           8 :     va_start(ap, fmt);
     860           8 :     pg_log_generic_v(PG_LOG_FATAL, gettext(fmt), ap);
     861           8 :     va_end(ap);
     862             : 
     863           8 :     exit(1);
     864             : }
     865             : 
     866             : /*
     867             :  * Is the specified relative path, or some prefix of it, listed in the set
     868             :  * of paths to ignore?
     869             :  *
     870             :  * Note that by "prefix" we mean a parent directory; for this purpose,
     871             :  * "aa/bb" is not a prefix of "aa/bbb", but it is a prefix of "aa/bb/cc".
     872             :  */
     873             : static bool
     874      148748 : should_ignore_relpath(verifier_context *context, char *relpath)
     875             : {
     876             :     SimpleStringListCell *cell;
     877             : 
     878      912530 :     for (cell = context->ignore_list.head; cell != NULL; cell = cell->next)
     879             :     {
     880      765888 :         char       *r = relpath;
     881      765888 :         char       *v = cell->val;
     882             : 
     883     1085930 :         while (*v != '\0' && *r == *v)
     884      320042 :             ++r, ++v;
     885             : 
     886      765888 :         if (*v == '\0' && (*r == '\0' || *r == '/'))
     887        2106 :             return true;
     888             :     }
     889             : 
     890      146642 :     return false;
     891             : }
     892             : 
     893             : /*
     894             :  * Helper function for manifest_files hash table.
     895             :  */
     896             : static uint32
     897      194490 : hash_string_pointer(char *s)
     898             : {
     899      194490 :     unsigned char *ss = (unsigned char *) s;
     900             : 
     901      194490 :     return hash_bytes(ss, strlen(s));
     902             : }
     903             : 
     904             : /*
     905             :  * Print out usage information and exit.
     906             :  */
     907             : static void
     908           2 : usage(void)
     909             : {
     910           2 :     printf(_("%s verifies a backup against the backup manifest.\n\n"), progname);
     911           2 :     printf(_("Usage:\n  %s [OPTION]... BACKUPDIR\n\n"), progname);
     912           2 :     printf(_("Options:\n"));
     913           2 :     printf(_("  -e, --exit-on-error         exit immediately on error\n"));
     914           2 :     printf(_("  -i, --ignore=RELATIVE_PATH  ignore indicated path\n"));
     915           2 :     printf(_("  -m, --manifest-path=PATH    use specified path for manifest\n"));
     916           2 :     printf(_("  -n, --no-parse-wal          do not try to parse WAL files\n"));
     917           2 :     printf(_("  -q, --quiet                 do not print any output, except for errors\n"));
     918           2 :     printf(_("  -s, --skip-checksums        skip checksum verification\n"));
     919           2 :     printf(_("  -w, --wal-directory=PATH    use specified path for WAL files\n"));
     920           2 :     printf(_("  -V, --version               output version information, then exit\n"));
     921           2 :     printf(_("  -?, --help                  show this help, then exit\n"));
     922           2 :     printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
     923           2 :     printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
     924           2 : }

Generated by: LCOV version 1.14