LCOV - code coverage report
Current view: top level - src/bin/pg_combinebackup - pg_combinebackup.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 287 452 63.5 %
Date: 2024-04-22 21:11:44 Functions: 13 15 86.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_combinebackup.c
       4             :  *      Combine incremental backups with prior backups.
       5             :  *
       6             :  * Copyright (c) 2017-2024, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/bin/pg_combinebackup/pg_combinebackup.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : #include "postgres_fe.h"
      14             : 
      15             : #include <dirent.h>
      16             : #include <fcntl.h>
      17             : #include <limits.h>
      18             : 
      19             : #include "backup_label.h"
      20             : #include "common/blkreftable.h"
      21             : #include "common/checksum_helper.h"
      22             : #include "common/controldata_utils.h"
      23             : #include "common/file_perm.h"
      24             : #include "common/file_utils.h"
      25             : #include "common/logging.h"
      26             : #include "copy_file.h"
      27             : #include "fe_utils/option_utils.h"
      28             : #include "getopt_long.h"
      29             : #include "lib/stringinfo.h"
      30             : #include "load_manifest.h"
      31             : #include "reconstruct.h"
      32             : #include "write_manifest.h"
      33             : 
      34             : /* Incremental file naming convention. */
      35             : #define INCREMENTAL_PREFIX          "INCREMENTAL."
      36             : #define INCREMENTAL_PREFIX_LENGTH   (sizeof(INCREMENTAL_PREFIX) - 1)
      37             : 
      38             : /*
      39             :  * Tracking for directories that need to be removed, or have their contents
      40             :  * removed, if the operation fails.
      41             :  */
      42             : typedef struct cb_cleanup_dir
      43             : {
      44             :     char       *target_path;
      45             :     bool        rmtopdir;
      46             :     struct cb_cleanup_dir *next;
      47             : } cb_cleanup_dir;
      48             : 
      49             : /*
      50             :  * Stores a tablespace mapping provided using -T, --tablespace-mapping.
      51             :  */
      52             : typedef struct cb_tablespace_mapping
      53             : {
      54             :     char        old_dir[MAXPGPATH];
      55             :     char        new_dir[MAXPGPATH];
      56             :     struct cb_tablespace_mapping *next;
      57             : } cb_tablespace_mapping;
      58             : 
      59             : /*
      60             :  * Stores data parsed from all command-line options.
      61             :  */
      62             : typedef struct cb_options
      63             : {
      64             :     bool        debug;
      65             :     char       *output;
      66             :     bool        dry_run;
      67             :     bool        no_sync;
      68             :     cb_tablespace_mapping *tsmappings;
      69             :     pg_checksum_type manifest_checksums;
      70             :     bool        no_manifest;
      71             :     DataDirSyncMethod sync_method;
      72             :     CopyMethod  copy_method;
      73             : } cb_options;
      74             : 
      75             : /*
      76             :  * Data about a tablespace.
      77             :  *
      78             :  * Every normal tablespace needs a tablespace mapping, but in-place tablespaces
      79             :  * don't, so the list of tablespaces can contain more entries than the list of
      80             :  * tablespace mappings.
      81             :  */
      82             : typedef struct cb_tablespace
      83             : {
      84             :     Oid         oid;
      85             :     bool        in_place;
      86             :     char        old_dir[MAXPGPATH];
      87             :     char        new_dir[MAXPGPATH];
      88             :     struct cb_tablespace *next;
      89             : } cb_tablespace;
      90             : 
      91             : /* Directories to be removed if we exit uncleanly. */
      92             : cb_cleanup_dir *cleanup_dir_list = NULL;
      93             : 
      94             : static void add_tablespace_mapping(cb_options *opt, char *arg);
      95             : static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
      96             : static uint64 check_control_files(int n_backups, char **backup_dirs);
      97             : static void check_input_dir_permissions(char *dir);
      98             : static void cleanup_directories_atexit(void);
      99             : static void create_output_directory(char *dirname, cb_options *opt);
     100             : static void help(const char *progname);
     101             : static bool parse_oid(char *s, Oid *result);
     102             : static void process_directory_recursively(Oid tsoid,
     103             :                                           char *input_directory,
     104             :                                           char *output_directory,
     105             :                                           char *relative_path,
     106             :                                           int n_prior_backups,
     107             :                                           char **prior_backup_dirs,
     108             :                                           manifest_data **manifests,
     109             :                                           manifest_writer *mwriter,
     110             :                                           cb_options *opt);
     111             : static int  read_pg_version_file(char *directory);
     112             : static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
     113             : static void reset_directory_cleanup_list(void);
     114             : static cb_tablespace *scan_for_existing_tablespaces(char *pathname,
     115             :                                                     cb_options *opt);
     116             : static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen);
     117             : 
     118             : /*
     119             :  * Main program.
     120             :  */
     121             : int
     122          44 : main(int argc, char *argv[])
     123             : {
     124             :     static struct option long_options[] = {
     125             :         {"debug", no_argument, NULL, 'd'},
     126             :         {"dry-run", no_argument, NULL, 'n'},
     127             :         {"no-sync", no_argument, NULL, 'N'},
     128             :         {"output", required_argument, NULL, 'o'},
     129             :         {"tablespace-mapping", no_argument, NULL, 'T'},
     130             :         {"manifest-checksums", required_argument, NULL, 1},
     131             :         {"no-manifest", no_argument, NULL, 2},
     132             :         {"sync-method", required_argument, NULL, 3},
     133             :         {"clone", no_argument, NULL, 4},
     134             :         {"copy-file-range", no_argument, NULL, 5},
     135             :         {NULL, 0, NULL, 0}
     136             :     };
     137             : 
     138             :     const char *progname;
     139             :     char       *last_input_dir;
     140             :     int         i;
     141             :     int         optindex;
     142             :     int         c;
     143             :     int         n_backups;
     144             :     int         n_prior_backups;
     145             :     int         version;
     146             :     uint64      system_identifier;
     147             :     char      **prior_backup_dirs;
     148             :     cb_options  opt;
     149             :     cb_tablespace *tablespaces;
     150             :     cb_tablespace *ts;
     151             :     StringInfo  last_backup_label;
     152             :     manifest_data **manifests;
     153             :     manifest_writer *mwriter;
     154             : 
     155          44 :     pg_logging_init(argv[0]);
     156          44 :     progname = get_progname(argv[0]);
     157          44 :     set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
     158          44 :     handle_help_version_opts(argc, argv, progname, help);
     159             : 
     160          40 :     memset(&opt, 0, sizeof(opt));
     161          40 :     opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
     162          40 :     opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
     163          40 :     opt.copy_method = COPY_METHOD_COPY;
     164             : 
     165             :     /* process command-line options */
     166          94 :     while ((c = getopt_long(argc, argv, "dnNPo:T:",
     167             :                             long_options, &optindex)) != -1)
     168             :     {
     169          56 :         switch (c)
     170             :         {
     171           8 :             case 'd':
     172           8 :                 opt.debug = true;
     173           8 :                 pg_logging_increase_verbosity();
     174           8 :                 break;
     175           0 :             case 'n':
     176           0 :                 opt.dry_run = true;
     177           0 :                 break;
     178           6 :             case 'N':
     179           6 :                 opt.no_sync = true;
     180           6 :                 break;
     181          34 :             case 'o':
     182          34 :                 opt.output = optarg;
     183          34 :                 break;
     184           0 :             case 'T':
     185           0 :                 add_tablespace_mapping(&opt, optarg);
     186           0 :                 break;
     187           4 :             case 1:
     188           4 :                 if (!pg_checksum_parse_type(optarg,
     189             :                                             &opt.manifest_checksums))
     190           0 :                     pg_fatal("unrecognized checksum algorithm: \"%s\"",
     191             :                              optarg);
     192           4 :                 break;
     193           2 :             case 2:
     194           2 :                 opt.no_manifest = true;
     195           2 :                 break;
     196           0 :             case 3:
     197           0 :                 if (!parse_sync_method(optarg, &opt.sync_method))
     198           0 :                     exit(1);
     199           0 :                 break;
     200           0 :             case 4:
     201           0 :                 opt.copy_method = COPY_METHOD_CLONE;
     202           0 :                 break;
     203           0 :             case 5:
     204           0 :                 opt.copy_method = COPY_METHOD_COPY_FILE_RANGE;
     205           0 :                 break;
     206           2 :             default:
     207             :                 /* getopt_long already emitted a complaint */
     208           2 :                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
     209           2 :                 exit(1);
     210             :         }
     211             :     }
     212             : 
     213          38 :     if (optind >= argc)
     214             :     {
     215           2 :         pg_log_error("%s: no input directories specified", progname);
     216           2 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
     217           2 :         exit(1);
     218             :     }
     219             : 
     220          36 :     if (opt.output == NULL)
     221           2 :         pg_fatal("no output directory specified");
     222             : 
     223             :     /* If no manifest is needed, no checksums are needed, either. */
     224          34 :     if (opt.no_manifest)
     225           2 :         opt.manifest_checksums = CHECKSUM_TYPE_NONE;
     226             : 
     227             :     /* Check that the platform supports the requested copy method. */
     228          34 :     if (opt.copy_method == COPY_METHOD_CLONE)
     229             :     {
     230             : #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
     231             :     (defined(__linux__) && defined(FICLONE))
     232             : 
     233             :         if (opt.dry_run)
     234             :             pg_log_debug("would use cloning to copy files");
     235             :         else
     236             :             pg_log_debug("will use cloning to copy files");
     237             : 
     238             : #else
     239           0 :         pg_fatal("file cloning not supported on this platform");
     240             : #endif
     241             :     }
     242          34 :     else if (opt.copy_method == COPY_METHOD_COPY_FILE_RANGE)
     243             :     {
     244             : #if defined(HAVE_COPY_FILE_RANGE)
     245             : 
     246           0 :         if (opt.dry_run)
     247           0 :             pg_log_debug("would use copy_file_range to copy blocks");
     248             :         else
     249           0 :             pg_log_debug("will use copy_file_range to copy blocks");
     250             : 
     251             : #else
     252             :         pg_fatal("copy_file_range not supported on this platform");
     253             : #endif
     254             :     }
     255             : 
     256             :     /* Read the server version from the final backup. */
     257          34 :     version = read_pg_version_file(argv[argc - 1]);
     258             : 
     259             :     /* Sanity-check control files. */
     260          34 :     n_backups = argc - optind;
     261          34 :     system_identifier = check_control_files(n_backups, argv + optind);
     262             : 
     263             :     /* Sanity-check backup_label files, and get the contents of the last one. */
     264          32 :     last_backup_label = check_backup_label_files(n_backups, argv + optind);
     265             : 
     266             :     /*
     267             :      * We'll need the pathnames to the prior backups. By "prior" we mean all
     268             :      * but the last one listed on the command line.
     269             :      */
     270          22 :     n_prior_backups = argc - optind - 1;
     271          22 :     prior_backup_dirs = argv + optind;
     272             : 
     273             :     /* Load backup manifests. */
     274          22 :     manifests = load_backup_manifests(n_backups, prior_backup_dirs);
     275             : 
     276             :     /*
     277             :      * Validate the manifest system identifier against the backup system
     278             :      * identifier.
     279             :      */
     280          62 :     for (i = 0; i < n_backups; i++)
     281             :     {
     282          42 :         if (manifests[i] &&
     283          42 :             manifests[i]->system_identifier != system_identifier)
     284             :         {
     285             :             char       *controlpath;
     286             : 
     287           2 :             controlpath = psprintf("%s/%s", prior_backup_dirs[i], "global/pg_control");
     288             : 
     289           2 :             pg_fatal("%s: manifest system identifier is %llu, but control file has %llu",
     290             :                      controlpath,
     291             :                      (unsigned long long) manifests[i]->system_identifier,
     292             :                      (unsigned long long) system_identifier);
     293             :         }
     294             :     }
     295             : 
     296             :     /* Figure out which tablespaces are going to be included in the output. */
     297          20 :     last_input_dir = argv[argc - 1];
     298          20 :     check_input_dir_permissions(last_input_dir);
     299          20 :     tablespaces = scan_for_existing_tablespaces(last_input_dir, &opt);
     300             : 
     301             :     /*
     302             :      * Create output directories.
     303             :      *
     304             :      * We create one output directory for the main data directory plus one for
     305             :      * each non-in-place tablespace. create_output_directory() will arrange
     306             :      * for those directories to be cleaned up on failure. In-place tablespaces
     307             :      * aren't handled at this stage because they're located beneath the main
     308             :      * output directory, and thus the cleanup of that directory will get rid
     309             :      * of them. Plus, the pg_tblspc directory that needs to contain them
     310             :      * doesn't exist yet.
     311             :      */
     312          20 :     atexit(cleanup_directories_atexit);
     313          20 :     create_output_directory(opt.output, &opt);
     314          20 :     for (ts = tablespaces; ts != NULL; ts = ts->next)
     315           0 :         if (!ts->in_place)
     316           0 :             create_output_directory(ts->new_dir, &opt);
     317             : 
     318             :     /* If we need to write a backup_manifest, prepare to do so. */
     319          20 :     if (!opt.dry_run && !opt.no_manifest)
     320             :     {
     321          18 :         mwriter = create_manifest_writer(opt.output, system_identifier);
     322             : 
     323             :         /*
     324             :          * Verify that we have a backup manifest for the final backup; else we
     325             :          * won't have the WAL ranges for the resulting manifest.
     326             :          */
     327          18 :         if (manifests[n_prior_backups] == NULL)
     328           0 :             pg_fatal("can't generate a manifest because no manifest is available for the final input backup");
     329             :     }
     330             :     else
     331           2 :         mwriter = NULL;
     332             : 
     333             :     /* Write backup label into output directory. */
     334          20 :     if (opt.dry_run)
     335           0 :         pg_log_debug("would generate \"%s/backup_label\"", opt.output);
     336             :     else
     337             :     {
     338          20 :         pg_log_debug("generating \"%s/backup_label\"", opt.output);
     339          20 :         last_backup_label->cursor = 0;
     340          20 :         write_backup_label(opt.output, last_backup_label,
     341             :                            opt.manifest_checksums, mwriter);
     342             :     }
     343             : 
     344             :     /* Process everything that's not part of a user-defined tablespace. */
     345          20 :     pg_log_debug("processing backup directory \"%s\"", last_input_dir);
     346          20 :     process_directory_recursively(InvalidOid, last_input_dir, opt.output,
     347             :                                   NULL, n_prior_backups, prior_backup_dirs,
     348             :                                   manifests, mwriter, &opt);
     349             : 
     350             :     /* Process user-defined tablespaces. */
     351          20 :     for (ts = tablespaces; ts != NULL; ts = ts->next)
     352             :     {
     353           0 :         pg_log_debug("processing tablespace directory \"%s\"", ts->old_dir);
     354             : 
     355             :         /*
     356             :          * If it's a normal tablespace, we need to set up a symbolic link from
     357             :          * pg_tblspc/${OID} to the target directory; if it's an in-place
     358             :          * tablespace, we need to create a directory at pg_tblspc/${OID}.
     359             :          */
     360           0 :         if (!ts->in_place)
     361             :         {
     362             :             char        linkpath[MAXPGPATH];
     363             : 
     364           0 :             snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output,
     365             :                      ts->oid);
     366             : 
     367           0 :             if (opt.dry_run)
     368           0 :                 pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
     369             :                              linkpath, ts->new_dir);
     370             :             else
     371             :             {
     372           0 :                 pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
     373             :                              linkpath, ts->new_dir);
     374           0 :                 if (symlink(ts->new_dir, linkpath) != 0)
     375           0 :                     pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
     376             :                              linkpath, ts->new_dir);
     377             :             }
     378             :         }
     379             :         else
     380             :         {
     381           0 :             if (opt.dry_run)
     382           0 :                 pg_log_debug("would create directory \"%s\"", ts->new_dir);
     383             :             else
     384             :             {
     385           0 :                 pg_log_debug("creating directory \"%s\"", ts->new_dir);
     386           0 :                 if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
     387           0 :                     pg_fatal("could not create directory \"%s\": %m",
     388             :                              ts->new_dir);
     389             :             }
     390             :         }
     391             : 
     392             :         /* OK, now handle the directory contents. */
     393           0 :         process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
     394             :                                       NULL, n_prior_backups, prior_backup_dirs,
     395             :                                       manifests, mwriter, &opt);
     396             :     }
     397             : 
     398             :     /* Finalize the backup_manifest, if we're generating one. */
     399          20 :     if (mwriter != NULL)
     400          18 :         finalize_manifest(mwriter,
     401          18 :                           manifests[n_prior_backups]->first_wal_range);
     402             : 
     403             :     /* fsync that output directory unless we've been told not to do so */
     404          20 :     if (!opt.no_sync)
     405             :     {
     406          14 :         if (opt.dry_run)
     407           0 :             pg_log_debug("would recursively fsync \"%s\"", opt.output);
     408             :         else
     409             :         {
     410          14 :             pg_log_debug("recursively fsyncing \"%s\"", opt.output);
     411          14 :             sync_pgdata(opt.output, version * 10000, opt.sync_method);
     412             :         }
     413             :     }
     414             : 
     415             :     /* It's a success, so don't remove the output directories. */
     416          20 :     reset_directory_cleanup_list();
     417          20 :     exit(0);
     418             : }
     419             : 
     420             : /*
     421             :  * Process the option argument for the -T, --tablespace-mapping switch.
     422             :  */
     423             : static void
     424           0 : add_tablespace_mapping(cb_options *opt, char *arg)
     425             : {
     426           0 :     cb_tablespace_mapping *tsmap = pg_malloc0(sizeof(cb_tablespace_mapping));
     427             :     char       *dst;
     428             :     char       *dst_ptr;
     429             :     char       *arg_ptr;
     430             : 
     431             :     /*
     432             :      * Basically, we just want to copy everything before the equals sign to
     433             :      * tsmap->old_dir and everything afterwards to tsmap->new_dir, but if
     434             :      * there's more or less than one equals sign, that's an error, and if
     435             :      * there's an equals sign preceded by a backslash, don't treat it as a
     436             :      * field separator but instead copy a literal equals sign.
     437             :      */
     438           0 :     dst_ptr = dst = tsmap->old_dir;
     439           0 :     for (arg_ptr = arg; *arg_ptr != '\0'; arg_ptr++)
     440             :     {
     441           0 :         if (dst_ptr - dst >= MAXPGPATH)
     442           0 :             pg_fatal("directory name too long");
     443             : 
     444           0 :         if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
     445             :             ;                   /* skip backslash escaping = */
     446           0 :         else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
     447             :         {
     448           0 :             if (tsmap->new_dir[0] != '\0')
     449           0 :                 pg_fatal("multiple \"=\" signs in tablespace mapping");
     450             :             else
     451           0 :                 dst = dst_ptr = tsmap->new_dir;
     452             :         }
     453             :         else
     454           0 :             *dst_ptr++ = *arg_ptr;
     455             :     }
     456           0 :     if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
     457           0 :         pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
     458             : 
     459             :     /*
     460             :      * All tablespaces are created with absolute directories, so specifying a
     461             :      * non-absolute path here would never match, possibly confusing users.
     462             :      *
     463             :      * In contrast to pg_basebackup, both the old and new directories are on
     464             :      * the local machine, so the local machine's definition of an absolute
     465             :      * path is the only relevant one.
     466             :      */
     467           0 :     if (!is_absolute_path(tsmap->old_dir))
     468           0 :         pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
     469             :                  tsmap->old_dir);
     470             : 
     471           0 :     if (!is_absolute_path(tsmap->new_dir))
     472           0 :         pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
     473             :                  tsmap->new_dir);
     474             : 
     475             :     /* Canonicalize paths to avoid spurious failures when comparing. */
     476           0 :     canonicalize_path(tsmap->old_dir);
     477           0 :     canonicalize_path(tsmap->new_dir);
     478             : 
     479             :     /* Add it to the list. */
     480           0 :     tsmap->next = opt->tsmappings;
     481           0 :     opt->tsmappings = tsmap;
     482           0 : }
     483             : 
     484             : /*
     485             :  * Check that the backup_label files form a coherent backup chain, and return
     486             :  * the contents of the backup_label file from the latest backup.
     487             :  */
     488             : static StringInfo
     489          32 : check_backup_label_files(int n_backups, char **backup_dirs)
     490             : {
     491          32 :     StringInfo  buf = makeStringInfo();
     492          32 :     StringInfo  lastbuf = buf;
     493             :     int         i;
     494          32 :     TimeLineID  check_tli = 0;
     495          32 :     XLogRecPtr  check_lsn = InvalidXLogRecPtr;
     496             : 
     497             :     /* Try to read each backup_label file in turn, last to first. */
     498          84 :     for (i = n_backups - 1; i >= 0; --i)
     499             :     {
     500             :         char        pathbuf[MAXPGPATH];
     501             :         int         fd;
     502             :         TimeLineID  start_tli;
     503             :         TimeLineID  previous_tli;
     504             :         XLogRecPtr  start_lsn;
     505             :         XLogRecPtr  previous_lsn;
     506             : 
     507             :         /* Open the backup_label file. */
     508          62 :         snprintf(pathbuf, MAXPGPATH, "%s/backup_label", backup_dirs[i]);
     509          62 :         pg_log_debug("reading \"%s\"", pathbuf);
     510          62 :         if ((fd = open(pathbuf, O_RDONLY, 0)) < 0)
     511           0 :             pg_fatal("could not open file \"%s\": %m", pathbuf);
     512             : 
     513             :         /*
     514             :          * Slurp the whole file into memory.
     515             :          *
     516             :          * The exact size limit that we impose here doesn't really matter --
     517             :          * most of what's supposed to be in the file is fixed size and quite
     518             :          * short. However, the length of the backup_label is limited (at least
     519             :          * by some parts of the code) to MAXPGPATH, so include that value in
     520             :          * the maximum length that we tolerate.
     521             :          */
     522          62 :         slurp_file(fd, pathbuf, buf, 10000 + MAXPGPATH);
     523             : 
     524             :         /* Close the file. */
     525          62 :         if (close(fd) != 0)
     526           0 :             pg_fatal("could not close \"%s\": %m", pathbuf);
     527             : 
     528             :         /* Parse the file contents. */
     529          62 :         parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
     530             :                            &previous_tli, &previous_lsn);
     531             : 
     532             :         /*
     533             :          * Sanity checks.
     534             :          *
     535             :          * XXX. It's actually not required that start_lsn == check_lsn. It
     536             :          * would be OK if start_lsn > check_lsn provided that start_lsn is
     537             :          * less than or equal to the relevant switchpoint. But at the moment
     538             :          * we don't have that information.
     539             :          */
     540          62 :         if (i > 0 && previous_tli == 0)
     541           2 :             pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
     542             :                      backup_dirs[i]);
     543          60 :         if (i == 0 && previous_tli != 0)
     544           2 :             pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
     545             :                      backup_dirs[i]);
     546          58 :         if (i < n_backups - 1 && start_tli != check_tli)
     547           0 :             pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
     548             :                      backup_dirs[i], start_tli, check_tli);
     549          58 :         if (i < n_backups - 1 && start_lsn != check_lsn)
     550           6 :             pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
     551             :                      backup_dirs[i],
     552             :                      LSN_FORMAT_ARGS(start_lsn),
     553             :                      LSN_FORMAT_ARGS(check_lsn));
     554          52 :         check_tli = previous_tli;
     555          52 :         check_lsn = previous_lsn;
     556             : 
     557             :         /*
     558             :          * The last backup label in the chain needs to be saved for later use,
     559             :          * while the others are only needed within this loop.
     560             :          */
     561          52 :         if (lastbuf == buf)
     562          30 :             buf = makeStringInfo();
     563             :         else
     564          22 :             resetStringInfo(buf);
     565             :     }
     566             : 
     567             :     /* Free memory that we don't need any more. */
     568          22 :     if (lastbuf != buf)
     569          22 :         destroyStringInfo(buf);
     570             : 
     571             :     /*
     572             :      * Return the data from the first backup_info that we read (which is the
     573             :      * backup_label from the last directory specified on the command line).
     574             :      */
     575          22 :     return lastbuf;
     576             : }
     577             : 
     578             : /*
     579             :  * Sanity check control files and return system_identifier.
     580             :  */
     581             : static uint64
     582          34 : check_control_files(int n_backups, char **backup_dirs)
     583             : {
     584             :     int         i;
     585          34 :     uint64      system_identifier = 0;  /* placate compiler */
     586             : 
     587             :     /* Try to read each control file in turn, last to first. */
     588         102 :     for (i = n_backups - 1; i >= 0; --i)
     589             :     {
     590             :         ControlFileData *control_file;
     591             :         bool        crc_ok;
     592             :         char       *controlpath;
     593             : 
     594          70 :         controlpath = psprintf("%s/%s", backup_dirs[i], "global/pg_control");
     595          70 :         pg_log_debug("reading \"%s\"", controlpath);
     596          70 :         control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
     597             : 
     598             :         /* Control file contents not meaningful if CRC is bad. */
     599          70 :         if (!crc_ok)
     600           0 :             pg_fatal("%s: CRC is incorrect", controlpath);
     601             : 
     602             :         /* Can't interpret control file if not current version. */
     603          70 :         if (control_file->pg_control_version != PG_CONTROL_VERSION)
     604           0 :             pg_fatal("%s: unexpected control file version",
     605             :                      controlpath);
     606             : 
     607             :         /* System identifiers should all match. */
     608          70 :         if (i == n_backups - 1)
     609          34 :             system_identifier = control_file->system_identifier;
     610          36 :         else if (system_identifier != control_file->system_identifier)
     611           2 :             pg_fatal("%s: expected system identifier %llu, but found %llu",
     612             :                      controlpath, (unsigned long long) system_identifier,
     613             :                      (unsigned long long) control_file->system_identifier);
     614             : 
     615             :         /* Release memory. */
     616          68 :         pfree(control_file);
     617          68 :         pfree(controlpath);
     618             :     }
     619             : 
     620             :     /*
     621             :      * If debug output is enabled, make a note of the system identifier that
     622             :      * we found in all of the relevant control files.
     623             :      */
     624          32 :     pg_log_debug("system identifier is %llu",
     625             :                  (unsigned long long) system_identifier);
     626             : 
     627          32 :     return system_identifier;
     628             : }
     629             : 
     630             : /*
     631             :  * Set default permissions for new files and directories based on the
     632             :  * permissions of the given directory. The intent here is that the output
     633             :  * directory should use the same permissions scheme as the final input
     634             :  * directory.
     635             :  */
     636             : static void
     637          20 : check_input_dir_permissions(char *dir)
     638             : {
     639             :     struct stat st;
     640             : 
     641          20 :     if (stat(dir, &st) != 0)
     642           0 :         pg_fatal("could not stat \"%s\": %m", dir);
     643             : 
     644          20 :     SetDataDirectoryCreatePerm(st.st_mode);
     645          20 : }
     646             : 
     647             : /*
     648             :  * Clean up output directories before exiting.
     649             :  */
     650             : static void
     651          20 : cleanup_directories_atexit(void)
     652             : {
     653          20 :     while (cleanup_dir_list != NULL)
     654             :     {
     655           0 :         cb_cleanup_dir *dir = cleanup_dir_list;
     656             : 
     657           0 :         if (dir->rmtopdir)
     658             :         {
     659           0 :             pg_log_info("removing output directory \"%s\"", dir->target_path);
     660           0 :             if (!rmtree(dir->target_path, dir->rmtopdir))
     661           0 :                 pg_log_error("failed to remove output directory");
     662             :         }
     663             :         else
     664             :         {
     665           0 :             pg_log_info("removing contents of output directory \"%s\"",
     666             :                         dir->target_path);
     667           0 :             if (!rmtree(dir->target_path, dir->rmtopdir))
     668           0 :                 pg_log_error("failed to remove contents of output directory");
     669             :         }
     670             : 
     671           0 :         cleanup_dir_list = cleanup_dir_list->next;
     672           0 :         pfree(dir);
     673             :     }
     674          20 : }
     675             : 
     676             : /*
     677             :  * Create the named output directory, unless it already exists or we're in
     678             :  * dry-run mode. If it already exists but is not empty, that's a fatal error.
     679             :  *
     680             :  * Adds the created directory to the list of directories to be cleaned up
     681             :  * at process exit.
     682             :  */
     683             : static void
     684          20 : create_output_directory(char *dirname, cb_options *opt)
     685             : {
     686          20 :     switch (pg_check_dir(dirname))
     687             :     {
     688          20 :         case 0:
     689          20 :             if (opt->dry_run)
     690             :             {
     691           0 :                 pg_log_debug("would create directory \"%s\"", dirname);
     692           0 :                 return;
     693             :             }
     694          20 :             pg_log_debug("creating directory \"%s\"", dirname);
     695          20 :             if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
     696           0 :                 pg_fatal("could not create directory \"%s\": %m", dirname);
     697          20 :             remember_to_cleanup_directory(dirname, true);
     698          20 :             break;
     699             : 
     700           0 :         case 1:
     701           0 :             pg_log_debug("using existing directory \"%s\"", dirname);
     702           0 :             remember_to_cleanup_directory(dirname, false);
     703           0 :             break;
     704             : 
     705           0 :         case 2:
     706             :         case 3:
     707             :         case 4:
     708           0 :             pg_fatal("directory \"%s\" exists but is not empty", dirname);
     709             : 
     710           0 :         case -1:
     711           0 :             pg_fatal("could not access directory \"%s\": %m", dirname);
     712             :     }
     713             : }
     714             : 
     715             : /*
     716             :  * help
     717             :  *
     718             :  * Prints help page for the program
     719             :  *
     720             :  * progname: the name of the executed program, such as "pg_combinebackup"
     721             :  */
     722             : static void
     723           2 : help(const char *progname)
     724             : {
     725           2 :     printf(_("%s reconstructs full backups from incrementals.\n\n"), progname);
     726           2 :     printf(_("Usage:\n"));
     727           2 :     printf(_("  %s [OPTION]... DIRECTORY...\n"), progname);
     728           2 :     printf(_("\nOptions:\n"));
     729           2 :     printf(_("  -d, --debug               generate lots of debugging output\n"));
     730           2 :     printf(_("  -n, --dry-run             don't actually do anything\n"));
     731           2 :     printf(_("  -N, --no-sync             do not wait for changes to be written safely to disk\n"));
     732           2 :     printf(_("  -o, --output              output directory\n"));
     733           2 :     printf(_("  -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
     734             :              "                            relocate tablespace in OLDDIR to NEWDIR\n"));
     735           2 :     printf(_("      --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
     736             :              "                            use algorithm for manifest checksums\n"));
     737           2 :     printf(_("      --no-manifest         suppress generation of backup manifest\n"));
     738           2 :     printf(_("      --sync-method=METHOD  set method for syncing files to disk\n"));
     739           2 :     printf(_("      --clone               clone (reflink) instead of copying files\n"));
     740           2 :     printf(_("      --copy-file-range     copy using copy_file_range() syscall\n"));
     741           2 :     printf(_("  -?, --help                show this help, then exit\n"));
     742             : 
     743           2 :     printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
     744           2 :     printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
     745           2 : }
     746             : 
     747             : /*
     748             :  * Try to parse a string as a non-zero OID without leading zeroes.
     749             :  *
     750             :  * If it works, return true and set *result to the answer, else return false.
     751             :  */
     752             : static bool
     753           0 : parse_oid(char *s, Oid *result)
     754             : {
     755             :     Oid         oid;
     756             :     char       *ep;
     757             : 
     758           0 :     errno = 0;
     759           0 :     oid = strtoul(s, &ep, 10);
     760           0 :     if (errno != 0 || *ep != '\0' || oid < 1 || oid > PG_UINT32_MAX)
     761           0 :         return false;
     762             : 
     763           0 :     *result = oid;
     764           0 :     return true;
     765             : }
     766             : 
     767             : /*
     768             :  * Copy files from the input directory to the output directory, reconstructing
     769             :  * full files from incremental files as required.
     770             :  *
     771             :  * If processing is a user-defined tablespace, the tsoid should be the OID
     772             :  * of that tablespace and input_directory and output_directory should be the
     773             :  * toplevel input and output directories for that tablespace. Otherwise,
     774             :  * tsoid should be InvalidOid and input_directory and output_directory should
     775             :  * be the main input and output directories.
     776             :  *
     777             :  * relative_path is the path beneath the given input and output directories
     778             :  * that we are currently processing. If NULL, it indicates that we're
     779             :  * processing the input and output directories themselves.
     780             :  *
     781             :  * n_prior_backups is the number of prior backups that we have available.
     782             :  * This doesn't count the very last backup, which is referenced by
     783             :  * output_directory, just the older ones. prior_backup_dirs is an array of
     784             :  * the locations of those previous backups.
     785             :  */
     786             : static void
     787         544 : process_directory_recursively(Oid tsoid,
     788             :                               char *input_directory,
     789             :                               char *output_directory,
     790             :                               char *relative_path,
     791             :                               int n_prior_backups,
     792             :                               char **prior_backup_dirs,
     793             :                               manifest_data **manifests,
     794             :                               manifest_writer *mwriter,
     795             :                               cb_options *opt)
     796             : {
     797             :     char        ifulldir[MAXPGPATH];
     798             :     char        ofulldir[MAXPGPATH];
     799             :     char        manifest_prefix[MAXPGPATH];
     800             :     DIR        *dir;
     801             :     struct dirent *de;
     802         544 :     bool        is_pg_tblspc = false;
     803         544 :     bool        is_pg_wal = false;
     804         544 :     bool        is_incremental_dir = false;
     805         544 :     manifest_data *latest_manifest = manifests[n_prior_backups];
     806             :     pg_checksum_type checksum_type;
     807             : 
     808             :     /*
     809             :      * Classify this directory.
     810             :      *
     811             :      * We set is_pg_tblspc only for the toplevel pg_tblspc directory, because
     812             :      * the symlinks in that specific directory require special handling.
     813             :      *
     814             :      * We set is_pg_wal for the toplevel WAL directory and all of its
     815             :      * subdirectories, because those files are not included in the backup
     816             :      * manifest and hence need special treatement. (Since incremental backup
     817             :      * does not exist in pre-v10 versions, we don't have to worry about the
     818             :      * old pg_xlog naming.)
     819             :      *
     820             :      * We set is_incremental_dir for directories that can contain incremental
     821             :      * files requiring reconstruction. If such files occur outside these
     822             :      * directories, we want to just copy them straight to the output
     823             :      * directory. This is to protect against a user creating a file with a
     824             :      * strange name like INCREMENTAL.config and then compaining that
     825             :      * incremental backups don't work properly. The test here is a bit tricky:
     826             :      * incremental files occur in subdirectories of base, in pg_global itself,
     827             :      * and in subdirectories of pg_tblspc only if in-place tablespaces are
     828             :      * used.
     829             :      */
     830         544 :     if (OidIsValid(tsoid))
     831           0 :         is_incremental_dir = true;
     832         544 :     else if (relative_path != NULL)
     833             :     {
     834         524 :         is_pg_tblspc = strcmp(relative_path, "pg_tblspc") == 0;
     835        1028 :         is_pg_wal = (strcmp(relative_path, "pg_wal") == 0 ||
     836         504 :                      strncmp(relative_path, "pg_wal/", 7) == 0);
     837         524 :         is_incremental_dir = strncmp(relative_path, "base/", 5) == 0 ||
     838         964 :             strcmp(relative_path, "global") == 0 ||
     839         440 :             strncmp(relative_path, "pg_tblspc/", 10) == 0;
     840             :     }
     841             : 
     842             :     /*
     843             :      * If we're under pg_wal, then we don't need checksums, because these
     844             :      * files aren't included in the backup manifest. Otherwise use whatever
     845             :      * type of checksum is configured.
     846             :      */
     847         544 :     if (!is_pg_wal)
     848         484 :         checksum_type = opt->manifest_checksums;
     849             :     else
     850          60 :         checksum_type = CHECKSUM_TYPE_NONE;
     851             : 
     852             :     /*
     853             :      * Append the relative path to the input and output directories, and
     854             :      * figure out the appropriate prefix to add to files in this directory
     855             :      * when looking them up in a backup manifest.
     856             :      */
     857         544 :     if (relative_path == NULL)
     858             :     {
     859          20 :         strlcpy(ifulldir, input_directory, MAXPGPATH);
     860          20 :         strlcpy(ofulldir, output_directory, MAXPGPATH);
     861          20 :         if (OidIsValid(tsoid))
     862           0 :             snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/", tsoid);
     863             :         else
     864          20 :             manifest_prefix[0] = '\0';
     865             :     }
     866             :     else
     867             :     {
     868         524 :         snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
     869             :                  relative_path);
     870         524 :         snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
     871             :                  relative_path);
     872         524 :         if (OidIsValid(tsoid))
     873           0 :             snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/",
     874             :                      tsoid, relative_path);
     875             :         else
     876         524 :             snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
     877             :     }
     878             : 
     879             :     /*
     880             :      * Toplevel output directories have already been created by the time this
     881             :      * function is called, but any subdirectories are our responsibility.
     882             :      */
     883         544 :     if (relative_path != NULL)
     884             :     {
     885         524 :         if (opt->dry_run)
     886           0 :             pg_log_debug("would create directory \"%s\"", ofulldir);
     887             :         else
     888             :         {
     889         524 :             pg_log_debug("creating directory \"%s\"", ofulldir);
     890         524 :             if (mkdir(ofulldir, pg_dir_create_mode) == -1)
     891           0 :                 pg_fatal("could not create directory \"%s\": %m", ofulldir);
     892             :         }
     893             :     }
     894             : 
     895             :     /* It's time to scan the directory. */
     896         544 :     if ((dir = opendir(ifulldir)) == NULL)
     897           0 :         pg_fatal("could not open directory \"%s\": %m", ifulldir);
     898       22770 :     while (errno = 0, (de = readdir(dir)) != NULL)
     899             :     {
     900             :         PGFileType  type;
     901             :         char        ifullpath[MAXPGPATH];
     902             :         char        ofullpath[MAXPGPATH];
     903             :         char        manifest_path[MAXPGPATH];
     904       22226 :         Oid         oid = InvalidOid;
     905       22226 :         int         checksum_length = 0;
     906       22226 :         uint8      *checksum_payload = NULL;
     907             :         pg_checksum_context checksum_ctx;
     908             : 
     909             :         /* Ignore "." and ".." entries. */
     910       22226 :         if (strcmp(de->d_name, ".") == 0 ||
     911       21682 :             strcmp(de->d_name, "..") == 0)
     912        1652 :             continue;
     913             : 
     914             :         /* Construct input path. */
     915       21138 :         snprintf(ifullpath, MAXPGPATH, "%s/%s", ifulldir, de->d_name);
     916             : 
     917             :         /* Figure out what kind of directory entry this is. */
     918       21138 :         type = get_dirent_type(ifullpath, de, false, PG_LOG_ERROR);
     919       21138 :         if (type == PGFILETYPE_ERROR)
     920           0 :             exit(1);
     921             : 
     922             :         /*
     923             :          * If we're processing pg_tblspc, then check whether the filename
     924             :          * looks like it could be a tablespace OID. If so, and if the
     925             :          * directory entry is a symbolic link or a directory, skip it.
     926             :          *
     927             :          * Our goal here is to ignore anything that would have been considered
     928             :          * by scan_for_existing_tablespaces to be a tablespace.
     929             :          */
     930       21138 :         if (is_pg_tblspc && parse_oid(de->d_name, &oid) &&
     931           0 :             (type == PGFILETYPE_LNK || type == PGFILETYPE_DIR))
     932           0 :             continue;
     933             : 
     934             :         /* If it's a directory, recurse. */
     935       21138 :         if (type == PGFILETYPE_DIR)
     936             :         {
     937             :             char        new_relative_path[MAXPGPATH];
     938             : 
     939             :             /* Append new pathname component to relative path. */
     940         524 :             if (relative_path == NULL)
     941         340 :                 strlcpy(new_relative_path, de->d_name, MAXPGPATH);
     942             :             else
     943         184 :                 snprintf(new_relative_path, MAXPGPATH, "%s/%s", relative_path,
     944         184 :                          de->d_name);
     945             : 
     946             :             /* And recurse. */
     947         524 :             process_directory_recursively(tsoid,
     948             :                                           input_directory, output_directory,
     949             :                                           new_relative_path,
     950             :                                           n_prior_backups, prior_backup_dirs,
     951             :                                           manifests, mwriter, opt);
     952         524 :             continue;
     953             :         }
     954             : 
     955             :         /* Skip anything that's not a regular file. */
     956       20614 :         if (type != PGFILETYPE_REG)
     957             :         {
     958           0 :             if (type == PGFILETYPE_LNK)
     959           0 :                 pg_log_warning("skipping symbolic link \"%s\"", ifullpath);
     960             :             else
     961           0 :                 pg_log_warning("skipping special file \"%s\"", ifullpath);
     962           0 :             continue;
     963             :         }
     964             : 
     965             :         /*
     966             :          * Skip the backup_label and backup_manifest files; they require
     967             :          * special handling and are handled elsewhere.
     968             :          */
     969       20614 :         if (relative_path == NULL &&
     970         148 :             (strcmp(de->d_name, "backup_label") == 0 ||
     971         128 :              strcmp(de->d_name, "backup_manifest") == 0))
     972          40 :             continue;
     973             : 
     974             :         /*
     975             :          * If it's an incremental file, hand it off to the reconstruction
     976             :          * code, which will figure out what to do.
     977             :          */
     978       20574 :         if (is_incremental_dir &&
     979       20366 :             strncmp(de->d_name, INCREMENTAL_PREFIX,
     980             :                     INCREMENTAL_PREFIX_LENGTH) == 0)
     981             :         {
     982             :             /* Output path should not include "INCREMENTAL." prefix. */
     983        9302 :             snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
     984        9302 :                      de->d_name + INCREMENTAL_PREFIX_LENGTH);
     985             : 
     986             : 
     987             :             /* Manifest path likewise omits incremental prefix. */
     988        9302 :             snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
     989        9302 :                      de->d_name + INCREMENTAL_PREFIX_LENGTH);
     990             : 
     991             :             /* Reconstruction logic will do the rest. */
     992        9302 :             reconstruct_from_incremental_file(ifullpath, ofullpath,
     993             :                                               manifest_prefix,
     994        9302 :                                               de->d_name + INCREMENTAL_PREFIX_LENGTH,
     995             :                                               n_prior_backups,
     996             :                                               prior_backup_dirs,
     997             :                                               manifests,
     998             :                                               manifest_path,
     999             :                                               checksum_type,
    1000             :                                               &checksum_length,
    1001             :                                               &checksum_payload,
    1002             :                                               opt->copy_method,
    1003        9302 :                                               opt->debug,
    1004        9302 :                                               opt->dry_run);
    1005             :         }
    1006             :         else
    1007             :         {
    1008             :             /* Construct the path that the backup_manifest will use. */
    1009       11272 :             snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
    1010       11272 :                      de->d_name);
    1011             : 
    1012             :             /*
    1013             :              * It's not an incremental file, so we need to copy the entire
    1014             :              * file to the output directory.
    1015             :              *
    1016             :              * If a checksum of the required type already exists in the
    1017             :              * backup_manifest for the final input directory, we can save some
    1018             :              * work by reusing that checksum instead of computing a new one.
    1019             :              */
    1020       11272 :             if (checksum_type != CHECKSUM_TYPE_NONE &&
    1021             :                 latest_manifest != NULL)
    1022             :             {
    1023             :                 manifest_file *mfile;
    1024             : 
    1025        7392 :                 mfile = manifest_files_lookup(latest_manifest->files,
    1026             :                                               manifest_path);
    1027        7392 :                 if (mfile == NULL)
    1028             :                 {
    1029             :                     char       *bmpath;
    1030             : 
    1031             :                     /*
    1032             :                      * The directory is out of sync with the backup_manifest,
    1033             :                      * so emit a warning.
    1034             :                      */
    1035           0 :                     bmpath = psprintf("%s/%s", input_directory,
    1036             :                                       "backup_manifest");
    1037           0 :                     pg_log_warning("\"%s\" contains no entry for \"%s\"",
    1038             :                                    bmpath, manifest_path);
    1039           0 :                     pfree(bmpath);
    1040             :                 }
    1041        7392 :                 else if (mfile->checksum_type == checksum_type)
    1042             :                 {
    1043        5462 :                     checksum_length = mfile->checksum_length;
    1044        5462 :                     checksum_payload = mfile->checksum_payload;
    1045             :                 }
    1046             :             }
    1047             : 
    1048             :             /*
    1049             :              * If we're reusing a checksum, then we don't need copy_file() to
    1050             :              * compute one for us, but otherwise, it needs to compute whatever
    1051             :              * type of checksum we need.
    1052             :              */
    1053       11272 :             if (checksum_length != 0)
    1054        5462 :                 pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_NONE);
    1055             :             else
    1056        5810 :                 pg_checksum_init(&checksum_ctx, checksum_type);
    1057             : 
    1058             :             /* Actually copy the file. */
    1059       11272 :             snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
    1060       11272 :             copy_file(ifullpath, ofullpath, &checksum_ctx,
    1061       11272 :                       opt->copy_method, opt->dry_run);
    1062             : 
    1063             :             /*
    1064             :              * If copy_file() performed a checksum calculation for us, then
    1065             :              * save the results (except in dry-run mode, when there's no
    1066             :              * point).
    1067             :              */
    1068       11272 :             if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
    1069             :             {
    1070        1930 :                 checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
    1071        1930 :                 checksum_length = pg_checksum_final(&checksum_ctx,
    1072             :                                                     checksum_payload);
    1073             :             }
    1074             :         }
    1075             : 
    1076             :         /* Generate manifest entry, if needed. */
    1077       20574 :         if (mwriter != NULL)
    1078             :         {
    1079             :             struct stat sb;
    1080             : 
    1081             :             /*
    1082             :              * In order to generate a manifest entry, we need the file size
    1083             :              * and mtime. We have no way to know the correct mtime except to
    1084             :              * stat() the file, so just do that and get the size as well.
    1085             :              *
    1086             :              * If we didn't need the mtime here, we could try to obtain the
    1087             :              * file size from the reconstruction or file copy process above,
    1088             :              * although that is actually not convenient in all cases. If we
    1089             :              * write the file ourselves then clearly we can keep a count of
    1090             :              * bytes, but if we use something like CopyFile() then it's
    1091             :              * trickier. Since we have to stat() anyway to get the mtime,
    1092             :              * there's no point in worrying about it.
    1093             :              */
    1094       18642 :             if (stat(ofullpath, &sb) < 0)
    1095           0 :                 pg_fatal("could not stat file \"%s\": %m", ofullpath);
    1096             : 
    1097             :             /* OK, now do the work. */
    1098       18642 :             add_file_to_manifest(mwriter, manifest_path,
    1099       18642 :                                  sb.st_size, sb.st_mtime,
    1100             :                                  checksum_type, checksum_length,
    1101             :                                  checksum_payload);
    1102             :         }
    1103             : 
    1104             :         /* Avoid leaking memory. */
    1105       20574 :         if (checksum_payload != NULL)
    1106       16694 :             pfree(checksum_payload);
    1107             :     }
    1108             : 
    1109         544 :     closedir(dir);
    1110         544 : }
    1111             : 
    1112             : /*
    1113             :  * Read the version number from PG_VERSION and convert it to the usual server
    1114             :  * version number format. (e.g. If PG_VERSION contains "14\n" this function
    1115             :  * will return 140000)
    1116             :  */
    1117             : static int
    1118          34 : read_pg_version_file(char *directory)
    1119             : {
    1120             :     char        filename[MAXPGPATH];
    1121             :     StringInfoData buf;
    1122             :     int         fd;
    1123             :     int         version;
    1124             :     char       *ep;
    1125             : 
    1126             :     /* Construct pathname. */
    1127          34 :     snprintf(filename, MAXPGPATH, "%s/PG_VERSION", directory);
    1128             : 
    1129             :     /* Open file. */
    1130          34 :     if ((fd = open(filename, O_RDONLY, 0)) < 0)
    1131           0 :         pg_fatal("could not open file \"%s\": %m", filename);
    1132             : 
    1133             :     /* Read into memory. Length limit of 128 should be more than generous. */
    1134          34 :     initStringInfo(&buf);
    1135          34 :     slurp_file(fd, filename, &buf, 128);
    1136             : 
    1137             :     /* Close the file. */
    1138          34 :     if (close(fd) != 0)
    1139           0 :         pg_fatal("could not close \"%s\": %m", filename);
    1140             : 
    1141             :     /* Convert to integer. */
    1142          34 :     errno = 0;
    1143          34 :     version = strtoul(buf.data, &ep, 10);
    1144          34 :     if (errno != 0 || *ep != '\n')
    1145             :     {
    1146             :         /*
    1147             :          * Incremental backup is not relevant to very old server versions that
    1148             :          * used multi-part version number (e.g. 9.6, or 8.4). So if we see
    1149             :          * what looks like the beginning of such a version number, just bail
    1150             :          * out.
    1151             :          */
    1152           0 :         if (version < 10 && *ep == '.')
    1153           0 :             pg_fatal("%s: server version too old\n", filename);
    1154           0 :         pg_fatal("%s: could not parse version number\n", filename);
    1155             :     }
    1156             : 
    1157             :     /* Debugging output. */
    1158          34 :     pg_log_debug("read server version %d from \"%s\"", version, filename);
    1159             : 
    1160             :     /* Release memory and return result. */
    1161          34 :     pfree(buf.data);
    1162          34 :     return version * 10000;
    1163             : }
    1164             : 
    1165             : /*
    1166             :  * Add a directory to the list of output directories to clean up.
    1167             :  */
    1168             : static void
    1169          20 : remember_to_cleanup_directory(char *target_path, bool rmtopdir)
    1170             : {
    1171          20 :     cb_cleanup_dir *dir = pg_malloc(sizeof(cb_cleanup_dir));
    1172             : 
    1173          20 :     dir->target_path = target_path;
    1174          20 :     dir->rmtopdir = rmtopdir;
    1175          20 :     dir->next = cleanup_dir_list;
    1176          20 :     cleanup_dir_list = dir;
    1177          20 : }
    1178             : 
    1179             : /*
    1180             :  * Empty out the list of directories scheduled for cleanup at exit.
    1181             :  *
    1182             :  * We want to remove the output directories only on a failure, so call this
    1183             :  * function when we know that the operation has succeeded.
    1184             :  *
    1185             :  * Since we only expect this to be called when we're about to exit, we could
    1186             :  * just set cleanup_dir_list to NULL and be done with it, but we free the
    1187             :  * memory to be tidy.
    1188             :  */
    1189             : static void
    1190          20 : reset_directory_cleanup_list(void)
    1191             : {
    1192          40 :     while (cleanup_dir_list != NULL)
    1193             :     {
    1194          20 :         cb_cleanup_dir *dir = cleanup_dir_list;
    1195             : 
    1196          20 :         cleanup_dir_list = cleanup_dir_list->next;
    1197          20 :         pfree(dir);
    1198             :     }
    1199          20 : }
    1200             : 
    1201             : /*
    1202             :  * Scan the pg_tblspc directory of the final input backup to get a canonical
    1203             :  * list of what tablespaces are part of the backup.
    1204             :  *
    1205             :  * 'pathname' should be the path to the toplevel backup directory for the
    1206             :  * final backup in the backup chain.
    1207             :  */
    1208             : static cb_tablespace *
    1209          20 : scan_for_existing_tablespaces(char *pathname, cb_options *opt)
    1210             : {
    1211             :     char        pg_tblspc[MAXPGPATH];
    1212             :     DIR        *dir;
    1213             :     struct dirent *de;
    1214          20 :     cb_tablespace *tslist = NULL;
    1215             : 
    1216          20 :     snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pathname);
    1217          20 :     pg_log_debug("scanning \"%s\"", pg_tblspc);
    1218             : 
    1219          20 :     if ((dir = opendir(pg_tblspc)) == NULL)
    1220           0 :         pg_fatal("could not open directory \"%s\": %m", pg_tblspc);
    1221             : 
    1222          60 :     while (errno = 0, (de = readdir(dir)) != NULL)
    1223             :     {
    1224             :         Oid         oid;
    1225             :         char        tblspcdir[MAXPGPATH];
    1226             :         char        link_target[MAXPGPATH];
    1227             :         int         link_length;
    1228             :         cb_tablespace *ts;
    1229             :         cb_tablespace *otherts;
    1230             :         PGFileType  type;
    1231             : 
    1232             :         /* Silently ignore "." and ".." entries. */
    1233          40 :         if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
    1234          40 :             continue;
    1235             : 
    1236             :         /* Construct full pathname. */
    1237           0 :         snprintf(tblspcdir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
    1238             : 
    1239             :         /* Ignore any file name that doesn't look like a proper OID. */
    1240           0 :         if (!parse_oid(de->d_name, &oid))
    1241             :         {
    1242           0 :             pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
    1243             :                          tblspcdir);
    1244           0 :             continue;
    1245             :         }
    1246             : 
    1247             :         /* Only symbolic links and directories are tablespaces. */
    1248           0 :         type = get_dirent_type(tblspcdir, de, false, PG_LOG_ERROR);
    1249           0 :         if (type == PGFILETYPE_ERROR)
    1250           0 :             exit(1);
    1251           0 :         if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
    1252             :         {
    1253           0 :             pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
    1254             :                          tblspcdir);
    1255           0 :             continue;
    1256             :         }
    1257             : 
    1258             :         /* Create a new tablespace object. */
    1259           0 :         ts = pg_malloc0(sizeof(cb_tablespace));
    1260           0 :         ts->oid = oid;
    1261             : 
    1262             :         /*
    1263             :          * If it's a link, it's not an in-place tablespace. Otherwise, it must
    1264             :          * be a directory, and thus an in-place tablespace.
    1265             :          */
    1266           0 :         if (type == PGFILETYPE_LNK)
    1267             :         {
    1268             :             cb_tablespace_mapping *tsmap;
    1269             : 
    1270             :             /* Read the link target. */
    1271           0 :             link_length = readlink(tblspcdir, link_target, sizeof(link_target));
    1272           0 :             if (link_length < 0)
    1273           0 :                 pg_fatal("could not read symbolic link \"%s\": %m",
    1274             :                          tblspcdir);
    1275           0 :             if (link_length >= sizeof(link_target))
    1276           0 :                 pg_fatal("symbolic link \"%s\" is too long", tblspcdir);
    1277           0 :             link_target[link_length] = '\0';
    1278           0 :             if (!is_absolute_path(link_target))
    1279           0 :                 pg_fatal("symbolic link \"%s\" is relative", tblspcdir);
    1280             : 
    1281             :             /* Canonicalize the link target. */
    1282           0 :             canonicalize_path(link_target);
    1283             : 
    1284             :             /*
    1285             :              * Find the corresponding tablespace mapping and copy the relevant
    1286             :              * details into the new tablespace entry.
    1287             :              */
    1288           0 :             for (tsmap = opt->tsmappings; tsmap != NULL; tsmap = tsmap->next)
    1289             :             {
    1290           0 :                 if (strcmp(tsmap->old_dir, link_target) == 0)
    1291             :                 {
    1292           0 :                     strlcpy(ts->old_dir, tsmap->old_dir, MAXPGPATH);
    1293           0 :                     strlcpy(ts->new_dir, tsmap->new_dir, MAXPGPATH);
    1294           0 :                     ts->in_place = false;
    1295           0 :                     break;
    1296             :                 }
    1297             :             }
    1298             : 
    1299             :             /* Every non-in-place tablespace must be mapped. */
    1300           0 :             if (tsmap == NULL)
    1301           0 :                 pg_fatal("tablespace at \"%s\" has no tablespace mapping",
    1302             :                          link_target);
    1303             :         }
    1304             :         else
    1305             :         {
    1306             :             /*
    1307             :              * For an in-place tablespace, there's no separate directory, so
    1308             :              * we just record the paths within the data directories.
    1309             :              */
    1310           0 :             snprintf(ts->old_dir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
    1311           0 :             snprintf(ts->new_dir, MAXPGPATH, "%s/pg_tblspc/%s", opt->output,
    1312           0 :                      de->d_name);
    1313           0 :             ts->in_place = true;
    1314             :         }
    1315             : 
    1316             :         /* Tablespaces should not share a directory. */
    1317           0 :         for (otherts = tslist; otherts != NULL; otherts = otherts->next)
    1318           0 :             if (strcmp(ts->new_dir, otherts->new_dir) == 0)
    1319           0 :                 pg_fatal("tablespaces with OIDs %u and %u both point at \"%s\"",
    1320             :                          otherts->oid, oid, ts->new_dir);
    1321             : 
    1322             :         /* Add this tablespace to the list. */
    1323           0 :         ts->next = tslist;
    1324           0 :         tslist = ts;
    1325             :     }
    1326             : 
    1327          20 :     if (closedir(dir) != 0)
    1328           0 :         pg_fatal("could not close directory \"%s\": %m", pg_tblspc);
    1329             : 
    1330          20 :     return tslist;
    1331             : }
    1332             : 
    1333             : /*
    1334             :  * Read a file into a StringInfo.
    1335             :  *
    1336             :  * fd is used for the actual file I/O, filename for error reporting purposes.
    1337             :  * A file longer than maxlen is a fatal error.
    1338             :  */
    1339             : static void
    1340          96 : slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
    1341             : {
    1342             :     struct stat st;
    1343             :     ssize_t     rb;
    1344             : 
    1345             :     /* Check file size, and complain if it's too large. */
    1346          96 :     if (fstat(fd, &st) != 0)
    1347           0 :         pg_fatal("could not stat \"%s\": %m", filename);
    1348          96 :     if (st.st_size > maxlen)
    1349           0 :         pg_fatal("file \"%s\" is too large", filename);
    1350             : 
    1351             :     /* Make sure we have enough space. */
    1352          96 :     enlargeStringInfo(buf, st.st_size);
    1353             : 
    1354             :     /* Read the data. */
    1355          96 :     rb = read(fd, &buf->data[buf->len], st.st_size);
    1356             : 
    1357             :     /*
    1358             :      * We don't expect any concurrent changes, so we should read exactly the
    1359             :      * expected number of bytes.
    1360             :      */
    1361          96 :     if (rb != st.st_size)
    1362             :     {
    1363           0 :         if (rb < 0)
    1364           0 :             pg_fatal("could not read file \"%s\": %m", filename);
    1365             :         else
    1366           0 :             pg_fatal("could not read file \"%s\": read only %zd of %lld bytes",
    1367             :                      filename, rb, (long long int) st.st_size);
    1368             :     }
    1369             : 
    1370             :     /* Adjust buffer length for new data and restore trailing-\0 invariant */
    1371          96 :     buf->len += rb;
    1372          96 :     buf->data[buf->len] = '\0';
    1373          96 : }

Generated by: LCOV version 1.14