LCOV - code coverage report
Current view: top level - src/bin/pg_basebackup - pg_basebackup.c (source / functions) Hit Total Coverage
Test: PostgreSQL 16beta1 Lines: 680 981 69.3 %
Date: 2023-05-30 23:12:14 Functions: 25 32 78.1 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_basebackup.c - receive a base backup using streaming replication protocol
       4             :  *
       5             :  * Author: Magnus Hagander <magnus@hagander.net>
       6             :  *
       7             :  * Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
       8             :  *
       9             :  * IDENTIFICATION
      10             :  *        src/bin/pg_basebackup/pg_basebackup.c
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres_fe.h"
      15             : 
      16             : #include <unistd.h>
      17             : #include <dirent.h>
      18             : #include <limits.h>
      19             : #include <sys/select.h>
      20             : #include <sys/stat.h>
      21             : #include <sys/wait.h>
      22             : #include <signal.h>
      23             : #include <time.h>
      24             : #ifdef HAVE_LIBZ
      25             : #include <zlib.h>
      26             : #endif
      27             : 
      28             : #include "access/xlog_internal.h"
      29             : #include "backup/basebackup.h"
      30             : #include "bbstreamer.h"
      31             : #include "common/compression.h"
      32             : #include "common/file_perm.h"
      33             : #include "common/file_utils.h"
      34             : #include "common/logging.h"
      35             : #include "fe_utils/option_utils.h"
      36             : #include "fe_utils/recovery_gen.h"
      37             : #include "getopt_long.h"
      38             : #include "receivelog.h"
      39             : #include "streamutil.h"
      40             : 
      41             : #define ERRCODE_DATA_CORRUPTED  "XX001"
      42             : 
      43             : typedef struct TablespaceListCell
      44             : {
      45             :     struct TablespaceListCell *next;
      46             :     char        old_dir[MAXPGPATH];
      47             :     char        new_dir[MAXPGPATH];
      48             : } TablespaceListCell;
      49             : 
      50             : typedef struct TablespaceList
      51             : {
      52             :     TablespaceListCell *head;
      53             :     TablespaceListCell *tail;
      54             : } TablespaceList;
      55             : 
      56             : typedef struct ArchiveStreamState
      57             : {
      58             :     int         tablespacenum;
      59             :     pg_compress_specification *compress;
      60             :     bbstreamer *streamer;
      61             :     bbstreamer *manifest_inject_streamer;
      62             :     PQExpBuffer manifest_buffer;
      63             :     char        manifest_filename[MAXPGPATH];
      64             :     FILE       *manifest_file;
      65             : } ArchiveStreamState;
      66             : 
      67             : typedef struct WriteTarState
      68             : {
      69             :     int         tablespacenum;
      70             :     bbstreamer *streamer;
      71             : } WriteTarState;
      72             : 
      73             : typedef struct WriteManifestState
      74             : {
      75             :     char        filename[MAXPGPATH];
      76             :     FILE       *file;
      77             : } WriteManifestState;
      78             : 
      79             : typedef void (*WriteDataCallback) (size_t nbytes, char *buf,
      80             :                                    void *callback_data);
      81             : 
      82             : /*
      83             :  * pg_xlog has been renamed to pg_wal in version 10.  This version number
      84             :  * should be compared with PQserverVersion().
      85             :  */
      86             : #define MINIMUM_VERSION_FOR_PG_WAL  100000
      87             : 
      88             : /*
      89             :  * Temporary replication slots are supported from version 10.
      90             :  */
      91             : #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
      92             : 
      93             : /*
      94             :  * Backup manifests are supported from version 13.
      95             :  */
      96             : #define MINIMUM_VERSION_FOR_MANIFESTS   130000
      97             : 
      98             : /*
      99             :  * Before v15, tar files received from the server will be improperly
     100             :  * terminated.
     101             :  */
     102             : #define MINIMUM_VERSION_FOR_TERMINATED_TARFILE 150000
     103             : 
     104             : /*
     105             :  * Different ways to include WAL
     106             :  */
     107             : typedef enum
     108             : {
     109             :     NO_WAL,
     110             :     FETCH_WAL,
     111             :     STREAM_WAL
     112             : } IncludeWal;
     113             : 
     114             : /*
     115             :  * Different places to perform compression
     116             :  */
     117             : typedef enum
     118             : {
     119             :     COMPRESS_LOCATION_UNSPECIFIED,
     120             :     COMPRESS_LOCATION_CLIENT,
     121             :     COMPRESS_LOCATION_SERVER
     122             : } CompressionLocation;
     123             : 
     124             : /* Global options */
     125             : static char *basedir = NULL;
     126             : static TablespaceList tablespace_dirs = {NULL, NULL};
     127             : static char *xlog_dir = NULL;
     128             : static char format = '\0';      /* p(lain)/t(ar) */
     129             : static char *label = "pg_basebackup base backup";
     130             : static bool noclean = false;
     131             : static bool checksum_failure = false;
     132             : static bool showprogress = false;
     133             : static bool estimatesize = true;
     134             : static int  verbose = 0;
     135             : static IncludeWal includewal = STREAM_WAL;
     136             : static bool fastcheckpoint = false;
     137             : static bool writerecoveryconf = false;
     138             : static bool do_sync = true;
     139             : static int  standby_message_timeout = 10 * 1000;    /* 10 sec = default */
     140             : static pg_time_t last_progress_report = 0;
     141             : static int32 maxrate = 0;       /* no limit by default */
     142             : static char *replication_slot = NULL;
     143             : static bool temp_replication_slot = true;
     144             : static char *backup_target = NULL;
     145             : static bool create_slot = false;
     146             : static bool no_slot = false;
     147             : static bool verify_checksums = true;
     148             : static bool manifest = true;
     149             : static bool manifest_force_encode = false;
     150             : static char *manifest_checksums = NULL;
     151             : 
     152             : static bool success = false;
     153             : static bool made_new_pgdata = false;
     154             : static bool found_existing_pgdata = false;
     155             : static bool made_new_xlogdir = false;
     156             : static bool found_existing_xlogdir = false;
     157             : static bool made_tablespace_dirs = false;
     158             : static bool found_tablespace_dirs = false;
     159             : 
     160             : /* Progress indicators */
     161             : static uint64 totalsize_kb;
     162             : static uint64 totaldone;
     163             : static int  tablespacecount;
     164             : static char *progress_filename = NULL;
     165             : 
     166             : /* Pipe to communicate with background wal receiver process */
     167             : #ifndef WIN32
     168             : static int  bgpipe[2] = {-1, -1};
     169             : #endif
     170             : 
     171             : /* Handle to child process */
     172             : static pid_t bgchild = -1;
     173             : static bool in_log_streamer = false;
     174             : 
     175             : /* Flag to indicate if child process exited unexpectedly */
     176             : static volatile sig_atomic_t bgchild_exited = false;
     177             : 
     178             : /* End position for xlog streaming, empty string if unknown yet */
     179             : static XLogRecPtr xlogendptr;
     180             : 
     181             : #ifndef WIN32
     182             : static int  has_xlogendptr = 0;
     183             : #else
     184             : static volatile LONG has_xlogendptr = 0;
     185             : #endif
     186             : 
     187             : /* Contents of configuration file to be generated */
     188             : static PQExpBuffer recoveryconfcontents = NULL;
     189             : 
     190             : /* Function headers */
     191             : static void usage(void);
     192             : static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
     193             : static void progress_update_filename(const char *filename);
     194             : static void progress_report(int tablespacenum, bool force, bool finished);
     195             : 
     196             : static bbstreamer *CreateBackupStreamer(char *archive_name, char *spclocation,
     197             :                                         bbstreamer **manifest_inject_streamer_p,
     198             :                                         bool is_recovery_guc_supported,
     199             :                                         bool expect_unterminated_tarfile,
     200             :                                         pg_compress_specification *compress);
     201             : static void ReceiveArchiveStreamChunk(size_t r, char *copybuf,
     202             :                                       void *callback_data);
     203             : static char GetCopyDataByte(size_t r, char *copybuf, size_t *cursor);
     204             : static char *GetCopyDataString(size_t r, char *copybuf, size_t *cursor);
     205             : static uint64 GetCopyDataUInt64(size_t r, char *copybuf, size_t *cursor);
     206             : static void GetCopyDataEnd(size_t r, char *copybuf, size_t cursor);
     207             : static void ReportCopyDataParseError(size_t r, char *copybuf);
     208             : static void ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
     209             :                            bool tablespacenum, pg_compress_specification *compress);
     210             : static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data);
     211             : static void ReceiveBackupManifest(PGconn *conn);
     212             : static void ReceiveBackupManifestChunk(size_t r, char *copybuf,
     213             :                                        void *callback_data);
     214             : static void ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf);
     215             : static void ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf,
     216             :                                                void *callback_data);
     217             : static void BaseBackup(char *compression_algorithm, char *compression_detail,
     218             :                        CompressionLocation compressloc,
     219             :                        pg_compress_specification *client_compress);
     220             : 
     221             : static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
     222             :                                  bool segment_finished);
     223             : 
     224             : static const char *get_tablespace_mapping(const char *dir);
     225             : static void tablespace_list_append(const char *arg);
     226             : 
     227             : 
     228             : static void
     229         520 : cleanup_directories_atexit(void)
     230             : {
     231         520 :     if (success || in_log_streamer)
     232         406 :         return;
     233             : 
     234         114 :     if (!noclean && !checksum_failure)
     235             :     {
     236         106 :         if (made_new_pgdata)
     237             :         {
     238          34 :             pg_log_info("removing data directory \"%s\"", basedir);
     239          34 :             if (!rmtree(basedir, true))
     240           0 :                 pg_log_error("failed to remove data directory");
     241             :         }
     242          72 :         else if (found_existing_pgdata)
     243             :         {
     244           0 :             pg_log_info("removing contents of data directory \"%s\"", basedir);
     245           0 :             if (!rmtree(basedir, false))
     246           0 :                 pg_log_error("failed to remove contents of data directory");
     247             :         }
     248             : 
     249         106 :         if (made_new_xlogdir)
     250             :         {
     251           0 :             pg_log_info("removing WAL directory \"%s\"", xlog_dir);
     252           0 :             if (!rmtree(xlog_dir, true))
     253           0 :                 pg_log_error("failed to remove WAL directory");
     254             :         }
     255         106 :         else if (found_existing_xlogdir)
     256             :         {
     257           0 :             pg_log_info("removing contents of WAL directory \"%s\"", xlog_dir);
     258           0 :             if (!rmtree(xlog_dir, false))
     259           0 :                 pg_log_error("failed to remove contents of WAL directory");
     260             :         }
     261             :     }
     262             :     else
     263             :     {
     264           8 :         if ((made_new_pgdata || found_existing_pgdata) && !checksum_failure)
     265           0 :             pg_log_info("data directory \"%s\" not removed at user's request", basedir);
     266             : 
     267           8 :         if (made_new_xlogdir || found_existing_xlogdir)
     268           0 :             pg_log_info("WAL directory \"%s\" not removed at user's request", xlog_dir);
     269             :     }
     270             : 
     271         114 :     if ((made_tablespace_dirs || found_tablespace_dirs) && !checksum_failure)
     272           0 :         pg_log_info("changes to tablespace directories will not be undone");
     273             : }
     274             : 
     275             : static void
     276         456 : disconnect_atexit(void)
     277             : {
     278         456 :     if (conn != NULL)
     279         238 :         PQfinish(conn);
     280         456 : }
     281             : 
     282             : #ifndef WIN32
     283             : /*
     284             :  * If the bgchild exits prematurely and raises a SIGCHLD signal, we can abort
     285             :  * processing rather than wait until the backup has finished and error out at
     286             :  * that time. On Windows, we use a background thread which can communicate
     287             :  * without the need for a signal handler.
     288             :  */
     289             : static void
     290         192 : sigchld_handler(SIGNAL_ARGS)
     291             : {
     292         192 :     bgchild_exited = true;
     293         192 : }
     294             : 
     295             : /*
     296             :  * On windows, our background thread dies along with the process. But on
     297             :  * Unix, if we have started a subprocess, we want to kill it off so it
     298             :  * doesn't remain running trying to stream data.
     299             :  */
     300             : static void
     301         196 : kill_bgchild_atexit(void)
     302             : {
     303         196 :     if (bgchild > 0 && !bgchild_exited)
     304           8 :         kill(bgchild, SIGTERM);
     305         196 : }
     306             : #endif
     307             : 
     308             : /*
     309             :  * Split argument into old_dir and new_dir and append to tablespace mapping
     310             :  * list.
     311             :  */
     312             : static void
     313          38 : tablespace_list_append(const char *arg)
     314             : {
     315          38 :     TablespaceListCell *cell = (TablespaceListCell *) pg_malloc0(sizeof(TablespaceListCell));
     316             :     char       *dst;
     317             :     char       *dst_ptr;
     318             :     const char *arg_ptr;
     319             : 
     320          38 :     dst_ptr = dst = cell->old_dir;
     321        1284 :     for (arg_ptr = arg; *arg_ptr; arg_ptr++)
     322             :     {
     323        1248 :         if (dst_ptr - dst >= MAXPGPATH)
     324           0 :             pg_fatal("directory name too long");
     325             : 
     326        1248 :         if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
     327             :             ;                   /* skip backslash escaping = */
     328        1244 :         else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
     329             :         {
     330          38 :             if (*cell->new_dir)
     331           2 :                 pg_fatal("multiple \"=\" signs in tablespace mapping");
     332             :             else
     333          36 :                 dst = dst_ptr = cell->new_dir;
     334             :         }
     335             :         else
     336        1206 :             *dst_ptr++ = *arg_ptr;
     337             :     }
     338             : 
     339          36 :     if (!*cell->old_dir || !*cell->new_dir)
     340           6 :         pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
     341             : 
     342             :     /*
     343             :      * All tablespaces are created with absolute directories, so specifying a
     344             :      * non-absolute path here would just never match, possibly confusing
     345             :      * users. Since we don't know whether the remote side is Windows or not,
     346             :      * and it might be different than the local side, permit any path that
     347             :      * could be absolute under either set of rules.
     348             :      *
     349             :      * (There is little practical risk of confusion here, because someone
     350             :      * running entirely on Linux isn't likely to have a relative path that
     351             :      * begins with a backslash or something that looks like a drive
     352             :      * specification. If they do, and they also incorrectly believe that a
     353             :      * relative path is acceptable here, we'll silently fail to warn them of
     354             :      * their mistake, and the -T option will just not get applied, same as if
     355             :      * they'd specified -T for a nonexistent tablespace.)
     356             :      */
     357          30 :     if (!is_nonwindows_absolute_path(cell->old_dir) &&
     358           2 :         !is_windows_absolute_path(cell->old_dir))
     359           2 :         pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
     360             :                  cell->old_dir);
     361             : 
     362          28 :     if (!is_absolute_path(cell->new_dir))
     363           2 :         pg_fatal("new directory is not an absolute path in tablespace mapping: %s",
     364             :                  cell->new_dir);
     365             : 
     366             :     /*
     367             :      * Comparisons done with these values should involve similarly
     368             :      * canonicalized path values.  This is particularly sensitive on Windows
     369             :      * where path values may not necessarily use Unix slashes.
     370             :      */
     371          26 :     canonicalize_path(cell->old_dir);
     372          26 :     canonicalize_path(cell->new_dir);
     373             : 
     374          26 :     if (tablespace_dirs.tail)
     375           0 :         tablespace_dirs.tail->next = cell;
     376             :     else
     377          26 :         tablespace_dirs.head = cell;
     378          26 :     tablespace_dirs.tail = cell;
     379          26 : }
     380             : 
     381             : 
     382             : static void
     383           2 : usage(void)
     384             : {
     385           2 :     printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
     386             :            progname);
     387           2 :     printf(_("Usage:\n"));
     388           2 :     printf(_("  %s [OPTION]...\n"), progname);
     389           2 :     printf(_("\nOptions controlling the output:\n"));
     390           2 :     printf(_("  -D, --pgdata=DIRECTORY receive base backup into directory\n"));
     391           2 :     printf(_("  -F, --format=p|t       output format (plain (default), tar)\n"));
     392           2 :     printf(_("  -r, --max-rate=RATE    maximum transfer rate to transfer data directory\n"
     393             :              "                         (in kB/s, or use suffix \"k\" or \"M\")\n"));
     394           2 :     printf(_("  -R, --write-recovery-conf\n"
     395             :              "                         write configuration for replication\n"));
     396           2 :     printf(_("  -t, --target=TARGET[:DETAIL]\n"
     397             :              "                         backup target (if other than client)\n"));
     398           2 :     printf(_("  -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
     399             :              "                         relocate tablespace in OLDDIR to NEWDIR\n"));
     400           2 :     printf(_("      --waldir=WALDIR    location for the write-ahead log directory\n"));
     401           2 :     printf(_("  -X, --wal-method=none|fetch|stream\n"
     402             :              "                         include required WAL files with specified method\n"));
     403           2 :     printf(_("  -z, --gzip             compress tar output\n"));
     404           2 :     printf(_("  -Z, --compress=[{client|server}-]METHOD[:DETAIL]\n"
     405             :              "                         compress on client or server as specified\n"));
     406           2 :     printf(_("  -Z, --compress=none    do not compress tar output\n"));
     407           2 :     printf(_("\nGeneral options:\n"));
     408           2 :     printf(_("  -c, --checkpoint=fast|spread\n"
     409             :              "                         set fast or spread checkpointing\n"));
     410           2 :     printf(_("  -C, --create-slot      create replication slot\n"));
     411           2 :     printf(_("  -l, --label=LABEL      set backup label\n"));
     412           2 :     printf(_("  -n, --no-clean         do not clean up after errors\n"));
     413           2 :     printf(_("  -N, --no-sync          do not wait for changes to be written safely to disk\n"));
     414           2 :     printf(_("  -P, --progress         show progress information\n"));
     415           2 :     printf(_("  -S, --slot=SLOTNAME    replication slot to use\n"));
     416           2 :     printf(_("  -v, --verbose          output verbose messages\n"));
     417           2 :     printf(_("  -V, --version          output version information, then exit\n"));
     418           2 :     printf(_("      --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
     419             :              "                         use algorithm for manifest checksums\n"));
     420           2 :     printf(_("      --manifest-force-encode\n"
     421             :              "                         hex encode all file names in manifest\n"));
     422           2 :     printf(_("      --no-estimate-size do not estimate backup size in server side\n"));
     423           2 :     printf(_("      --no-manifest      suppress generation of backup manifest\n"));
     424           2 :     printf(_("      --no-slot          prevent creation of temporary replication slot\n"));
     425           2 :     printf(_("      --no-verify-checksums\n"
     426             :              "                         do not verify checksums\n"));
     427           2 :     printf(_("  -?, --help             show this help, then exit\n"));
     428           2 :     printf(_("\nConnection options:\n"));
     429           2 :     printf(_("  -d, --dbname=CONNSTR   connection string\n"));
     430           2 :     printf(_("  -h, --host=HOSTNAME    database server host or socket directory\n"));
     431           2 :     printf(_("  -p, --port=PORT        database server port number\n"));
     432           2 :     printf(_("  -s, --status-interval=INTERVAL\n"
     433             :              "                         time between status packets sent to server (in seconds)\n"));
     434           2 :     printf(_("  -U, --username=NAME    connect as specified database user\n"));
     435           2 :     printf(_("  -w, --no-password      never prompt for password\n"));
     436           2 :     printf(_("  -W, --password         force password prompt (should happen automatically)\n"));
     437           2 :     printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
     438           2 :     printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
     439           2 : }
     440             : 
     441             : 
     442             : /*
     443             :  * Called in the background process every time data is received.
     444             :  * On Unix, we check to see if there is any data on our pipe
     445             :  * (which would mean we have a stop position), and if it is, check if
     446             :  * it is time to stop.
     447             :  * On Windows, we are in a single process, so we can just check if it's
     448             :  * time to stop.
     449             :  */
     450             : static bool
     451        6808 : reached_end_position(XLogRecPtr segendpos, uint32 timeline,
     452             :                      bool segment_finished)
     453             : {
     454        6808 :     if (!has_xlogendptr)
     455             :     {
     456             : #ifndef WIN32
     457             :         fd_set      fds;
     458        6610 :         struct timeval tv = {0};
     459             :         int         r;
     460             : 
     461             :         /*
     462             :          * Don't have the end pointer yet - check our pipe to see if it has
     463             :          * been sent yet.
     464             :          */
     465        6610 :         FD_ZERO(&fds);
     466        6610 :         FD_SET(bgpipe[0], &fds);
     467             : 
     468        6610 :         r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
     469        6610 :         if (r == 1)
     470             :         {
     471         182 :             char        xlogend[64] = {0};
     472             :             uint32      hi,
     473             :                         lo;
     474             : 
     475         182 :             r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
     476         182 :             if (r < 0)
     477           0 :                 pg_fatal("could not read from ready pipe: %m");
     478             : 
     479         182 :             if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
     480           0 :                 pg_fatal("could not parse write-ahead log location \"%s\"",
     481             :                          xlogend);
     482         182 :             xlogendptr = ((uint64) hi) << 32 | lo;
     483         182 :             has_xlogendptr = 1;
     484             : 
     485             :             /*
     486             :              * Fall through to check if we've reached the point further
     487             :              * already.
     488             :              */
     489             :         }
     490             :         else
     491             :         {
     492             :             /*
     493             :              * No data received on the pipe means we don't know the end
     494             :              * position yet - so just say it's not time to stop yet.
     495             :              */
     496        6428 :             return false;
     497             :         }
     498             : #else
     499             : 
     500             :         /*
     501             :          * On win32, has_xlogendptr is set by the main thread, so if it's not
     502             :          * set here, we just go back and wait until it shows up.
     503             :          */
     504             :         return false;
     505             : #endif
     506             :     }
     507             : 
     508             :     /*
     509             :      * At this point we have an end pointer, so compare it to the current
     510             :      * position to figure out if it's time to stop.
     511             :      */
     512         380 :     if (segendpos >= xlogendptr)
     513         364 :         return true;
     514             : 
     515             :     /*
     516             :      * Have end pointer, but haven't reached it yet - so tell the caller to
     517             :      * keep streaming.
     518             :      */
     519          16 :     return false;
     520             : }
     521             : 
     522             : typedef struct
     523             : {
     524             :     PGconn     *bgconn;
     525             :     XLogRecPtr  startptr;
     526             :     char        xlog[MAXPGPATH];    /* directory or tarfile depending on mode */
     527             :     char       *sysidentifier;
     528             :     int         timeline;
     529             :     pg_compress_algorithm wal_compress_algorithm;
     530             :     int         wal_compress_level;
     531             : } logstreamer_param;
     532             : 
     533             : static int
     534         188 : LogStreamerMain(logstreamer_param *param)
     535             : {
     536         188 :     StreamCtl   stream = {0};
     537             : 
     538         188 :     in_log_streamer = true;
     539             : 
     540         188 :     stream.startpos = param->startptr;
     541         188 :     stream.timeline = param->timeline;
     542         188 :     stream.sysidentifier = param->sysidentifier;
     543         188 :     stream.stream_stop = reached_end_position;
     544             : #ifndef WIN32
     545         188 :     stream.stop_socket = bgpipe[0];
     546             : #else
     547             :     stream.stop_socket = PGINVALID_SOCKET;
     548             : #endif
     549         188 :     stream.standby_message_timeout = standby_message_timeout;
     550         188 :     stream.synchronous = false;
     551             :     /* fsync happens at the end of pg_basebackup for all data */
     552         188 :     stream.do_sync = false;
     553         188 :     stream.mark_done = true;
     554         188 :     stream.partial_suffix = NULL;
     555         188 :     stream.replication_slot = replication_slot;
     556         188 :     if (format == 'p')
     557         174 :         stream.walmethod = CreateWalDirectoryMethod(param->xlog,
     558             :                                                     PG_COMPRESSION_NONE, 0,
     559         174 :                                                     stream.do_sync);
     560             :     else
     561          14 :         stream.walmethod = CreateWalTarMethod(param->xlog,
     562             :                                               param->wal_compress_algorithm,
     563             :                                               param->wal_compress_level,
     564          14 :                                               stream.do_sync);
     565             : 
     566         188 :     if (!ReceiveXlogStream(param->bgconn, &stream))
     567             :     {
     568             :         /*
     569             :          * Any errors will already have been reported in the function process,
     570             :          * but we need to tell the parent that we didn't shutdown in a nice
     571             :          * way.
     572             :          */
     573             : #ifdef WIN32
     574             :         /*
     575             :          * In order to signal the main thread of an ungraceful exit we set the
     576             :          * same flag that we use on Unix to signal SIGCHLD.
     577             :          */
     578             :         bgchild_exited = true;
     579             : #endif
     580           6 :         return 1;
     581             :     }
     582             : 
     583         182 :     if (!stream.walmethod->ops->finish(stream.walmethod))
     584             :     {
     585           0 :         pg_log_error("could not finish writing WAL files: %m");
     586             : #ifdef WIN32
     587             :         bgchild_exited = true;
     588             : #endif
     589           0 :         return 1;
     590             :     }
     591             : 
     592         182 :     PQfinish(param->bgconn);
     593             : 
     594         182 :     stream.walmethod->ops->free(stream.walmethod);
     595             : 
     596         182 :     return 0;
     597             : }
     598             : 
     599             : /*
     600             :  * Initiate background process for receiving xlog during the backup.
     601             :  * The background stream will use its own database connection so we can
     602             :  * stream the logfile in parallel with the backups.
     603             :  */
     604             : static void
     605         198 : StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier,
     606             :                  pg_compress_algorithm wal_compress_algorithm,
     607             :                  int wal_compress_level)
     608             : {
     609             :     logstreamer_param *param;
     610             :     uint32      hi,
     611             :                 lo;
     612             :     char        statusdir[MAXPGPATH];
     613             : 
     614         198 :     param = pg_malloc0(sizeof(logstreamer_param));
     615         198 :     param->timeline = timeline;
     616         198 :     param->sysidentifier = sysidentifier;
     617         198 :     param->wal_compress_algorithm = wal_compress_algorithm;
     618         198 :     param->wal_compress_level = wal_compress_level;
     619             : 
     620             :     /* Convert the starting position */
     621         198 :     if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
     622           0 :         pg_fatal("could not parse write-ahead log location \"%s\"",
     623             :                  startpos);
     624         198 :     param->startptr = ((uint64) hi) << 32 | lo;
     625             :     /* Round off to even segment position */
     626         198 :     param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
     627             : 
     628             : #ifndef WIN32
     629             :     /* Create our background pipe */
     630         198 :     if (pipe(bgpipe) < 0)
     631           0 :         pg_fatal("could not create pipe for background process: %m");
     632             : #endif
     633             : 
     634             :     /* Get a second connection */
     635         198 :     param->bgconn = GetConnection();
     636         198 :     if (!param->bgconn)
     637             :         /* Error message already written in GetConnection() */
     638           0 :         exit(1);
     639             : 
     640             :     /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
     641         198 :     snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
     642             :              basedir,
     643         198 :              PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
     644             :              "pg_xlog" : "pg_wal");
     645             : 
     646             :     /* Temporary replication slots are only supported in 10 and newer */
     647         198 :     if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_TEMP_SLOTS)
     648           0 :         temp_replication_slot = false;
     649             : 
     650             :     /*
     651             :      * Create replication slot if requested
     652             :      */
     653         198 :     if (temp_replication_slot && !replication_slot)
     654         184 :         replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
     655         198 :     if (temp_replication_slot || create_slot)
     656             :     {
     657         188 :         if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
     658             :                                    temp_replication_slot, true, true, false, false))
     659           2 :             exit(1);
     660             : 
     661         186 :         if (verbose)
     662             :         {
     663           0 :             if (temp_replication_slot)
     664           0 :                 pg_log_info("created temporary replication slot \"%s\"",
     665             :                             replication_slot);
     666             :             else
     667           0 :                 pg_log_info("created replication slot \"%s\"",
     668             :                             replication_slot);
     669             :         }
     670             :     }
     671             : 
     672         196 :     if (format == 'p')
     673             :     {
     674             :         /*
     675             :          * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
     676             :          * pg_wal or pg_xlog) depending on the target server so we can write
     677             :          * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
     678             :          * tar file may arrive later.
     679             :          */
     680         180 :         snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
     681             :                  basedir,
     682         180 :                  PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
     683             :                  "pg_xlog" : "pg_wal");
     684             : 
     685         180 :         if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
     686           0 :             pg_fatal("could not create directory \"%s\": %m", statusdir);
     687             :     }
     688             : 
     689             :     /*
     690             :      * Start a child process and tell it to start streaming. On Unix, this is
     691             :      * a fork(). On Windows, we create a thread.
     692             :      */
     693             : #ifndef WIN32
     694         196 :     bgchild = fork();
     695         384 :     if (bgchild == 0)
     696             :     {
     697             :         /* in child process */
     698         188 :         exit(LogStreamerMain(param));
     699             :     }
     700         196 :     else if (bgchild < 0)
     701           0 :         pg_fatal("could not create background process: %m");
     702             : 
     703             :     /*
     704             :      * Else we are in the parent process and all is well.
     705             :      */
     706         196 :     atexit(kill_bgchild_atexit);
     707             : #else                           /* WIN32 */
     708             :     bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
     709             :     if (bgchild == 0)
     710             :         pg_fatal("could not create background thread: %m");
     711             : #endif
     712         196 : }
     713             : 
     714             : /*
     715             :  * Verify that the given directory exists and is empty. If it does not
     716             :  * exist, it is created. If it exists but is not empty, an error will
     717             :  * be given and the process ended.
     718             :  */
     719             : static void
     720         296 : verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
     721             : {
     722         296 :     switch (pg_check_dir(dirname))
     723             :     {
     724         272 :         case 0:
     725             : 
     726             :             /*
     727             :              * Does not exist, so create
     728             :              */
     729         272 :             if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
     730           0 :                 pg_fatal("could not create directory \"%s\": %m", dirname);
     731         272 :             if (created)
     732         272 :                 *created = true;
     733         272 :             return;
     734          22 :         case 1:
     735             : 
     736             :             /*
     737             :              * Exists, empty
     738             :              */
     739          22 :             if (found)
     740          22 :                 *found = true;
     741          22 :             return;
     742           2 :         case 2:
     743             :         case 3:
     744             :         case 4:
     745             : 
     746             :             /*
     747             :              * Exists, not empty
     748             :              */
     749           2 :             pg_fatal("directory \"%s\" exists but is not empty", dirname);
     750           0 :         case -1:
     751             : 
     752             :             /*
     753             :              * Access problem
     754             :              */
     755           0 :             pg_fatal("could not access directory \"%s\": %m", dirname);
     756             :     }
     757             : }
     758             : 
     759             : /*
     760             :  * Callback to update our notion of the current filename.
     761             :  *
     762             :  * No other code should modify progress_filename!
     763             :  */
     764             : static void
     765      194330 : progress_update_filename(const char *filename)
     766             : {
     767             :     /* We needn't maintain this variable if not doing verbose reports. */
     768      194330 :     if (showprogress && verbose)
     769             :     {
     770           0 :         free(progress_filename);
     771           0 :         if (filename)
     772           0 :             progress_filename = pg_strdup(filename);
     773             :         else
     774           0 :             progress_filename = NULL;
     775             :     }
     776      194330 : }
     777             : 
     778             : /*
     779             :  * Print a progress report based on the global variables. If verbose output
     780             :  * is enabled, also print the current file name.
     781             :  *
     782             :  * Progress report is written at maximum once per second, unless the force
     783             :  * parameter is set to true.
     784             :  *
     785             :  * If finished is set to true, this is the last progress report. The cursor
     786             :  * is moved to the next line.
     787             :  */
     788             : static void
     789         334 : progress_report(int tablespacenum, bool force, bool finished)
     790             : {
     791             :     int         percent;
     792             :     char        totaldone_str[32];
     793             :     char        totalsize_str[32];
     794             :     pg_time_t   now;
     795             : 
     796         334 :     if (!showprogress)
     797         334 :         return;
     798             : 
     799           0 :     now = time(NULL);
     800           0 :     if (now == last_progress_report && !force && !finished)
     801           0 :         return;                 /* Max once per second */
     802             : 
     803           0 :     last_progress_report = now;
     804           0 :     percent = totalsize_kb ? (int) ((totaldone / 1024) * 100 / totalsize_kb) : 0;
     805             : 
     806             :     /*
     807             :      * Avoid overflowing past 100% or the full size. This may make the total
     808             :      * size number change as we approach the end of the backup (the estimate
     809             :      * will always be wrong if WAL is included), but that's better than having
     810             :      * the done column be bigger than the total.
     811             :      */
     812           0 :     if (percent > 100)
     813           0 :         percent = 100;
     814           0 :     if (totaldone / 1024 > totalsize_kb)
     815           0 :         totalsize_kb = totaldone / 1024;
     816             : 
     817           0 :     snprintf(totaldone_str, sizeof(totaldone_str), UINT64_FORMAT,
     818             :              totaldone / 1024);
     819           0 :     snprintf(totalsize_str, sizeof(totalsize_str), UINT64_FORMAT, totalsize_kb);
     820             : 
     821             : #define VERBOSE_FILENAME_LENGTH 35
     822           0 :     if (verbose)
     823             :     {
     824           0 :         if (!progress_filename)
     825             : 
     826             :             /*
     827             :              * No filename given, so clear the status line (used for last
     828             :              * call)
     829             :              */
     830           0 :             fprintf(stderr,
     831           0 :                     ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
     832             :                              "%*s/%s kB (100%%), %d/%d tablespaces %*s",
     833             :                              tablespacecount),
     834           0 :                     (int) strlen(totalsize_str),
     835             :                     totaldone_str, totalsize_str,
     836             :                     tablespacenum, tablespacecount,
     837             :                     VERBOSE_FILENAME_LENGTH + 5, "");
     838             :         else
     839             :         {
     840           0 :             bool        truncate = (strlen(progress_filename) > VERBOSE_FILENAME_LENGTH);
     841             : 
     842           0 :             fprintf(stderr,
     843           0 :                     ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
     844             :                              "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
     845             :                              tablespacecount),
     846           0 :                     (int) strlen(totalsize_str),
     847             :                     totaldone_str, totalsize_str, percent,
     848             :                     tablespacenum, tablespacecount,
     849             :             /* Prefix with "..." if we do leading truncation */
     850             :                     truncate ? "..." : "",
     851             :                     truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
     852             :                     truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
     853             :             /* Truncate filename at beginning if it's too long */
     854           0 :                     truncate ? progress_filename + strlen(progress_filename) - VERBOSE_FILENAME_LENGTH + 3 : progress_filename);
     855             :         }
     856             :     }
     857             :     else
     858           0 :         fprintf(stderr,
     859           0 :                 ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
     860             :                          "%*s/%s kB (%d%%), %d/%d tablespaces",
     861             :                          tablespacecount),
     862           0 :                 (int) strlen(totalsize_str),
     863             :                 totaldone_str, totalsize_str, percent,
     864             :                 tablespacenum, tablespacecount);
     865             : 
     866             :     /*
     867             :      * Stay on the same line if reporting to a terminal and we're not done
     868             :      * yet.
     869             :      */
     870           0 :     fputc((!finished && isatty(fileno(stderr))) ? '\r' : '\n', stderr);
     871             : }
     872             : 
     873             : static int32
     874           2 : parse_max_rate(char *src)
     875             : {
     876             :     double      result;
     877             :     char       *after_num;
     878           2 :     char       *suffix = NULL;
     879             : 
     880           2 :     errno = 0;
     881           2 :     result = strtod(src, &after_num);
     882           2 :     if (src == after_num)
     883           0 :         pg_fatal("transfer rate \"%s\" is not a valid value", src);
     884           2 :     if (errno != 0)
     885           0 :         pg_fatal("invalid transfer rate \"%s\": %m", src);
     886             : 
     887           2 :     if (result <= 0)
     888             :     {
     889             :         /*
     890             :          * Reject obviously wrong values here.
     891             :          */
     892           0 :         pg_fatal("transfer rate must be greater than zero");
     893             :     }
     894             : 
     895             :     /*
     896             :      * Evaluate suffix, after skipping over possible whitespace. Lack of
     897             :      * suffix means kilobytes.
     898             :      */
     899           2 :     while (*after_num != '\0' && isspace((unsigned char) *after_num))
     900           0 :         after_num++;
     901             : 
     902           2 :     if (*after_num != '\0')
     903             :     {
     904           0 :         suffix = after_num;
     905           0 :         if (*after_num == 'k')
     906             :         {
     907             :             /* kilobyte is the expected unit. */
     908           0 :             after_num++;
     909             :         }
     910           0 :         else if (*after_num == 'M')
     911             :         {
     912           0 :             after_num++;
     913           0 :             result *= 1024.0;
     914             :         }
     915             :     }
     916             : 
     917             :     /* The rest can only consist of white space. */
     918           2 :     while (*after_num != '\0' && isspace((unsigned char) *after_num))
     919           0 :         after_num++;
     920             : 
     921           2 :     if (*after_num != '\0')
     922           0 :         pg_fatal("invalid --max-rate unit: \"%s\"", suffix);
     923             : 
     924             :     /* Valid integer? */
     925           2 :     if ((uint64) result != (uint64) ((uint32) result))
     926           0 :         pg_fatal("transfer rate \"%s\" exceeds integer range", src);
     927             : 
     928             :     /*
     929             :      * The range is checked on the server side too, but avoid the server
     930             :      * connection if a nonsensical value was passed.
     931             :      */
     932           2 :     if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
     933           0 :         pg_fatal("transfer rate \"%s\" is out of range", src);
     934             : 
     935           2 :     return (int32) result;
     936             : }
     937             : 
     938             : /*
     939             :  * Basic parsing of a value specified for -Z/--compress.
     940             :  *
     941             :  * We're not concerned here with understanding exactly what behavior the
     942             :  * user wants, but we do need to know whether the user is requesting client
     943             :  * or server side compression or leaving it unspecified, and we need to
     944             :  * separate the name of the compression algorithm from the detail string.
     945             :  *
     946             :  * For instance, if the user writes --compress client-lz4:6, we want to
     947             :  * separate that into (a) client-side compression, (b) algorithm "lz4",
     948             :  * and (c) detail "6". Note, however, that all the client/server prefix is
     949             :  * optional, and so is the detail. The algorithm name is required, unless
     950             :  * the whole string is an integer, in which case we assume "gzip" as the
     951             :  * algorithm and use the integer as the detail.
     952             :  *
     953             :  * We're not concerned with validation at this stage, so if the user writes
     954             :  * --compress client-turkey:sandwich, the requested algorithm is "turkey"
     955             :  * and the detail string is "sandwich". We'll sort out whether that's legal
     956             :  * at a later stage.
     957             :  */
     958             : static void
     959          56 : backup_parse_compress_options(char *option, char **algorithm, char **detail,
     960             :                               CompressionLocation *locationres)
     961             : {
     962             :     /*
     963             :      * Strip off any "client-" or "server-" prefix, calculating the location.
     964             :      */
     965          56 :     if (strncmp(option, "server-", 7) == 0)
     966             :     {
     967          26 :         *locationres = COMPRESS_LOCATION_SERVER;
     968          26 :         option += 7;
     969             :     }
     970          30 :     else if (strncmp(option, "client-", 7) == 0)
     971             :     {
     972           2 :         *locationres = COMPRESS_LOCATION_CLIENT;
     973           2 :         option += 7;
     974             :     }
     975             :     else
     976          28 :         *locationres = COMPRESS_LOCATION_UNSPECIFIED;
     977             : 
     978             :     /* fallback to the common parsing for the algorithm and detail */
     979          56 :     parse_compress_options(option, algorithm, detail);
     980          56 : }
     981             : 
     982             : /*
     983             :  * Read a stream of COPY data and invoke the provided callback for each
     984             :  * chunk.
     985             :  */
     986             : static void
     987         232 : ReceiveCopyData(PGconn *conn, WriteDataCallback callback,
     988             :                 void *callback_data)
     989             : {
     990             :     PGresult   *res;
     991             : 
     992             :     /* Get the COPY data stream. */
     993         232 :     res = PQgetResult(conn);
     994         232 :     if (PQresultStatus(res) != PGRES_COPY_OUT)
     995           0 :         pg_fatal("could not get COPY data stream: %s",
     996             :                  PQerrorMessage(conn));
     997         232 :     PQclear(res);
     998             : 
     999             :     /* Loop over chunks until done. */
    1000             :     while (1)
    1001      468310 :     {
    1002             :         int         r;
    1003             :         char       *copybuf;
    1004             : 
    1005      468542 :         r = PQgetCopyData(conn, &copybuf, 0);
    1006      468542 :         if (r == -1)
    1007             :         {
    1008             :             /* End of chunk. */
    1009         226 :             break;
    1010             :         }
    1011      468316 :         else if (r == -2)
    1012           0 :             pg_fatal("could not read COPY data: %s",
    1013             :                      PQerrorMessage(conn));
    1014             : 
    1015      468316 :         if (bgchild_exited)
    1016           6 :             pg_fatal("background process terminated unexpectedly");
    1017             : 
    1018      468310 :         (*callback) (r, copybuf, callback_data);
    1019             : 
    1020      468310 :         PQfreemem(copybuf);
    1021             :     }
    1022         226 : }
    1023             : 
    1024             : /*
    1025             :  * Figure out what to do with an archive received from the server based on
    1026             :  * the options selected by the user.  We may just write the results directly
    1027             :  * to a file, or we might compress first, or we might extract the tar file
    1028             :  * and write each member separately. This function doesn't do any of that
    1029             :  * directly, but it works out what kind of bbstreamer we need to create so
    1030             :  * that the right stuff happens when, down the road, we actually receive
    1031             :  * the data.
    1032             :  */
    1033             : static bbstreamer *
    1034         272 : CreateBackupStreamer(char *archive_name, char *spclocation,
    1035             :                      bbstreamer **manifest_inject_streamer_p,
    1036             :                      bool is_recovery_guc_supported,
    1037             :                      bool expect_unterminated_tarfile,
    1038             :                      pg_compress_specification *compress)
    1039             : {
    1040         272 :     bbstreamer *streamer = NULL;
    1041         272 :     bbstreamer *manifest_inject_streamer = NULL;
    1042             :     bool        inject_manifest;
    1043             :     bool        is_tar,
    1044             :                 is_tar_gz,
    1045             :                 is_tar_lz4,
    1046             :                 is_tar_zstd,
    1047             :                 is_compressed_tar;
    1048             :     bool        must_parse_archive;
    1049         272 :     int         archive_name_len = strlen(archive_name);
    1050             : 
    1051             :     /*
    1052             :      * Normally, we emit the backup manifest as a separate file, but when
    1053             :      * we're writing a tarfile to stdout, we don't have that option, so
    1054             :      * include it in the one tarfile we've got.
    1055             :      */
    1056         272 :     inject_manifest = (format == 't' && strcmp(basedir, "-") == 0 && manifest);
    1057             : 
    1058             :     /* Is this a tar archive? */
    1059         544 :     is_tar = (archive_name_len > 4 &&
    1060         272 :               strcmp(archive_name + archive_name_len - 4, ".tar") == 0);
    1061             : 
    1062             :     /* Is this a .tar.gz archive? */
    1063         544 :     is_tar_gz = (archive_name_len > 7 &&
    1064         272 :                  strcmp(archive_name + archive_name_len - 7, ".tar.gz") == 0);
    1065             : 
    1066             :     /* Is this a .tar.lz4 archive? */
    1067         330 :     is_tar_lz4 = (archive_name_len > 8 &&
    1068          58 :                   strcmp(archive_name + archive_name_len - 8, ".tar.lz4") == 0);
    1069             : 
    1070             :     /* Is this a .tar.zst archive? */
    1071         330 :     is_tar_zstd = (archive_name_len > 8 &&
    1072          58 :                    strcmp(archive_name + archive_name_len - 8, ".tar.zst") == 0);
    1073             : 
    1074             :     /* Is this any kind of compressed tar? */
    1075         272 :     is_compressed_tar = is_tar_gz || is_tar_lz4 || is_tar_zstd;
    1076             : 
    1077             :     /*
    1078             :      * Injecting the manifest into a compressed tar file would be possible if
    1079             :      * we decompressed it, parsed the tarfile, generated a new tarfile, and
    1080             :      * recompressed it, but compressing and decompressing multiple times just
    1081             :      * to inject the manifest seems inefficient enough that it's probably not
    1082             :      * what the user wants. So, instead, reject the request and tell the user
    1083             :      * to specify something more reasonable.
    1084             :      */
    1085         272 :     if (inject_manifest && is_compressed_tar)
    1086             :     {
    1087           0 :         pg_log_error("cannot inject manifest into a compressed tar file");
    1088           0 :         pg_log_error_hint("Use client-side compression, send the output to a directory rather than standard output, or use %s.",
    1089             :                           "--no-manifest");
    1090           0 :         exit(1);
    1091             :     }
    1092             : 
    1093             :     /*
    1094             :      * We have to parse the archive if (1) we're suppose to extract it, or if
    1095             :      * (2) we need to inject backup_manifest or recovery configuration into
    1096             :      * it. However, we only know how to parse tar archives.
    1097             :      */
    1098         294 :     must_parse_archive = (format == 'p' || inject_manifest ||
    1099          22 :                           (spclocation == NULL && writerecoveryconf));
    1100             : 
    1101             :     /* At present, we only know how to parse tar archives. */
    1102         272 :     if (must_parse_archive && !is_tar && !is_compressed_tar)
    1103             :     {
    1104           0 :         pg_log_error("cannot parse archive \"%s\"", archive_name);
    1105           0 :         pg_log_error_detail("Only tar archives can be parsed.");
    1106           0 :         if (format == 'p')
    1107           0 :             pg_log_error_detail("Plain format requires pg_basebackup to parse the archive.");
    1108           0 :         if (inject_manifest)
    1109           0 :             pg_log_error_detail("Using - as the output directory requires pg_basebackup to parse the archive.");
    1110           0 :         if (writerecoveryconf)
    1111           0 :             pg_log_error_detail("The -R option requires pg_basebackup to parse the archive.");
    1112           0 :         exit(1);
    1113             :     }
    1114             : 
    1115         272 :     if (format == 'p')
    1116             :     {
    1117             :         const char *directory;
    1118             : 
    1119             :         /*
    1120             :          * In plain format, we must extract the archive. The data for the main
    1121             :          * tablespace will be written to the base directory, and the data for
    1122             :          * other tablespaces will be written to the directory where they're
    1123             :          * located on the server, after applying any user-specified tablespace
    1124             :          * mappings.
    1125             :          *
    1126             :          * In the case of an in-place tablespace, spclocation will be a
    1127             :          * relative path. We just convert it to an absolute path by prepending
    1128             :          * basedir.
    1129             :          */
    1130         244 :         if (spclocation == NULL)
    1131         196 :             directory = basedir;
    1132          48 :         else if (!is_absolute_path(spclocation))
    1133          22 :             directory = psprintf("%s/%s", basedir, spclocation);
    1134             :         else
    1135          26 :             directory = get_tablespace_mapping(spclocation);
    1136         244 :         streamer = bbstreamer_extractor_new(directory,
    1137             :                                             get_tablespace_mapping,
    1138             :                                             progress_update_filename);
    1139             :     }
    1140             :     else
    1141             :     {
    1142             :         FILE       *archive_file;
    1143             :         char        archive_filename[MAXPGPATH];
    1144             : 
    1145             :         /*
    1146             :          * In tar format, we just write the archive without extracting it.
    1147             :          * Normally, we write it to the archive name provided by the caller,
    1148             :          * but when the base directory is "-" that means we need to write to
    1149             :          * standard output.
    1150             :          */
    1151          28 :         if (strcmp(basedir, "-") == 0)
    1152             :         {
    1153           0 :             snprintf(archive_filename, sizeof(archive_filename), "-");
    1154           0 :             archive_file = stdout;
    1155             :         }
    1156             :         else
    1157             :         {
    1158          28 :             snprintf(archive_filename, sizeof(archive_filename),
    1159             :                      "%s/%s", basedir, archive_name);
    1160          28 :             archive_file = NULL;
    1161             :         }
    1162             : 
    1163          28 :         if (compress->algorithm == PG_COMPRESSION_NONE)
    1164          20 :             streamer = bbstreamer_plain_writer_new(archive_filename,
    1165             :                                                    archive_file);
    1166           8 :         else if (compress->algorithm == PG_COMPRESSION_GZIP)
    1167             :         {
    1168           8 :             strlcat(archive_filename, ".gz", sizeof(archive_filename));
    1169           8 :             streamer = bbstreamer_gzip_writer_new(archive_filename,
    1170             :                                                   archive_file, compress);
    1171             :         }
    1172           0 :         else if (compress->algorithm == PG_COMPRESSION_LZ4)
    1173             :         {
    1174           0 :             strlcat(archive_filename, ".lz4", sizeof(archive_filename));
    1175           0 :             streamer = bbstreamer_plain_writer_new(archive_filename,
    1176             :                                                    archive_file);
    1177           0 :             streamer = bbstreamer_lz4_compressor_new(streamer, compress);
    1178             :         }
    1179           0 :         else if (compress->algorithm == PG_COMPRESSION_ZSTD)
    1180             :         {
    1181           0 :             strlcat(archive_filename, ".zst", sizeof(archive_filename));
    1182           0 :             streamer = bbstreamer_plain_writer_new(archive_filename,
    1183             :                                                    archive_file);
    1184           0 :             streamer = bbstreamer_zstd_compressor_new(streamer, compress);
    1185             :         }
    1186             :         else
    1187             :         {
    1188             :             Assert(false);      /* not reachable */
    1189             :         }
    1190             : 
    1191             :         /*
    1192             :          * If we need to parse the archive for whatever reason, then we'll
    1193             :          * also need to re-archive, because, if the output format is tar, the
    1194             :          * only point of parsing the archive is to be able to inject stuff
    1195             :          * into it.
    1196             :          */
    1197          28 :         if (must_parse_archive)
    1198           0 :             streamer = bbstreamer_tar_archiver_new(streamer);
    1199          28 :         progress_update_filename(archive_filename);
    1200             :     }
    1201             : 
    1202             :     /*
    1203             :      * If we're supposed to inject the backup manifest into the results, it
    1204             :      * should be done here, so that the file content can be injected directly,
    1205             :      * without worrying about the details of the tar format.
    1206             :      */
    1207         272 :     if (inject_manifest)
    1208           0 :         manifest_inject_streamer = streamer;
    1209             : 
    1210             :     /*
    1211             :      * If this is the main tablespace and we're supposed to write recovery
    1212             :      * information, arrange to do that.
    1213             :      */
    1214         272 :     if (spclocation == NULL && writerecoveryconf)
    1215             :     {
    1216             :         Assert(must_parse_archive);
    1217           4 :         streamer = bbstreamer_recovery_injector_new(streamer,
    1218             :                                                     is_recovery_guc_supported,
    1219             :                                                     recoveryconfcontents);
    1220             :     }
    1221             : 
    1222             :     /*
    1223             :      * If we're doing anything that involves understanding the contents of the
    1224             :      * archive, we'll need to parse it. If not, we can skip parsing it, but
    1225             :      * old versions of the server send improperly terminated tarfiles, so if
    1226             :      * we're talking to such a server we'll need to add the terminator here.
    1227             :      */
    1228         272 :     if (must_parse_archive)
    1229         244 :         streamer = bbstreamer_tar_parser_new(streamer);
    1230          28 :     else if (expect_unterminated_tarfile)
    1231           0 :         streamer = bbstreamer_tar_terminator_new(streamer);
    1232             : 
    1233             :     /*
    1234             :      * If the user has requested a server compressed archive along with
    1235             :      * archive extraction at client then we need to decompress it.
    1236             :      */
    1237         272 :     if (format == 'p')
    1238             :     {
    1239         244 :         if (is_tar_gz)
    1240           2 :             streamer = bbstreamer_gzip_decompressor_new(streamer);
    1241         242 :         else if (is_tar_lz4)
    1242           2 :             streamer = bbstreamer_lz4_decompressor_new(streamer);
    1243         240 :         else if (is_tar_zstd)
    1244           0 :             streamer = bbstreamer_zstd_decompressor_new(streamer);
    1245             :     }
    1246             : 
    1247             :     /* Return the results. */
    1248         272 :     *manifest_inject_streamer_p = manifest_inject_streamer;
    1249         272 :     return streamer;
    1250             : }
    1251             : 
    1252             : /*
    1253             :  * Receive all of the archives the server wants to send - and the backup
    1254             :  * manifest if present - as a single COPY stream.
    1255             :  */
    1256             : static void
    1257         232 : ReceiveArchiveStream(PGconn *conn, pg_compress_specification *compress)
    1258             : {
    1259             :     ArchiveStreamState state;
    1260             : 
    1261             :     /* Set up initial state. */
    1262         232 :     memset(&state, 0, sizeof(state));
    1263         232 :     state.tablespacenum = -1;
    1264         232 :     state.compress = compress;
    1265             : 
    1266             :     /* All the real work happens in ReceiveArchiveStreamChunk. */
    1267         232 :     ReceiveCopyData(conn, ReceiveArchiveStreamChunk, &state);
    1268             : 
    1269             :     /* If we wrote the backup manifest to a file, close the file. */
    1270         226 :     if (state.manifest_file !=NULL)
    1271             :     {
    1272         208 :         fclose(state.manifest_file);
    1273         208 :         state.manifest_file = NULL;
    1274             :     }
    1275             : 
    1276             :     /*
    1277             :      * If we buffered the backup manifest in order to inject it into the
    1278             :      * output tarfile, do that now.
    1279             :      */
    1280         226 :     if (state.manifest_inject_streamer != NULL &&
    1281           0 :         state.manifest_buffer != NULL)
    1282             :     {
    1283           0 :         bbstreamer_inject_file(state.manifest_inject_streamer,
    1284             :                                "backup_manifest",
    1285           0 :                                state.manifest_buffer->data,
    1286           0 :                                state.manifest_buffer->len);
    1287           0 :         destroyPQExpBuffer(state.manifest_buffer);
    1288           0 :         state.manifest_buffer = NULL;
    1289             :     }
    1290             : 
    1291             :     /* If there's still an archive in progress, end processing. */
    1292         226 :     if (state.streamer != NULL)
    1293             :     {
    1294         212 :         bbstreamer_finalize(state.streamer);
    1295         212 :         bbstreamer_free(state.streamer);
    1296         212 :         state.streamer = NULL;
    1297             :     }
    1298         226 : }
    1299             : 
    1300             : /*
    1301             :  * Receive one chunk of data sent by the server as part of a single COPY
    1302             :  * stream that includes all archives and the manifest.
    1303             :  */
    1304             : static void
    1305      468310 : ReceiveArchiveStreamChunk(size_t r, char *copybuf, void *callback_data)
    1306             : {
    1307      468310 :     ArchiveStreamState *state = callback_data;
    1308      468310 :     size_t      cursor = 0;
    1309             : 
    1310             :     /* Each CopyData message begins with a type byte. */
    1311      468310 :     switch (GetCopyDataByte(r, copybuf, &cursor))
    1312             :     {
    1313         286 :         case 'n':
    1314             :             {
    1315             :                 /* New archive. */
    1316             :                 char       *archive_name;
    1317             :                 char       *spclocation;
    1318             : 
    1319             :                 /*
    1320             :                  * We force a progress report at the end of each tablespace. A
    1321             :                  * new tablespace starts when the previous one ends, except in
    1322             :                  * the case of the very first one.
    1323             :                  */
    1324         286 :                 if (++state->tablespacenum > 0)
    1325          54 :                     progress_report(state->tablespacenum, true, false);
    1326             : 
    1327             :                 /* Sanity check. */
    1328         286 :                 if (state->manifest_buffer != NULL ||
    1329         286 :                     state->manifest_file !=NULL)
    1330           0 :                     pg_fatal("archives must precede manifest");
    1331             : 
    1332             :                 /* Parse the rest of the CopyData message. */
    1333         286 :                 archive_name = GetCopyDataString(r, copybuf, &cursor);
    1334         286 :                 spclocation = GetCopyDataString(r, copybuf, &cursor);
    1335         286 :                 GetCopyDataEnd(r, copybuf, cursor);
    1336             : 
    1337             :                 /*
    1338             :                  * Basic sanity checks on the archive name: it shouldn't be
    1339             :                  * empty, it shouldn't start with a dot, and it shouldn't
    1340             :                  * contain a path separator.
    1341             :                  */
    1342         286 :                 if (archive_name[0] == '\0' || archive_name[0] == '.' ||
    1343         286 :                     strchr(archive_name, '/') != NULL ||
    1344         286 :                     strchr(archive_name, '\\') != NULL)
    1345           0 :                     pg_fatal("invalid archive name: \"%s\"",
    1346             :                              archive_name);
    1347             : 
    1348             :                 /*
    1349             :                  * An empty spclocation is treated as NULL. We expect this
    1350             :                  * case to occur for the data directory itself, but not for
    1351             :                  * any archives that correspond to tablespaces.
    1352             :                  */
    1353         286 :                 if (spclocation[0] == '\0')
    1354         232 :                     spclocation = NULL;
    1355             : 
    1356             :                 /* End processing of any prior archive. */
    1357         286 :                 if (state->streamer != NULL)
    1358             :                 {
    1359          54 :                     bbstreamer_finalize(state->streamer);
    1360          54 :                     bbstreamer_free(state->streamer);
    1361          54 :                     state->streamer = NULL;
    1362             :                 }
    1363             : 
    1364             :                 /*
    1365             :                  * Create an appropriate backup streamer, unless a backup
    1366             :                  * target was specified. In that case, it's up to the server
    1367             :                  * to put the backup wherever it needs to go.
    1368             :                  */
    1369         286 :                 if (backup_target == NULL)
    1370             :                 {
    1371             :                     /*
    1372             :                      * We know that recovery GUCs are supported, because this
    1373             :                      * protocol can only be used on v15+.
    1374             :                      */
    1375         272 :                     state->streamer =
    1376         272 :                         CreateBackupStreamer(archive_name,
    1377             :                                              spclocation,
    1378             :                                              &state->manifest_inject_streamer,
    1379             :                                              true, false,
    1380             :                                              state->compress);
    1381             :                 }
    1382         286 :                 break;
    1383             :             }
    1384             : 
    1385      467522 :         case 'd':
    1386             :             {
    1387             :                 /* Archive or manifest data. */
    1388      467522 :                 if (state->manifest_buffer != NULL)
    1389             :                 {
    1390             :                     /* Manifest data, buffer in memory. */
    1391           0 :                     appendPQExpBuffer(state->manifest_buffer, copybuf + 1,
    1392             :                                       r - 1);
    1393             :                 }
    1394      467522 :                 else if (state->manifest_file !=NULL)
    1395             :                 {
    1396             :                     /* Manifest data, write to disk. */
    1397        1066 :                     if (fwrite(copybuf + 1, r - 1, 1,
    1398             :                                state->manifest_file) != 1)
    1399             :                     {
    1400             :                         /*
    1401             :                          * If fwrite() didn't set errno, assume that the
    1402             :                          * problem is that we're out of disk space.
    1403             :                          */
    1404           0 :                         if (errno == 0)
    1405           0 :                             errno = ENOSPC;
    1406           0 :                         pg_fatal("could not write to file \"%s\": %m",
    1407             :                                  state->manifest_filename);
    1408             :                     }
    1409             :                 }
    1410      466456 :                 else if (state->streamer != NULL)
    1411             :                 {
    1412             :                     /* Archive data. */
    1413      466456 :                     bbstreamer_content(state->streamer, NULL, copybuf + 1,
    1414      466456 :                                        r - 1, BBSTREAMER_UNKNOWN);
    1415             :                 }
    1416             :                 else
    1417           0 :                     pg_fatal("unexpected payload data");
    1418      467522 :                 break;
    1419             :             }
    1420             : 
    1421         280 :         case 'p':
    1422             :             {
    1423             :                 /*
    1424             :                  * Progress report.
    1425             :                  *
    1426             :                  * The remainder of the message is expected to be an 8-byte
    1427             :                  * count of bytes completed.
    1428             :                  */
    1429         280 :                 totaldone = GetCopyDataUInt64(r, copybuf, &cursor);
    1430         280 :                 GetCopyDataEnd(r, copybuf, cursor);
    1431             : 
    1432             :                 /*
    1433             :                  * The server shouldn't send progress report messages too
    1434             :                  * often, so we force an update each time we receive one.
    1435             :                  */
    1436         280 :                 progress_report(state->tablespacenum, true, false);
    1437         280 :                 break;
    1438             :             }
    1439             : 
    1440         222 :         case 'm':
    1441             :             {
    1442             :                 /*
    1443             :                  * Manifest data will be sent next. This message is not
    1444             :                  * expected to have any further payload data.
    1445             :                  */
    1446         222 :                 GetCopyDataEnd(r, copybuf, cursor);
    1447             : 
    1448             :                 /*
    1449             :                  * If a backup target was specified, figuring out where to put
    1450             :                  * the manifest is the server's problem. Otherwise, we need to
    1451             :                  * deal with it.
    1452             :                  */
    1453         222 :                 if (backup_target == NULL)
    1454             :                 {
    1455             :                     /*
    1456             :                      * If we're supposed inject the manifest into the archive,
    1457             :                      * we prepare to buffer it in memory; otherwise, we
    1458             :                      * prepare to write it to a temporary file.
    1459             :                      */
    1460         208 :                     if (state->manifest_inject_streamer != NULL)
    1461           0 :                         state->manifest_buffer = createPQExpBuffer();
    1462             :                     else
    1463             :                     {
    1464         208 :                         snprintf(state->manifest_filename,
    1465             :                                  sizeof(state->manifest_filename),
    1466             :                                  "%s/backup_manifest.tmp", basedir);
    1467         208 :                         state->manifest_file =
    1468         208 :                             fopen(state->manifest_filename, "wb");
    1469         208 :                         if (state->manifest_file == NULL)
    1470           0 :                             pg_fatal("could not create file \"%s\": %m",
    1471             :                                      state->manifest_filename);
    1472             :                     }
    1473             :                 }
    1474         222 :                 break;
    1475             :             }
    1476             : 
    1477           0 :         default:
    1478           0 :             ReportCopyDataParseError(r, copybuf);
    1479           0 :             break;
    1480             :     }
    1481      468310 : }
    1482             : 
    1483             : /*
    1484             :  * Get a single byte from a CopyData message.
    1485             :  *
    1486             :  * Bail out if none remain.
    1487             :  */
    1488             : static char
    1489      468310 : GetCopyDataByte(size_t r, char *copybuf, size_t *cursor)
    1490             : {
    1491      468310 :     if (*cursor >= r)
    1492           0 :         ReportCopyDataParseError(r, copybuf);
    1493             : 
    1494      468310 :     return copybuf[(*cursor)++];
    1495             : }
    1496             : 
    1497             : /*
    1498             :  * Get a NUL-terminated string from a CopyData message.
    1499             :  *
    1500             :  * Bail out if the terminating NUL cannot be found.
    1501             :  */
    1502             : static char *
    1503         572 : GetCopyDataString(size_t r, char *copybuf, size_t *cursor)
    1504             : {
    1505         572 :     size_t      startpos = *cursor;
    1506         572 :     size_t      endpos = startpos;
    1507             : 
    1508             :     while (1)
    1509             :     {
    1510        4088 :         if (endpos >= r)
    1511           0 :             ReportCopyDataParseError(r, copybuf);
    1512        4088 :         if (copybuf[endpos] == '\0')
    1513         572 :             break;
    1514        3516 :         ++endpos;
    1515             :     }
    1516             : 
    1517         572 :     *cursor = endpos + 1;
    1518         572 :     return &copybuf[startpos];
    1519             : }
    1520             : 
    1521             : /*
    1522             :  * Get an unsigned 64-bit integer from a CopyData message.
    1523             :  *
    1524             :  * Bail out if there are not at least 8 bytes remaining.
    1525             :  */
    1526             : static uint64
    1527         280 : GetCopyDataUInt64(size_t r, char *copybuf, size_t *cursor)
    1528             : {
    1529             :     uint64      result;
    1530             : 
    1531         280 :     if (*cursor + sizeof(uint64) > r)
    1532           0 :         ReportCopyDataParseError(r, copybuf);
    1533         280 :     memcpy(&result, &copybuf[*cursor], sizeof(uint64));
    1534         280 :     *cursor += sizeof(uint64);
    1535         280 :     return pg_ntoh64(result);
    1536             : }
    1537             : 
    1538             : /*
    1539             :  * Bail out if we didn't parse the whole message.
    1540             :  */
    1541             : static void
    1542         788 : GetCopyDataEnd(size_t r, char *copybuf, size_t cursor)
    1543             : {
    1544         788 :     if (r != cursor)
    1545           0 :         ReportCopyDataParseError(r, copybuf);
    1546         788 : }
    1547             : 
    1548             : /*
    1549             :  * Report failure to parse a CopyData message from the server. Then exit.
    1550             :  *
    1551             :  * As a debugging aid, we try to give some hint about what kind of message
    1552             :  * provoked the failure. Perhaps this is not detailed enough, but it's not
    1553             :  * clear that it's worth expending any more code on what should be a
    1554             :  * can't-happen case.
    1555             :  */
    1556             : static void
    1557           0 : ReportCopyDataParseError(size_t r, char *copybuf)
    1558             : {
    1559           0 :     if (r == 0)
    1560           0 :         pg_fatal("empty COPY message");
    1561             :     else
    1562           0 :         pg_fatal("malformed COPY message of type %d, length %zu",
    1563             :                  copybuf[0], r);
    1564             : }
    1565             : 
    1566             : /*
    1567             :  * Receive raw tar data from the server, and stream it to the appropriate
    1568             :  * location. If we're writing a single tarfile to standard output, also
    1569             :  * receive the backup manifest and inject it into that tarfile.
    1570             :  */
    1571             : static void
    1572           0 : ReceiveTarFile(PGconn *conn, char *archive_name, char *spclocation,
    1573             :                bool tablespacenum, pg_compress_specification *compress)
    1574             : {
    1575             :     WriteTarState state;
    1576             :     bbstreamer *manifest_inject_streamer;
    1577             :     bool        is_recovery_guc_supported;
    1578             :     bool        expect_unterminated_tarfile;
    1579             : 
    1580             :     /* Pass all COPY data through to the backup streamer. */
    1581           0 :     memset(&state, 0, sizeof(state));
    1582           0 :     is_recovery_guc_supported =
    1583           0 :         PQserverVersion(conn) >= MINIMUM_VERSION_FOR_RECOVERY_GUC;
    1584           0 :     expect_unterminated_tarfile =
    1585           0 :         PQserverVersion(conn) < MINIMUM_VERSION_FOR_TERMINATED_TARFILE;
    1586           0 :     state.streamer = CreateBackupStreamer(archive_name, spclocation,
    1587             :                                           &manifest_inject_streamer,
    1588             :                                           is_recovery_guc_supported,
    1589             :                                           expect_unterminated_tarfile,
    1590             :                                           compress);
    1591           0 :     state.tablespacenum = tablespacenum;
    1592           0 :     ReceiveCopyData(conn, ReceiveTarCopyChunk, &state);
    1593           0 :     progress_update_filename(NULL);
    1594             : 
    1595             :     /*
    1596             :      * The decision as to whether we need to inject the backup manifest into
    1597             :      * the output at this stage is made by CreateBackupStreamer; if that is
    1598             :      * needed, manifest_inject_streamer will be non-NULL; otherwise, it will
    1599             :      * be NULL.
    1600             :      */
    1601           0 :     if (manifest_inject_streamer != NULL)
    1602             :     {
    1603             :         PQExpBufferData buf;
    1604             : 
    1605             :         /* Slurp the entire backup manifest into a buffer. */
    1606           0 :         initPQExpBuffer(&buf);
    1607           0 :         ReceiveBackupManifestInMemory(conn, &buf);
    1608           0 :         if (PQExpBufferDataBroken(buf))
    1609           0 :             pg_fatal("out of memory");
    1610             : 
    1611             :         /* Inject it into the output tarfile. */
    1612           0 :         bbstreamer_inject_file(manifest_inject_streamer, "backup_manifest",
    1613           0 :                                buf.data, buf.len);
    1614             : 
    1615             :         /* Free memory. */
    1616           0 :         termPQExpBuffer(&buf);
    1617             :     }
    1618             : 
    1619             :     /* Cleanup. */
    1620           0 :     bbstreamer_finalize(state.streamer);
    1621           0 :     bbstreamer_free(state.streamer);
    1622             : 
    1623           0 :     progress_report(tablespacenum, true, false);
    1624             : 
    1625             :     /*
    1626             :      * Do not sync the resulting tar file yet, all files are synced once at
    1627             :      * the end.
    1628             :      */
    1629           0 : }
    1630             : 
    1631             : /*
    1632             :  * Receive one chunk of tar-format data from the server.
    1633             :  */
    1634             : static void
    1635           0 : ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data)
    1636             : {
    1637           0 :     WriteTarState *state = callback_data;
    1638             : 
    1639           0 :     bbstreamer_content(state->streamer, NULL, copybuf, r, BBSTREAMER_UNKNOWN);
    1640             : 
    1641           0 :     totaldone += r;
    1642           0 :     progress_report(state->tablespacenum, false, false);
    1643           0 : }
    1644             : 
    1645             : 
    1646             : /*
    1647             :  * Retrieve tablespace path, either relocated or original depending on whether
    1648             :  * -T was passed or not.
    1649             :  */
    1650             : static const char *
    1651          80 : get_tablespace_mapping(const char *dir)
    1652             : {
    1653             :     TablespaceListCell *cell;
    1654             :     char        canon_dir[MAXPGPATH];
    1655             : 
    1656             :     /* Canonicalize path for comparison consistency */
    1657          80 :     strlcpy(canon_dir, dir, sizeof(canon_dir));
    1658          80 :     canonicalize_path(canon_dir);
    1659             : 
    1660          80 :     for (cell = tablespace_dirs.head; cell; cell = cell->next)
    1661          78 :         if (strcmp(canon_dir, cell->old_dir) == 0)
    1662          78 :             return cell->new_dir;
    1663             : 
    1664           2 :     return dir;
    1665             : }
    1666             : 
    1667             : /*
    1668             :  * Receive the backup manifest file and write it out to a file.
    1669             :  */
    1670             : static void
    1671           0 : ReceiveBackupManifest(PGconn *conn)
    1672             : {
    1673             :     WriteManifestState state;
    1674             : 
    1675           0 :     snprintf(state.filename, sizeof(state.filename),
    1676             :              "%s/backup_manifest.tmp", basedir);
    1677           0 :     state.file = fopen(state.filename, "wb");
    1678           0 :     if (state.file == NULL)
    1679           0 :         pg_fatal("could not create file \"%s\": %m", state.filename);
    1680             : 
    1681           0 :     ReceiveCopyData(conn, ReceiveBackupManifestChunk, &state);
    1682             : 
    1683           0 :     fclose(state.file);
    1684           0 : }
    1685             : 
    1686             : /*
    1687             :  * Receive one chunk of the backup manifest file and write it out to a file.
    1688             :  */
    1689             : static void
    1690           0 : ReceiveBackupManifestChunk(size_t r, char *copybuf, void *callback_data)
    1691             : {
    1692           0 :     WriteManifestState *state = callback_data;
    1693             : 
    1694           0 :     errno = 0;
    1695           0 :     if (fwrite(copybuf, r, 1, state->file) != 1)
    1696             :     {
    1697             :         /* if write didn't set errno, assume problem is no disk space */
    1698           0 :         if (errno == 0)
    1699           0 :             errno = ENOSPC;
    1700           0 :         pg_fatal("could not write to file \"%s\": %m", state->filename);
    1701             :     }
    1702           0 : }
    1703             : 
    1704             : /*
    1705             :  * Receive the backup manifest file and write it out to a file.
    1706             :  */
    1707             : static void
    1708           0 : ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf)
    1709             : {
    1710           0 :     ReceiveCopyData(conn, ReceiveBackupManifestInMemoryChunk, buf);
    1711           0 : }
    1712             : 
    1713             : /*
    1714             :  * Receive one chunk of the backup manifest file and write it out to a file.
    1715             :  */
    1716             : static void
    1717           0 : ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf,
    1718             :                                    void *callback_data)
    1719             : {
    1720           0 :     PQExpBuffer buf = callback_data;
    1721             : 
    1722           0 :     appendPQExpBuffer(buf, copybuf, r);
    1723           0 : }
    1724             : 
    1725             : static void
    1726         268 : BaseBackup(char *compression_algorithm, char *compression_detail,
    1727             :            CompressionLocation compressloc, pg_compress_specification *client_compress)
    1728             : {
    1729             :     PGresult   *res;
    1730             :     char       *sysidentifier;
    1731             :     TimeLineID  latesttli;
    1732             :     TimeLineID  starttli;
    1733             :     char       *basebkp;
    1734             :     int         i;
    1735             :     char        xlogstart[64];
    1736         268 :     char        xlogend[64] = {0};
    1737             :     int         minServerMajor,
    1738             :                 maxServerMajor;
    1739             :     int         serverVersion,
    1740             :                 serverMajor;
    1741             :     int         writing_to_stdout;
    1742         268 :     bool        use_new_option_syntax = false;
    1743             :     PQExpBufferData buf;
    1744             : 
    1745             :     Assert(conn != NULL);
    1746         268 :     initPQExpBuffer(&buf);
    1747             : 
    1748             :     /*
    1749             :      * Check server version. BASE_BACKUP command was introduced in 9.1, so we
    1750             :      * can't work with servers older than 9.1.
    1751             :      */
    1752         268 :     minServerMajor = 901;
    1753         268 :     maxServerMajor = PG_VERSION_NUM / 100;
    1754         268 :     serverVersion = PQserverVersion(conn);
    1755         268 :     serverMajor = serverVersion / 100;
    1756         268 :     if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
    1757             :     {
    1758           0 :         const char *serverver = PQparameterStatus(conn, "server_version");
    1759             : 
    1760           0 :         pg_fatal("incompatible server version %s",
    1761             :                  serverver ? serverver : "'unknown'");
    1762             :     }
    1763         268 :     if (serverMajor >= 1500)
    1764         268 :         use_new_option_syntax = true;
    1765             : 
    1766             :     /*
    1767             :      * If WAL streaming was requested, also check that the server is new
    1768             :      * enough for that.
    1769             :      */
    1770         268 :     if (includewal == STREAM_WAL && !CheckServerVersionForStreaming(conn))
    1771             :     {
    1772             :         /*
    1773             :          * Error message already written in CheckServerVersionForStreaming(),
    1774             :          * but add a hint about using -X none.
    1775             :          */
    1776           0 :         pg_log_error_hint("Use -X none or -X fetch to disable log streaming.");
    1777           0 :         exit(1);
    1778             :     }
    1779             : 
    1780             :     /*
    1781             :      * Build contents of configuration file if requested
    1782             :      */
    1783         268 :     if (writerecoveryconf)
    1784           4 :         recoveryconfcontents = GenerateRecoveryConfig(conn, replication_slot);
    1785             : 
    1786             :     /*
    1787             :      * Run IDENTIFY_SYSTEM so we can get the timeline
    1788             :      */
    1789         268 :     if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
    1790           0 :         exit(1);
    1791             : 
    1792             :     /*
    1793             :      * Start the actual backup
    1794             :      */
    1795         268 :     AppendStringCommandOption(&buf, use_new_option_syntax, "LABEL", label);
    1796         268 :     if (estimatesize)
    1797         268 :         AppendPlainCommandOption(&buf, use_new_option_syntax, "PROGRESS");
    1798         268 :     if (includewal == FETCH_WAL)
    1799          30 :         AppendPlainCommandOption(&buf, use_new_option_syntax, "WAL");
    1800         268 :     if (fastcheckpoint)
    1801             :     {
    1802         248 :         if (use_new_option_syntax)
    1803         248 :             AppendStringCommandOption(&buf, use_new_option_syntax,
    1804             :                                       "CHECKPOINT", "fast");
    1805             :         else
    1806           0 :             AppendPlainCommandOption(&buf, use_new_option_syntax, "FAST");
    1807             :     }
    1808         268 :     if (includewal != NO_WAL)
    1809             :     {
    1810         252 :         if (use_new_option_syntax)
    1811         252 :             AppendIntegerCommandOption(&buf, use_new_option_syntax, "WAIT", 0);
    1812             :         else
    1813           0 :             AppendPlainCommandOption(&buf, use_new_option_syntax, "NOWAIT");
    1814             :     }
    1815         268 :     if (maxrate > 0)
    1816           2 :         AppendIntegerCommandOption(&buf, use_new_option_syntax, "MAX_RATE",
    1817             :                                    maxrate);
    1818         268 :     if (format == 't')
    1819          22 :         AppendPlainCommandOption(&buf, use_new_option_syntax, "TABLESPACE_MAP");
    1820         268 :     if (!verify_checksums)
    1821             :     {
    1822           2 :         if (use_new_option_syntax)
    1823           2 :             AppendIntegerCommandOption(&buf, use_new_option_syntax,
    1824             :                                        "VERIFY_CHECKSUMS", 0);
    1825             :         else
    1826           0 :             AppendPlainCommandOption(&buf, use_new_option_syntax,
    1827             :                                      "NOVERIFY_CHECKSUMS");
    1828             :     }
    1829             : 
    1830         268 :     if (manifest)
    1831             :     {
    1832         266 :         AppendStringCommandOption(&buf, use_new_option_syntax, "MANIFEST",
    1833         266 :                                   manifest_force_encode ? "force-encode" : "yes");
    1834         266 :         if (manifest_checksums != NULL)
    1835          14 :             AppendStringCommandOption(&buf, use_new_option_syntax,
    1836             :                                       "MANIFEST_CHECKSUMS", manifest_checksums);
    1837             :     }
    1838             : 
    1839         268 :     if (backup_target != NULL)
    1840             :     {
    1841             :         char       *colon;
    1842             : 
    1843          24 :         if (serverMajor < 1500)
    1844           0 :             pg_fatal("backup targets are not supported by this server version");
    1845             : 
    1846          24 :         if (writerecoveryconf)
    1847           0 :             pg_fatal("recovery configuration cannot be written when a backup target is used");
    1848             : 
    1849          24 :         AppendPlainCommandOption(&buf, use_new_option_syntax, "TABLESPACE_MAP");
    1850             : 
    1851          24 :         if ((colon = strchr(backup_target, ':')) == NULL)
    1852             :         {
    1853          12 :             AppendStringCommandOption(&buf, use_new_option_syntax,
    1854             :                                       "TARGET", backup_target);
    1855             :         }
    1856             :         else
    1857             :         {
    1858             :             char       *target;
    1859             : 
    1860          12 :             target = pnstrdup(backup_target, colon - backup_target);
    1861          12 :             AppendStringCommandOption(&buf, use_new_option_syntax,
    1862             :                                       "TARGET", target);
    1863          12 :             AppendStringCommandOption(&buf, use_new_option_syntax,
    1864             :                                       "TARGET_DETAIL", colon + 1);
    1865             :         }
    1866             :     }
    1867         244 :     else if (serverMajor >= 1500)
    1868         244 :         AppendStringCommandOption(&buf, use_new_option_syntax,
    1869             :                                   "TARGET", "client");
    1870             : 
    1871         268 :     if (compressloc == COMPRESS_LOCATION_SERVER)
    1872             :     {
    1873          48 :         if (!use_new_option_syntax)
    1874           0 :             pg_fatal("server does not support server-side compression");
    1875          48 :         AppendStringCommandOption(&buf, use_new_option_syntax,
    1876             :                                   "COMPRESSION", compression_algorithm);
    1877          48 :         if (compression_detail != NULL)
    1878          22 :             AppendStringCommandOption(&buf, use_new_option_syntax,
    1879             :                                       "COMPRESSION_DETAIL",
    1880             :                                       compression_detail);
    1881             :     }
    1882             : 
    1883         268 :     if (verbose)
    1884           0 :         pg_log_info("initiating base backup, waiting for checkpoint to complete");
    1885             : 
    1886         268 :     if (showprogress && !verbose)
    1887             :     {
    1888           0 :         fprintf(stderr, _("waiting for checkpoint"));
    1889           0 :         if (isatty(fileno(stderr)))
    1890           0 :             fprintf(stderr, "\r");
    1891             :         else
    1892           0 :             fprintf(stderr, "\n");
    1893             :     }
    1894             : 
    1895         268 :     if (use_new_option_syntax && buf.len > 0)
    1896         268 :         basebkp = psprintf("BASE_BACKUP (%s)", buf.data);
    1897             :     else
    1898           0 :         basebkp = psprintf("BASE_BACKUP %s", buf.data);
    1899             : 
    1900         268 :     if (PQsendQuery(conn, basebkp) == 0)
    1901           0 :         pg_fatal("could not send replication command \"%s\": %s",
    1902             :                  "BASE_BACKUP", PQerrorMessage(conn));
    1903             : 
    1904             :     /*
    1905             :      * Get the starting WAL location
    1906             :      */
    1907         268 :     res = PQgetResult(conn);
    1908         268 :     if (PQresultStatus(res) != PGRES_TUPLES_OK)
    1909          32 :         pg_fatal("could not initiate base backup: %s",
    1910             :                  PQerrorMessage(conn));
    1911         236 :     if (PQntuples(res) != 1)
    1912           0 :         pg_fatal("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields",
    1913             :                  PQntuples(res), PQnfields(res), 1, 2);
    1914             : 
    1915         236 :     strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
    1916             : 
    1917         236 :     if (verbose)
    1918           0 :         pg_log_info("checkpoint completed");
    1919             : 
    1920             :     /*
    1921             :      * 9.3 and later sends the TLI of the starting point. With older servers,
    1922             :      * assume it's the same as the latest timeline reported by
    1923             :      * IDENTIFY_SYSTEM.
    1924             :      */
    1925         236 :     if (PQnfields(res) >= 2)
    1926         236 :         starttli = atoi(PQgetvalue(res, 0, 1));
    1927             :     else
    1928           0 :         starttli = latesttli;
    1929         236 :     PQclear(res);
    1930             : 
    1931         236 :     if (verbose && includewal != NO_WAL)
    1932           0 :         pg_log_info("write-ahead log start point: %s on timeline %u",
    1933             :                     xlogstart, starttli);
    1934             : 
    1935             :     /*
    1936             :      * Get the header
    1937             :      */
    1938         236 :     res = PQgetResult(conn);
    1939         236 :     if (PQresultStatus(res) != PGRES_TUPLES_OK)
    1940           0 :         pg_fatal("could not get backup header: %s",
    1941             :                  PQerrorMessage(conn));
    1942         236 :     if (PQntuples(res) < 1)
    1943           0 :         pg_fatal("no data returned from server");
    1944             : 
    1945             :     /*
    1946             :      * Sum up the total size, for progress reporting
    1947             :      */
    1948         236 :     totalsize_kb = totaldone = 0;
    1949         236 :     tablespacecount = PQntuples(res);
    1950         524 :     for (i = 0; i < PQntuples(res); i++)
    1951             :     {
    1952         290 :         totalsize_kb += atol(PQgetvalue(res, i, 2));
    1953             : 
    1954             :         /*
    1955             :          * Verify tablespace directories are empty. Don't bother with the
    1956             :          * first once since it can be relocated, and it will be checked before
    1957             :          * we do anything anyway.
    1958             :          *
    1959             :          * Note that this is skipped for tar format backups and backups that
    1960             :          * the server is storing to a target location, since in that case we
    1961             :          * won't be storing anything into these directories and thus should
    1962             :          * not create them.
    1963             :          */
    1964         290 :         if (backup_target == NULL && format == 'p' && !PQgetisnull(res, i, 1))
    1965             :         {
    1966          50 :             char       *path = PQgetvalue(res, i, 1);
    1967             : 
    1968          50 :             if (is_absolute_path(path))
    1969          28 :                 path = unconstify(char *, get_tablespace_mapping(path));
    1970             :             else
    1971             :             {
    1972             :                 /* This is an in-place tablespace, so prepend basedir. */
    1973          22 :                 path = psprintf("%s/%s", basedir, path);
    1974             :             }
    1975             : 
    1976          50 :             verify_dir_is_empty_or_create(path, &made_tablespace_dirs, &found_tablespace_dirs);
    1977             :         }
    1978             :     }
    1979             : 
    1980             :     /*
    1981             :      * When writing to stdout, require a single tablespace
    1982             :      */
    1983         256 :     writing_to_stdout = format == 't' && basedir != NULL &&
    1984          22 :         strcmp(basedir, "-") == 0;
    1985         234 :     if (writing_to_stdout && PQntuples(res) > 1)
    1986           0 :         pg_fatal("can only write single tablespace to stdout, database has %d",
    1987             :                  PQntuples(res));
    1988             : 
    1989             :     /*
    1990             :      * If we're streaming WAL, start the streaming session before we start
    1991             :      * receiving the actual data chunks.
    1992             :      */
    1993         234 :     if (includewal == STREAM_WAL)
    1994             :     {
    1995             :         pg_compress_algorithm wal_compress_algorithm;
    1996             :         int         wal_compress_level;
    1997             : 
    1998         198 :         if (verbose)
    1999           0 :             pg_log_info("starting background WAL receiver");
    2000             : 
    2001         198 :         if (client_compress->algorithm == PG_COMPRESSION_GZIP)
    2002             :         {
    2003           6 :             wal_compress_algorithm = PG_COMPRESSION_GZIP;
    2004           6 :             wal_compress_level = client_compress->level;
    2005             :         }
    2006             :         else
    2007             :         {
    2008         192 :             wal_compress_algorithm = PG_COMPRESSION_NONE;
    2009         192 :             wal_compress_level = 0;
    2010             :         }
    2011             : 
    2012         198 :         StartLogStreamer(xlogstart, starttli, sysidentifier,
    2013             :                          wal_compress_algorithm,
    2014             :                          wal_compress_level);
    2015             :     }
    2016             : 
    2017         232 :     if (serverMajor >= 1500)
    2018             :     {
    2019             :         /* Receive a single tar stream with everything. */
    2020         232 :         ReceiveArchiveStream(conn, client_compress);
    2021             :     }
    2022             :     else
    2023             :     {
    2024             :         /* Receive a tar file for each tablespace in turn */
    2025           0 :         for (i = 0; i < PQntuples(res); i++)
    2026             :         {
    2027             :             char        archive_name[MAXPGPATH];
    2028             :             char       *spclocation;
    2029             : 
    2030             :             /*
    2031             :              * If we write the data out to a tar file, it will be named
    2032             :              * base.tar if it's the main data directory or <tablespaceoid>.tar
    2033             :              * if it's for another tablespace. CreateBackupStreamer() will
    2034             :              * arrange to add an extension to the archive name if
    2035             :              * pg_basebackup is performing compression, depending on the
    2036             :              * compression type.
    2037             :              */
    2038           0 :             if (PQgetisnull(res, i, 0))
    2039             :             {
    2040           0 :                 strlcpy(archive_name, "base.tar", sizeof(archive_name));
    2041           0 :                 spclocation = NULL;
    2042             :             }
    2043             :             else
    2044             :             {
    2045           0 :                 snprintf(archive_name, sizeof(archive_name),
    2046             :                          "%s.tar", PQgetvalue(res, i, 0));
    2047           0 :                 spclocation = PQgetvalue(res, i, 1);
    2048             :             }
    2049             : 
    2050           0 :             ReceiveTarFile(conn, archive_name, spclocation, i,
    2051             :                            client_compress);
    2052             :         }
    2053             : 
    2054             :         /*
    2055             :          * Now receive backup manifest, if appropriate.
    2056             :          *
    2057             :          * If we're writing a tarfile to stdout, ReceiveTarFile will have
    2058             :          * already processed the backup manifest and included it in the output
    2059             :          * tarfile.  Such a configuration doesn't allow for writing multiple
    2060             :          * files.
    2061             :          *
    2062             :          * If we're talking to an older server, it won't send a backup
    2063             :          * manifest, so don't try to receive one.
    2064             :          */
    2065           0 :         if (!writing_to_stdout && manifest)
    2066           0 :             ReceiveBackupManifest(conn);
    2067             :     }
    2068             : 
    2069         226 :     if (showprogress)
    2070             :     {
    2071           0 :         progress_update_filename(NULL);
    2072           0 :         progress_report(PQntuples(res), true, true);
    2073             :     }
    2074             : 
    2075         226 :     PQclear(res);
    2076             : 
    2077             :     /*
    2078             :      * Get the stop position
    2079             :      */
    2080         226 :     res = PQgetResult(conn);
    2081         226 :     if (PQresultStatus(res) != PGRES_TUPLES_OK)
    2082           2 :         pg_fatal("backup failed: %s",
    2083             :                  PQerrorMessage(conn));
    2084         224 :     if (PQntuples(res) != 1)
    2085           0 :         pg_fatal("no write-ahead log end position returned from server");
    2086         224 :     strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
    2087         224 :     if (verbose && includewal != NO_WAL)
    2088           0 :         pg_log_info("write-ahead log end point: %s", xlogend);
    2089         224 :     PQclear(res);
    2090             : 
    2091         224 :     res = PQgetResult(conn);
    2092         224 :     if (PQresultStatus(res) != PGRES_COMMAND_OK)
    2093             :     {
    2094           6 :         const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
    2095             : 
    2096           6 :         if (sqlstate &&
    2097           6 :             strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
    2098             :         {
    2099           6 :             pg_log_error("checksum error occurred");
    2100           6 :             checksum_failure = true;
    2101             :         }
    2102             :         else
    2103             :         {
    2104           0 :             pg_log_error("final receive failed: %s",
    2105             :                          PQerrorMessage(conn));
    2106             :         }
    2107           6 :         exit(1);
    2108             :     }
    2109             : 
    2110         218 :     if (bgchild > 0)
    2111             :     {
    2112             : #ifndef WIN32
    2113             :         int         status;
    2114             :         pid_t       r;
    2115             : #else
    2116             :         DWORD       status;
    2117             : 
    2118             :         /*
    2119             :          * get a pointer sized version of bgchild to avoid warnings about
    2120             :          * casting to a different size on WIN64.
    2121             :          */
    2122             :         intptr_t    bgchild_handle = bgchild;
    2123             :         uint32      hi,
    2124             :                     lo;
    2125             : #endif
    2126             : 
    2127         182 :         if (verbose)
    2128           0 :             pg_log_info("waiting for background process to finish streaming ...");
    2129             : 
    2130             : #ifndef WIN32
    2131         182 :         if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
    2132           0 :             pg_fatal("could not send command to background pipe: %m");
    2133             : 
    2134             :         /* Just wait for the background process to exit */
    2135         182 :         r = waitpid(bgchild, &status, 0);
    2136         182 :         if (r == (pid_t) -1)
    2137           0 :             pg_fatal("could not wait for child process: %m");
    2138         182 :         if (r != bgchild)
    2139           0 :             pg_fatal("child %d died, expected %d", (int) r, (int) bgchild);
    2140         182 :         if (status != 0)
    2141           0 :             pg_fatal("%s", wait_result_to_str(status));
    2142             :         /* Exited normally, we're happy! */
    2143             : #else                           /* WIN32 */
    2144             : 
    2145             :         /*
    2146             :          * On Windows, since we are in the same process, we can just store the
    2147             :          * value directly in the variable, and then set the flag that says
    2148             :          * it's there.
    2149             :          */
    2150             :         if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
    2151             :             pg_fatal("could not parse write-ahead log location \"%s\"",
    2152             :                      xlogend);
    2153             :         xlogendptr = ((uint64) hi) << 32 | lo;
    2154             :         InterlockedIncrement(&has_xlogendptr);
    2155             : 
    2156             :         /* First wait for the thread to exit */
    2157             :         if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
    2158             :             WAIT_OBJECT_0)
    2159             :         {
    2160             :             _dosmaperr(GetLastError());
    2161             :             pg_fatal("could not wait for child thread: %m");
    2162             :         }
    2163             :         if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
    2164             :         {
    2165             :             _dosmaperr(GetLastError());
    2166             :             pg_fatal("could not get child thread exit status: %m");
    2167             :         }
    2168             :         if (status != 0)
    2169             :             pg_fatal("child thread exited with error %u",
    2170             :                      (unsigned int) status);
    2171             :         /* Exited normally, we're happy */
    2172             : #endif
    2173             :     }
    2174             : 
    2175             :     /* Free the configuration file contents */
    2176         218 :     destroyPQExpBuffer(recoveryconfcontents);
    2177             : 
    2178             :     /*
    2179             :      * End of copy data. Final result is already checked inside the loop.
    2180             :      */
    2181         218 :     PQclear(res);
    2182         218 :     PQfinish(conn);
    2183         218 :     conn = NULL;
    2184             : 
    2185             :     /*
    2186             :      * Make data persistent on disk once backup is completed. For tar format
    2187             :      * sync the parent directory and all its contents as each tar file was not
    2188             :      * synced after being completed.  In plain format, all the data of the
    2189             :      * base directory is synced, taking into account all the tablespaces.
    2190             :      * Errors are not considered fatal.
    2191             :      *
    2192             :      * If, however, there's a backup target, we're not writing anything
    2193             :      * locally, so in that case we skip this step.
    2194             :      */
    2195         218 :     if (do_sync && backup_target == NULL)
    2196             :     {
    2197           0 :         if (verbose)
    2198           0 :             pg_log_info("syncing data to disk ...");
    2199           0 :         if (format == 't')
    2200             :         {
    2201           0 :             if (strcmp(basedir, "-") != 0)
    2202           0 :                 (void) fsync_dir_recurse(basedir);
    2203             :         }
    2204             :         else
    2205             :         {
    2206           0 :             (void) fsync_pgdata(basedir, serverVersion);
    2207             :         }
    2208             :     }
    2209             : 
    2210             :     /*
    2211             :      * After synchronizing data to disk, perform a durable rename of
    2212             :      * backup_manifest.tmp to backup_manifest, if we wrote such a file. This
    2213             :      * way, a failure or system crash before we reach this point will leave us
    2214             :      * without a backup_manifest file, decreasing the chances that a directory
    2215             :      * we leave behind will be mistaken for a valid backup.
    2216             :      */
    2217         218 :     if (!writing_to_stdout && manifest && backup_target == NULL)
    2218             :     {
    2219             :         char        tmp_filename[MAXPGPATH];
    2220             :         char        filename[MAXPGPATH];
    2221             : 
    2222         202 :         if (verbose)
    2223           0 :             pg_log_info("renaming backup_manifest.tmp to backup_manifest");
    2224             : 
    2225         202 :         snprintf(tmp_filename, MAXPGPATH, "%s/backup_manifest.tmp", basedir);
    2226         202 :         snprintf(filename, MAXPGPATH, "%s/backup_manifest", basedir);
    2227             : 
    2228         202 :         if (do_sync)
    2229             :         {
    2230             :             /* durable_rename emits its own log message in case of failure */
    2231           0 :             if (durable_rename(tmp_filename, filename) != 0)
    2232           0 :                 exit(1);
    2233             :         }
    2234             :         else
    2235             :         {
    2236         202 :             if (rename(tmp_filename, filename) != 0)
    2237           0 :                 pg_fatal("could not rename file \"%s\" to \"%s\": %m",
    2238             :                          tmp_filename, filename);
    2239             :         }
    2240             :     }
    2241             : 
    2242         218 :     if (verbose)
    2243           0 :         pg_log_info("base backup completed");
    2244         218 : }
    2245             : 
    2246             : 
    2247             : int
    2248         336 : main(int argc, char **argv)
    2249             : {
    2250             :     static struct option long_options[] = {
    2251             :         {"help", no_argument, NULL, '?'},
    2252             :         {"version", no_argument, NULL, 'V'},
    2253             :         {"pgdata", required_argument, NULL, 'D'},
    2254             :         {"format", required_argument, NULL, 'F'},
    2255             :         {"checkpoint", required_argument, NULL, 'c'},
    2256             :         {"create-slot", no_argument, NULL, 'C'},
    2257             :         {"max-rate", required_argument, NULL, 'r'},
    2258             :         {"write-recovery-conf", no_argument, NULL, 'R'},
    2259             :         {"slot", required_argument, NULL, 'S'},
    2260             :         {"target", required_argument, NULL, 't'},
    2261             :         {"tablespace-mapping", required_argument, NULL, 'T'},
    2262             :         {"wal-method", required_argument, NULL, 'X'},
    2263             :         {"gzip", no_argument, NULL, 'z'},
    2264             :         {"compress", required_argument, NULL, 'Z'},
    2265             :         {"label", required_argument, NULL, 'l'},
    2266             :         {"no-clean", no_argument, NULL, 'n'},
    2267             :         {"no-sync", no_argument, NULL, 'N'},
    2268             :         {"dbname", required_argument, NULL, 'd'},
    2269             :         {"host", required_argument, NULL, 'h'},
    2270             :         {"port", required_argument, NULL, 'p'},
    2271             :         {"username", required_argument, NULL, 'U'},
    2272             :         {"no-password", no_argument, NULL, 'w'},
    2273             :         {"password", no_argument, NULL, 'W'},
    2274             :         {"status-interval", required_argument, NULL, 's'},
    2275             :         {"verbose", no_argument, NULL, 'v'},
    2276             :         {"progress", no_argument, NULL, 'P'},
    2277             :         {"waldir", required_argument, NULL, 1},
    2278             :         {"no-slot", no_argument, NULL, 2},
    2279             :         {"no-verify-checksums", no_argument, NULL, 3},
    2280             :         {"no-estimate-size", no_argument, NULL, 4},
    2281             :         {"no-manifest", no_argument, NULL, 5},
    2282             :         {"manifest-force-encode", no_argument, NULL, 6},
    2283             :         {"manifest-checksums", required_argument, NULL, 7},
    2284             :         {NULL, 0, NULL, 0}
    2285             :     };
    2286             :     int         c;
    2287             : 
    2288             :     int         option_index;
    2289         336 :     char       *compression_algorithm = "none";
    2290         336 :     char       *compression_detail = NULL;
    2291         336 :     CompressionLocation compressloc = COMPRESS_LOCATION_UNSPECIFIED;
    2292             :     pg_compress_specification client_compress;
    2293             : 
    2294         336 :     pg_logging_init(argv[0]);
    2295         336 :     progname = get_progname(argv[0]);
    2296         336 :     set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
    2297             : 
    2298         336 :     if (argc > 1)
    2299             :     {
    2300         334 :         if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
    2301             :         {
    2302           2 :             usage();
    2303           2 :             exit(0);
    2304             :         }
    2305         332 :         else if (strcmp(argv[1], "-V") == 0
    2306         332 :                  || strcmp(argv[1], "--version") == 0)
    2307             :         {
    2308           2 :             puts("pg_basebackup (PostgreSQL) " PG_VERSION);
    2309           2 :             exit(0);
    2310             :         }
    2311             :     }
    2312             : 
    2313         332 :     atexit(cleanup_directories_atexit);
    2314             : 
    2315        1732 :     while ((c = getopt_long(argc, argv, "c:Cd:D:F:h:l:nNp:Pr:Rs:S:t:T:U:vwWX:zZ:",
    2316             :                             long_options, &option_index)) != -1)
    2317             :     {
    2318        1414 :         switch (c)
    2319             :         {
    2320         284 :             case 'c':
    2321         284 :                 if (pg_strcasecmp(optarg, "fast") == 0)
    2322         284 :                     fastcheckpoint = true;
    2323           0 :                 else if (pg_strcasecmp(optarg, "spread") == 0)
    2324           0 :                     fastcheckpoint = false;
    2325             :                 else
    2326           0 :                     pg_fatal("invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"",
    2327             :                              optarg);
    2328         284 :                 break;
    2329          12 :             case 'C':
    2330          12 :                 create_slot = true;
    2331          12 :                 break;
    2332           2 :             case 'd':
    2333           2 :                 connection_string = pg_strdup(optarg);
    2334           2 :                 break;
    2335         298 :             case 'D':
    2336         298 :                 basedir = pg_strdup(optarg);
    2337         298 :                 break;
    2338          48 :             case 'F':
    2339          48 :                 if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
    2340          24 :                     format = 'p';
    2341          24 :                 else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
    2342          24 :                     format = 't';
    2343             :                 else
    2344           0 :                     pg_fatal("invalid output format \"%s\", must be \"plain\" or \"tar\"",
    2345             :                              optarg);
    2346          48 :                 break;
    2347         108 :             case 'h':
    2348         108 :                 dbhost = pg_strdup(optarg);
    2349         108 :                 break;
    2350           0 :             case 'l':
    2351           0 :                 label = pg_strdup(optarg);
    2352           0 :                 break;
    2353           2 :             case 'n':
    2354           2 :                 noclean = true;
    2355           2 :                 break;
    2356         284 :             case 'N':
    2357         284 :                 do_sync = false;
    2358         284 :                 break;
    2359         108 :             case 'p':
    2360         108 :                 dbport = pg_strdup(optarg);
    2361         108 :                 break;
    2362           0 :             case 'P':
    2363           0 :                 showprogress = true;
    2364           0 :                 break;
    2365           2 :             case 'r':
    2366           2 :                 maxrate = parse_max_rate(optarg);
    2367           2 :                 break;
    2368           4 :             case 'R':
    2369           4 :                 writerecoveryconf = true;
    2370           4 :                 break;
    2371           0 :             case 's':
    2372           0 :                 if (!option_parse_int(optarg, "-s/--status-interval", 0,
    2373             :                                       INT_MAX / 1000,
    2374             :                                       &standby_message_timeout))
    2375           0 :                     exit(1);
    2376           0 :                 standby_message_timeout *= 1000;
    2377           0 :                 break;
    2378          18 :             case 'S':
    2379             : 
    2380             :                 /*
    2381             :                  * When specifying replication slot name, use a permanent
    2382             :                  * slot.
    2383             :                  */
    2384          18 :                 replication_slot = pg_strdup(optarg);
    2385          18 :                 temp_replication_slot = false;
    2386          18 :                 break;
    2387          34 :             case 't':
    2388          34 :                 backup_target = pg_strdup(optarg);
    2389          34 :                 break;
    2390          38 :             case 'T':
    2391          38 :                 tablespace_list_append(optarg);
    2392          26 :                 break;
    2393          14 :             case 'U':
    2394          14 :                 dbuser = pg_strdup(optarg);
    2395          14 :                 break;
    2396           0 :             case 'v':
    2397           0 :                 verbose++;
    2398           0 :                 break;
    2399           0 :             case 'w':
    2400           0 :                 dbgetpassword = -1;
    2401           0 :                 break;
    2402           0 :             case 'W':
    2403           0 :                 dbgetpassword = 1;
    2404           0 :                 break;
    2405          70 :             case 'X':
    2406          70 :                 if (strcmp(optarg, "n") == 0 ||
    2407          70 :                     strcmp(optarg, "none") == 0)
    2408             :                 {
    2409          22 :                     includewal = NO_WAL;
    2410             :                 }
    2411          48 :                 else if (strcmp(optarg, "f") == 0 ||
    2412          48 :                          strcmp(optarg, "fetch") == 0)
    2413             :                 {
    2414          30 :                     includewal = FETCH_WAL;
    2415             :                 }
    2416          18 :                 else if (strcmp(optarg, "s") == 0 ||
    2417          18 :                          strcmp(optarg, "stream") == 0)
    2418             :                 {
    2419          18 :                     includewal = STREAM_WAL;
    2420             :                 }
    2421             :                 else
    2422           0 :                     pg_fatal("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
    2423             :                              optarg);
    2424          70 :                 break;
    2425           2 :             case 'z':
    2426           2 :                 compression_algorithm = "gzip";
    2427           2 :                 compression_detail = NULL;
    2428           2 :                 compressloc = COMPRESS_LOCATION_UNSPECIFIED;
    2429           2 :                 break;
    2430          56 :             case 'Z':
    2431          56 :                 backup_parse_compress_options(optarg, &compression_algorithm,
    2432             :                                               &compression_detail, &compressloc);
    2433          56 :                 break;
    2434           2 :             case 1:
    2435           2 :                 xlog_dir = pg_strdup(optarg);
    2436           2 :                 break;
    2437           6 :             case 2:
    2438           6 :                 no_slot = true;
    2439           6 :                 break;
    2440           2 :             case 3:
    2441           2 :                 verify_checksums = false;
    2442           2 :                 break;
    2443           0 :             case 4:
    2444           0 :                 estimatesize = false;
    2445           0 :                 break;
    2446           2 :             case 5:
    2447           2 :                 manifest = false;
    2448           2 :                 break;
    2449           2 :             case 6:
    2450           2 :                 manifest_force_encode = true;
    2451           2 :                 break;
    2452          14 :             case 7:
    2453          14 :                 manifest_checksums = pg_strdup(optarg);
    2454          14 :                 break;
    2455           2 :             default:
    2456             :                 /* getopt_long already emitted a complaint */
    2457           2 :                 pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2458           2 :                 exit(1);
    2459             :         }
    2460             :     }
    2461             : 
    2462             :     /*
    2463             :      * Any non-option arguments?
    2464             :      */
    2465         318 :     if (optind < argc)
    2466             :     {
    2467           0 :         pg_log_error("too many command-line arguments (first is \"%s\")",
    2468             :                      argv[optind]);
    2469           0 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2470           0 :         exit(1);
    2471             :     }
    2472             : 
    2473             :     /*
    2474             :      * Setting the backup target to 'client' is equivalent to leaving out the
    2475             :      * option. This logic allows us to assume elsewhere that the backup is
    2476             :      * being stored locally if and only if backup_target == NULL.
    2477             :      */
    2478         318 :     if (backup_target != NULL && strcmp(backup_target, "client") == 0)
    2479             :     {
    2480           0 :         pg_free(backup_target);
    2481           0 :         backup_target = NULL;
    2482             :     }
    2483             : 
    2484             :     /*
    2485             :      * Can't use --format with --target. Without --target, default format is
    2486             :      * tar.
    2487             :      */
    2488         318 :     if (backup_target != NULL && format != '\0')
    2489             :     {
    2490           2 :         pg_log_error("cannot specify both format and backup target");
    2491           2 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2492           2 :         exit(1);
    2493             :     }
    2494         316 :     if (format == '\0')
    2495         282 :         format = 'p';
    2496             : 
    2497             :     /*
    2498             :      * Either directory or backup target should be specified, but not both
    2499             :      */
    2500         316 :     if (basedir == NULL && backup_target == NULL)
    2501             :     {
    2502           2 :         pg_log_error("must specify output directory or backup target");
    2503           2 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2504           2 :         exit(1);
    2505             :     }
    2506         314 :     if (basedir != NULL && backup_target != NULL)
    2507             :     {
    2508           4 :         pg_log_error("cannot specify both output directory and backup target");
    2509           4 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2510           4 :         exit(1);
    2511             :     }
    2512             : 
    2513             :     /*
    2514             :      * If the user has not specified where to perform backup compression,
    2515             :      * default to the client, unless the user specified --target, in which
    2516             :      * case the server is the only choice.
    2517             :      */
    2518         310 :     if (compressloc == COMPRESS_LOCATION_UNSPECIFIED)
    2519             :     {
    2520         282 :         if (backup_target == NULL)
    2521         256 :             compressloc = COMPRESS_LOCATION_CLIENT;
    2522             :         else
    2523          26 :             compressloc = COMPRESS_LOCATION_SERVER;
    2524             :     }
    2525             : 
    2526             :     /*
    2527             :      * If any compression that we're doing is happening on the client side, we
    2528             :      * must try to parse the compression algorithm and detail, but if it's all
    2529             :      * on the server side, then we're just going to pass through whatever was
    2530             :      * requested and let the server decide what to do.
    2531             :      */
    2532         310 :     if (compressloc == COMPRESS_LOCATION_CLIENT)
    2533             :     {
    2534             :         pg_compress_algorithm alg;
    2535             :         char       *error_detail;
    2536             : 
    2537         258 :         if (!parse_compress_algorithm(compression_algorithm, &alg))
    2538           4 :             pg_fatal("unrecognized compression algorithm: \"%s\"",
    2539             :                      compression_algorithm);
    2540             : 
    2541         254 :         parse_compress_specification(alg, compression_detail, &client_compress);
    2542         254 :         error_detail = validate_compress_specification(&client_compress);
    2543         254 :         if (error_detail != NULL)
    2544          20 :             pg_fatal("invalid compression specification: %s",
    2545             :                      error_detail);
    2546             :     }
    2547             :     else
    2548             :     {
    2549             :         Assert(compressloc == COMPRESS_LOCATION_SERVER);
    2550          52 :         client_compress.algorithm = PG_COMPRESSION_NONE;
    2551          52 :         client_compress.options = 0;
    2552             :     }
    2553             : 
    2554             :     /*
    2555             :      * Can't perform client-side compression if the backup is not being sent
    2556             :      * to the client.
    2557             :      */
    2558         286 :     if (backup_target != NULL && compressloc == COMPRESS_LOCATION_CLIENT)
    2559             :     {
    2560           0 :         pg_log_error("client-side compression is not possible when a backup target is specified");
    2561           0 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2562           0 :         exit(1);
    2563             :     }
    2564             : 
    2565             :     /*
    2566             :      * Client-side compression doesn't make sense unless tar format is in use.
    2567             :      */
    2568         286 :     if (format == 'p' && compressloc == COMPRESS_LOCATION_CLIENT &&
    2569         212 :         client_compress.algorithm != PG_COMPRESSION_NONE)
    2570             :     {
    2571           0 :         pg_log_error("only tar mode backups can be compressed");
    2572           0 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2573           0 :         exit(1);
    2574             :     }
    2575             : 
    2576             :     /*
    2577             :      * Sanity checks for WAL method.
    2578             :      */
    2579         286 :     if (backup_target != NULL && includewal == STREAM_WAL)
    2580             :     {
    2581           4 :         pg_log_error("WAL cannot be streamed when a backup target is specified");
    2582           4 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2583           4 :         exit(1);
    2584             :     }
    2585         282 :     if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
    2586             :     {
    2587           0 :         pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
    2588           0 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2589           0 :         exit(1);
    2590             :     }
    2591             : 
    2592         282 :     if (replication_slot && includewal != STREAM_WAL)
    2593             :     {
    2594           2 :         pg_log_error("replication slots can only be used with WAL streaming");
    2595           2 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2596           2 :         exit(1);
    2597             :     }
    2598             : 
    2599             :     /*
    2600             :      * Sanity checks for replication slot options.
    2601             :      */
    2602         280 :     if (no_slot)
    2603             :     {
    2604           6 :         if (replication_slot)
    2605             :         {
    2606           4 :             pg_log_error("--no-slot cannot be used with slot name");
    2607           4 :             pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2608           4 :             exit(1);
    2609             :         }
    2610           2 :         temp_replication_slot = false;
    2611             :     }
    2612             : 
    2613         276 :     if (create_slot)
    2614             :     {
    2615           8 :         if (!replication_slot)
    2616             :         {
    2617           4 :             pg_log_error("%s needs a slot to be specified using --slot",
    2618             :                          "--create-slot");
    2619           4 :             pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2620           4 :             exit(1);
    2621             :         }
    2622             : 
    2623           4 :         if (no_slot)
    2624             :         {
    2625           0 :             pg_log_error("%s and %s are incompatible options",
    2626             :                          "--create-slot", "--no-slot");
    2627           0 :             pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2628           0 :             exit(1);
    2629             :         }
    2630             :     }
    2631             : 
    2632             :     /*
    2633             :      * Sanity checks on WAL directory.
    2634             :      */
    2635         272 :     if (xlog_dir)
    2636             :     {
    2637           2 :         if (backup_target != NULL)
    2638             :         {
    2639           0 :             pg_log_error("WAL directory location cannot be specified along with a backup target");
    2640           0 :             pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2641           0 :             exit(1);
    2642             :         }
    2643           2 :         if (format != 'p')
    2644             :         {
    2645           0 :             pg_log_error("WAL directory location can only be specified in plain mode");
    2646           0 :             pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2647           0 :             exit(1);
    2648             :         }
    2649             : 
    2650             :         /* clean up xlog directory name, check it's absolute */
    2651           2 :         canonicalize_path(xlog_dir);
    2652           2 :         if (!is_absolute_path(xlog_dir))
    2653             :         {
    2654           0 :             pg_log_error("WAL directory location must be an absolute path");
    2655           0 :             pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2656           0 :             exit(1);
    2657             :         }
    2658             :     }
    2659             : 
    2660             :     /*
    2661             :      * Sanity checks for progress reporting options.
    2662             :      */
    2663         272 :     if (showprogress && !estimatesize)
    2664             :     {
    2665           0 :         pg_log_error("%s and %s are incompatible options",
    2666             :                      "--progress", "--no-estimate-size");
    2667           0 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2668           0 :         exit(1);
    2669             :     }
    2670             : 
    2671             :     /*
    2672             :      * Sanity checks for backup manifest options.
    2673             :      */
    2674         272 :     if (!manifest && manifest_checksums != NULL)
    2675             :     {
    2676           0 :         pg_log_error("%s and %s are incompatible options",
    2677             :                      "--no-manifest", "--manifest-checksums");
    2678           0 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2679           0 :         exit(1);
    2680             :     }
    2681             : 
    2682         272 :     if (!manifest && manifest_force_encode)
    2683             :     {
    2684           0 :         pg_log_error("%s and %s are incompatible options",
    2685             :                      "--no-manifest", "--manifest-force-encode");
    2686           0 :         pg_log_error_hint("Try \"%s --help\" for more information.", progname);
    2687           0 :         exit(1);
    2688             :     }
    2689             : 
    2690             :     /* connection in replication mode to server */
    2691         272 :     conn = GetConnection();
    2692         272 :     if (!conn)
    2693             :     {
    2694             :         /* Error message already written in GetConnection() */
    2695           4 :         exit(1);
    2696             :     }
    2697         268 :     atexit(disconnect_atexit);
    2698             : 
    2699             : #ifndef WIN32
    2700             : 
    2701             :     /*
    2702             :      * Trap SIGCHLD to be able to handle the WAL stream process exiting. There
    2703             :      * is no SIGCHLD on Windows, there we rely on the background thread
    2704             :      * setting the signal variable on unexpected but graceful exit. If the WAL
    2705             :      * stream thread crashes on Windows it will bring down the entire process
    2706             :      * as it's a thread, so there is nothing to catch should that happen. A
    2707             :      * crash on UNIX will be caught by the signal handler.
    2708             :      */
    2709         268 :     pqsignal(SIGCHLD, sigchld_handler);
    2710             : #endif
    2711             : 
    2712             :     /*
    2713             :      * Set umask so that directories/files are created with the same
    2714             :      * permissions as directories/files in the source data directory.
    2715             :      *
    2716             :      * pg_mode_mask is set to owner-only by default and then updated in
    2717             :      * GetConnection() where we get the mode from the server-side with
    2718             :      * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
    2719             :      */
    2720         268 :     umask(pg_mode_mask);
    2721             : 
    2722             :     /* Backup manifests are supported in 13 and newer versions */
    2723         268 :     if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_MANIFESTS)
    2724           0 :         manifest = false;
    2725             : 
    2726             :     /*
    2727             :      * If an output directory was specified, verify that it exists, or create
    2728             :      * it. Note that for a tar backup, an output directory of "-" means we are
    2729             :      * writing to stdout, so do nothing in that case.
    2730             :      */
    2731         268 :     if (basedir != NULL && (format == 'p' || strcmp(basedir, "-") != 0))
    2732         244 :         verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata);
    2733             : 
    2734             :     /* determine remote server's xlog segment size */
    2735         268 :     if (!RetrieveWalSegSize(conn))
    2736           0 :         exit(1);
    2737             : 
    2738             :     /* Create pg_wal symlink, if required */
    2739         268 :     if (xlog_dir)
    2740             :     {
    2741             :         char       *linkloc;
    2742             : 
    2743           2 :         verify_dir_is_empty_or_create(xlog_dir, &made_new_xlogdir, &found_existing_xlogdir);
    2744             : 
    2745             :         /*
    2746             :          * Form name of the place where the symlink must go. pg_xlog has been
    2747             :          * renamed to pg_wal in post-10 clusters.
    2748             :          */
    2749           2 :         linkloc = psprintf("%s/%s", basedir,
    2750           2 :                            PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
    2751             :                            "pg_xlog" : "pg_wal");
    2752             : 
    2753           2 :         if (symlink(xlog_dir, linkloc) != 0)
    2754           0 :             pg_fatal("could not create symbolic link \"%s\": %m", linkloc);
    2755           2 :         free(linkloc);
    2756             :     }
    2757             : 
    2758         268 :     BaseBackup(compression_algorithm, compression_detail, compressloc,
    2759             :                &client_compress);
    2760             : 
    2761         218 :     success = true;
    2762         218 :     return 0;
    2763             : }

Generated by: LCOV version 1.14