LCOV - code coverage report
Current view: top level - src/backend/replication - basebackup.c (source / functions) Hit Total Coverage
Test: PostgreSQL 13beta1 Lines: 488 615 79.3 %
Date: 2020-06-03 09:06:53 Functions: 15 16 93.8 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * basebackup.c
       4             :  *    code for taking a base backup and streaming it to a standby
       5             :  *
       6             :  * Portions Copyright (c) 2010-2020, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/replication/basebackup.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : #include "postgres.h"
      14             : 
      15             : #include <sys/stat.h>
      16             : #include <unistd.h>
      17             : #include <time.h>
      18             : 
      19             : #include "access/xlog_internal.h" /* for pg_start/stop_backup */
      20             : #include "catalog/pg_type.h"
      21             : #include "common/file_perm.h"
      22             : #include "commands/progress.h"
      23             : #include "lib/stringinfo.h"
      24             : #include "libpq/libpq.h"
      25             : #include "libpq/pqformat.h"
      26             : #include "miscadmin.h"
      27             : #include "nodes/pg_list.h"
      28             : #include "pgstat.h"
      29             : #include "pgtar.h"
      30             : #include "port.h"
      31             : #include "postmaster/syslogger.h"
      32             : #include "replication/basebackup.h"
      33             : #include "replication/backup_manifest.h"
      34             : #include "replication/walsender.h"
      35             : #include "replication/walsender_private.h"
      36             : #include "storage/bufpage.h"
      37             : #include "storage/checksum.h"
      38             : #include "storage/dsm_impl.h"
      39             : #include "storage/fd.h"
      40             : #include "storage/ipc.h"
      41             : #include "storage/reinit.h"
      42             : #include "utils/builtins.h"
      43             : #include "utils/ps_status.h"
      44             : #include "utils/relcache.h"
      45             : #include "utils/resowner.h"
      46             : #include "utils/timestamp.h"
      47             : 
      48             : typedef struct
      49             : {
      50             :     const char *label;
      51             :     bool        progress;
      52             :     bool        fastcheckpoint;
      53             :     bool        nowait;
      54             :     bool        includewal;
      55             :     uint32      maxrate;
      56             :     bool        sendtblspcmapfile;
      57             :     backup_manifest_option manifest;
      58             :     pg_checksum_type manifest_checksum_type;
      59             : } basebackup_options;
      60             : 
      61             : static int64 sendDir(const char *path, int basepathlen, bool sizeonly,
      62             :                      List *tablespaces, bool sendtblspclinks,
      63             :                      backup_manifest_info *manifest, const char *spcoid);
      64             : static bool sendFile(const char *readfilename, const char *tarfilename,
      65             :                      struct stat *statbuf, bool missing_ok, Oid dboid,
      66             :                      backup_manifest_info *manifest, const char *spcoid);
      67             : static void sendFileWithContent(const char *filename, const char *content,
      68             :                                 backup_manifest_info *manifest);
      69             : static int64 _tarWriteHeader(const char *filename, const char *linktarget,
      70             :                              struct stat *statbuf, bool sizeonly);
      71             : static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
      72             :                           bool sizeonly);
      73             : static void send_int8_string(StringInfoData *buf, int64 intval);
      74             : static void SendBackupHeader(List *tablespaces);
      75             : static void perform_base_backup(basebackup_options *opt);
      76             : static void parse_basebackup_options(List *options, basebackup_options *opt);
      77             : static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli);
      78             : static int  compareWalFileNames(const ListCell *a, const ListCell *b);
      79             : static void throttle(size_t increment);
      80             : static void update_basebackup_progress(int64 delta);
      81             : static bool is_checksummed_file(const char *fullpath, const char *filename);
      82             : 
      83             : /* Was the backup currently in-progress initiated in recovery mode? */
      84             : static bool backup_started_in_recovery = false;
      85             : 
      86             : /* Relative path of temporary statistics directory */
      87             : static char *statrelpath = NULL;
      88             : 
      89             : /*
      90             :  * Size of each block sent into the tar stream for larger files.
      91             :  */
      92             : #define TAR_SEND_SIZE 32768
      93             : 
      94             : /*
      95             :  * How frequently to throttle, as a fraction of the specified rate-second.
      96             :  */
      97             : #define THROTTLING_FREQUENCY    8
      98             : 
      99             : /*
     100             :  * Checks whether we encountered any error in fread().  fread() doesn't give
     101             :  * any clue what has happened, so we check with ferror().  Also, neither
     102             :  * fread() nor ferror() set errno, so we just throw a generic error.
     103             :  */
     104             : #define CHECK_FREAD_ERROR(fp, filename) \
     105             : do { \
     106             :     if (ferror(fp)) \
     107             :         ereport(ERROR, \
     108             :                 (errmsg("could not read from file \"%s\"", filename))); \
     109             : } while (0)
     110             : 
     111             : /* The actual number of bytes, transfer of which may cause sleep. */
     112             : static uint64 throttling_sample;
     113             : 
     114             : /* Amount of data already transferred but not yet throttled.  */
     115             : static int64 throttling_counter;
     116             : 
     117             : /* The minimum time required to transfer throttling_sample bytes. */
     118             : static TimeOffset elapsed_min_unit;
     119             : 
     120             : /* The last check of the transfer rate. */
     121             : static TimestampTz throttled_last;
     122             : 
     123             : /* The starting XLOG position of the base backup. */
     124             : static XLogRecPtr startptr;
     125             : 
     126             : /* Total number of checksum failures during base backup. */
     127             : static long long int total_checksum_failures;
     128             : 
     129             : /* Do not verify checksums. */
     130             : static bool noverify_checksums = false;
     131             : 
     132             : /*
     133             :  * Total amount of backup data that will be streamed.
     134             :  * -1 means that the size is not estimated.
     135             :  */
     136             : static int64 backup_total = 0;
     137             : 
     138             : /* Amount of backup data already streamed */
     139             : static int64 backup_streamed = 0;
     140             : 
     141             : /*
     142             :  * Definition of one element part of an exclusion list, used for paths part
     143             :  * of checksum validation or base backups.  "name" is the name of the file
     144             :  * or path to check for exclusion.  If "match_prefix" is true, any items
     145             :  * matching the name as prefix are excluded.
     146             :  */
     147             : struct exclude_list_item
     148             : {
     149             :     const char *name;
     150             :     bool        match_prefix;
     151             : };
     152             : 
     153             : /*
     154             :  * The contents of these directories are removed or recreated during server
     155             :  * start so they are not included in backups.  The directories themselves are
     156             :  * kept and included as empty to preserve access permissions.
     157             :  *
     158             :  * Note: this list should be kept in sync with the filter lists in pg_rewind's
     159             :  * filemap.c.
     160             :  */
     161             : static const char *const excludeDirContents[] =
     162             : {
     163             :     /*
     164             :      * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even
     165             :      * when stats_temp_directory is set because PGSS_TEXT_FILE is always
     166             :      * created there.
     167             :      */
     168             :     PG_STAT_TMP_DIR,
     169             : 
     170             :     /*
     171             :      * It is generally not useful to backup the contents of this directory
     172             :      * even if the intention is to restore to another master. See backup.sgml
     173             :      * for a more detailed description.
     174             :      */
     175             :     "pg_replslot",
     176             : 
     177             :     /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
     178             :     PG_DYNSHMEM_DIR,
     179             : 
     180             :     /* Contents removed on startup, see AsyncShmemInit(). */
     181             :     "pg_notify",
     182             : 
     183             :     /*
     184             :      * Old contents are loaded for possible debugging but are not required for
     185             :      * normal operation, see SerialInit().
     186             :      */
     187             :     "pg_serial",
     188             : 
     189             :     /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
     190             :     "pg_snapshots",
     191             : 
     192             :     /* Contents zeroed on startup, see StartupSUBTRANS(). */
     193             :     "pg_subtrans",
     194             : 
     195             :     /* end of list */
     196             :     NULL
     197             : };
     198             : 
     199             : /*
     200             :  * List of files excluded from backups.
     201             :  */
     202             : static const struct exclude_list_item excludeFiles[] =
     203             : {
     204             :     /* Skip auto conf temporary file. */
     205             :     {PG_AUTOCONF_FILENAME ".tmp", false},
     206             : 
     207             :     /* Skip current log file temporary file */
     208             :     {LOG_METAINFO_DATAFILE_TMP, false},
     209             : 
     210             :     /*
     211             :      * Skip relation cache because it is rebuilt on startup.  This includes
     212             :      * temporary files.
     213             :      */
     214             :     {RELCACHE_INIT_FILENAME, true},
     215             : 
     216             :     /*
     217             :      * If there's a backup_label or tablespace_map file, it belongs to a
     218             :      * backup started by the user with pg_start_backup().  It is *not* correct
     219             :      * for this backup.  Our backup_label/tablespace_map is injected into the
     220             :      * tar separately.
     221             :      */
     222             :     {BACKUP_LABEL_FILE, false},
     223             :     {TABLESPACE_MAP, false},
     224             : 
     225             :     /*
     226             :      * If there's a backup_manifest, it belongs to a backup that was used to
     227             :      * start this server. It is *not* correct for this backup. Our
     228             :      * backup_manifest is injected into the backup separately if users want
     229             :      * it.
     230             :      */
     231             :     {"backup_manifest", false},
     232             : 
     233             :     {"postmaster.pid", false},
     234             :     {"postmaster.opts", false},
     235             : 
     236             :     /* end of list */
     237             :     {NULL, false}
     238             : };
     239             : 
     240             : /*
     241             :  * List of files excluded from checksum validation.
     242             :  *
     243             :  * Note: this list should be kept in sync with what pg_checksums.c
     244             :  * includes.
     245             :  */
     246             : static const struct exclude_list_item noChecksumFiles[] = {
     247             :     {"pg_control", false},
     248             :     {"pg_filenode.map", false},
     249             :     {"pg_internal.init", true},
     250             :     {"PG_VERSION", false},
     251             : #ifdef EXEC_BACKEND
     252             :     {"config_exec_params", true},
     253             : #endif
     254             :     {NULL, false}
     255             : };
     256             : 
     257             : /*
     258             :  * Actually do a base backup for the specified tablespaces.
     259             :  *
     260             :  * This is split out mainly to avoid complaints about "variable might be
     261             :  * clobbered by longjmp" from stupider versions of gcc.
     262             :  */
     263             : static void
     264         144 : perform_base_backup(basebackup_options *opt)
     265             : {
     266             :     TimeLineID  starttli;
     267             :     XLogRecPtr  endptr;
     268             :     TimeLineID  endtli;
     269             :     StringInfo  labelfile;
     270         144 :     StringInfo  tblspc_map_file = NULL;
     271             :     backup_manifest_info manifest;
     272             :     int         datadirpathlen;
     273         144 :     List       *tablespaces = NIL;
     274             : 
     275         144 :     backup_total = 0;
     276         144 :     backup_streamed = 0;
     277         144 :     pgstat_progress_start_command(PROGRESS_COMMAND_BASEBACKUP, InvalidOid);
     278             : 
     279             :     /*
     280             :      * If the estimation of the total backup size is disabled, make the
     281             :      * backup_total column in the view return NULL by setting the parameter to
     282             :      * -1.
     283             :      */
     284         144 :     if (!opt->progress)
     285             :     {
     286           0 :         backup_total = -1;
     287           0 :         pgstat_progress_update_param(PROGRESS_BASEBACKUP_BACKUP_TOTAL,
     288             :                                      backup_total);
     289             :     }
     290             : 
     291             :     /* we're going to use a BufFile, so we need a ResourceOwner */
     292             :     Assert(CurrentResourceOwner == NULL);
     293         144 :     CurrentResourceOwner = ResourceOwnerCreate(NULL, "base backup");
     294             : 
     295         144 :     datadirpathlen = strlen(DataDir);
     296             : 
     297         144 :     backup_started_in_recovery = RecoveryInProgress();
     298             : 
     299         144 :     labelfile = makeStringInfo();
     300         144 :     tblspc_map_file = makeStringInfo();
     301         144 :     InitializeBackupManifest(&manifest, opt->manifest,
     302             :                              opt->manifest_checksum_type);
     303             : 
     304         144 :     total_checksum_failures = 0;
     305             : 
     306         144 :     pgstat_progress_update_param(PROGRESS_BASEBACKUP_PHASE,
     307             :                                  PROGRESS_BASEBACKUP_PHASE_WAIT_CHECKPOINT);
     308         432 :     startptr = do_pg_start_backup(opt->label, opt->fastcheckpoint, &starttli,
     309             :                                   labelfile, &tablespaces,
     310             :                                   tblspc_map_file,
     311         144 :                                   opt->progress, opt->sendtblspcmapfile);
     312             : 
     313             :     /*
     314             :      * Once do_pg_start_backup has been called, ensure that any failure causes
     315             :      * us to abort the backup so we don't "leak" a backup counter. For this
     316             :      * reason, *all* functionality between do_pg_start_backup() and the end of
     317             :      * do_pg_stop_backup() should be inside the error cleanup block!
     318             :      */
     319             : 
     320         144 :     PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
     321             :     {
     322             :         ListCell   *lc;
     323             :         tablespaceinfo *ti;
     324         144 :         int         tblspc_streamed = 0;
     325             : 
     326             :         /*
     327             :          * Calculate the relative path of temporary statistics directory in
     328             :          * order to skip the files which are located in that directory later.
     329             :          */
     330         144 :         if (is_absolute_path(pgstat_stat_directory) &&
     331           0 :             strncmp(pgstat_stat_directory, DataDir, datadirpathlen) == 0)
     332           0 :             statrelpath = psprintf("./%s", pgstat_stat_directory + datadirpathlen + 1);
     333         144 :         else if (strncmp(pgstat_stat_directory, "./", 2) != 0)
     334         144 :             statrelpath = psprintf("./%s", pgstat_stat_directory);
     335             :         else
     336           0 :             statrelpath = pgstat_stat_directory;
     337             : 
     338             :         /* Add a node for the base directory at the end */
     339         144 :         ti = palloc0(sizeof(tablespaceinfo));
     340         144 :         if (opt->progress)
     341         144 :             ti->size = sendDir(".", 1, true, tablespaces, true, NULL, NULL);
     342             :         else
     343           0 :             ti->size = -1;
     344         144 :         tablespaces = lappend(tablespaces, ti);
     345             : 
     346             :         /*
     347             :          * Calculate the total backup size by summing up the size of each
     348             :          * tablespace
     349             :          */
     350         144 :         if (opt->progress)
     351             :         {
     352         320 :             foreach(lc, tablespaces)
     353             :             {
     354         176 :                 tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
     355             : 
     356         176 :                 backup_total += tmp->size;
     357             :             }
     358             :         }
     359             : 
     360             :         /* Report that we are now streaming database files as a base backup */
     361             :         {
     362         144 :             const int   index[] = {
     363             :                 PROGRESS_BASEBACKUP_PHASE,
     364             :                 PROGRESS_BASEBACKUP_BACKUP_TOTAL,
     365             :                 PROGRESS_BASEBACKUP_TBLSPC_TOTAL
     366             :             };
     367         288 :             const int64 val[] = {
     368             :                 PROGRESS_BASEBACKUP_PHASE_STREAM_BACKUP,
     369         144 :                 backup_total, list_length(tablespaces)
     370             :             };
     371             : 
     372         144 :             pgstat_progress_update_multi_param(3, index, val);
     373             :         }
     374             : 
     375             :         /* Send the starting position of the backup */
     376         144 :         SendXlogRecPtrResult(startptr, starttli);
     377             : 
     378             :         /* Send tablespace header */
     379         144 :         SendBackupHeader(tablespaces);
     380             : 
     381             :         /* Setup and activate network throttling, if client requested it */
     382         144 :         if (opt->maxrate > 0)
     383             :         {
     384           0 :             throttling_sample =
     385           0 :                 (int64) opt->maxrate * (int64) 1024 / THROTTLING_FREQUENCY;
     386             : 
     387             :             /*
     388             :              * The minimum amount of time for throttling_sample bytes to be
     389             :              * transferred.
     390             :              */
     391           0 :             elapsed_min_unit = USECS_PER_SEC / THROTTLING_FREQUENCY;
     392             : 
     393             :             /* Enable throttling. */
     394           0 :             throttling_counter = 0;
     395             : 
     396             :             /* The 'real data' starts now (header was ignored). */
     397           0 :             throttled_last = GetCurrentTimestamp();
     398             :         }
     399             :         else
     400             :         {
     401             :             /* Disable throttling. */
     402         144 :             throttling_counter = -1;
     403             :         }
     404             : 
     405             :         /* Send off our tablespaces one by one */
     406         314 :         foreach(lc, tablespaces)
     407             :         {
     408         176 :             tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
     409             :             StringInfoData buf;
     410             : 
     411             :             /* Send CopyOutResponse message */
     412         176 :             pq_beginmessage(&buf, 'H');
     413         176 :             pq_sendbyte(&buf, 0);   /* overall format */
     414         176 :             pq_sendint16(&buf, 0);  /* natts */
     415         176 :             pq_endmessage(&buf);
     416             : 
     417         176 :             if (ti->path == NULL)
     418             :             {
     419             :                 struct stat statbuf;
     420             : 
     421             :                 /* In the main tar, include the backup_label first... */
     422         144 :                 sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data,
     423             :                                     &manifest);
     424             : 
     425             :                 /*
     426             :                  * Send tablespace_map file if required and then the bulk of
     427             :                  * the files.
     428             :                  */
     429         144 :                 if (tblspc_map_file && opt->sendtblspcmapfile)
     430             :                 {
     431          10 :                     sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data,
     432             :                                         &manifest);
     433          10 :                     sendDir(".", 1, false, tablespaces, false,
     434             :                             &manifest, NULL);
     435             :                 }
     436             :                 else
     437         134 :                     sendDir(".", 1, false, tablespaces, true,
     438             :                             &manifest, NULL);
     439             : 
     440             :                 /* ... and pg_control after everything else. */
     441         138 :                 if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
     442           0 :                     ereport(ERROR,
     443             :                             (errcode_for_file_access(),
     444             :                              errmsg("could not stat file \"%s\": %m",
     445             :                                     XLOG_CONTROL_FILE)));
     446         138 :                 sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
     447             :                          false, InvalidOid, &manifest, NULL);
     448             :             }
     449             :             else
     450          32 :                 sendTablespace(ti->path, ti->oid, false, &manifest);
     451             : 
     452             :             /*
     453             :              * If we're including WAL, and this is the main data directory we
     454             :              * don't terminate the tar stream here. Instead, we will append
     455             :              * the xlog files below and terminate it then. This is safe since
     456             :              * the main data directory is always sent *last*.
     457             :              */
     458         170 :             if (opt->includewal && ti->path == NULL)
     459             :             {
     460             :                 Assert(lnext(tablespaces, lc) == NULL);
     461             :             }
     462             :             else
     463         168 :                 pq_putemptymessage('c');    /* CopyDone */
     464             : 
     465         170 :             tblspc_streamed++;
     466         170 :             pgstat_progress_update_param(PROGRESS_BASEBACKUP_TBLSPC_STREAMED,
     467             :                                          tblspc_streamed);
     468             :         }
     469             : 
     470         138 :         pgstat_progress_update_param(PROGRESS_BASEBACKUP_PHASE,
     471             :                                      PROGRESS_BASEBACKUP_PHASE_WAIT_WAL_ARCHIVE);
     472         138 :         endptr = do_pg_stop_backup(labelfile->data, !opt->nowait, &endtli);
     473             :     }
     474         282 :     PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
     475             : 
     476             : 
     477         138 :     if (opt->includewal)
     478             :     {
     479             :         /*
     480             :          * We've left the last tar file "open", so we can now append the
     481             :          * required WAL files to it.
     482             :          */
     483             :         char        pathbuf[MAXPGPATH];
     484             :         XLogSegNo   segno;
     485             :         XLogSegNo   startsegno;
     486             :         XLogSegNo   endsegno;
     487             :         struct stat statbuf;
     488           2 :         List       *historyFileList = NIL;
     489           2 :         List       *walFileList = NIL;
     490             :         char        firstoff[MAXFNAMELEN];
     491             :         char        lastoff[MAXFNAMELEN];
     492             :         DIR        *dir;
     493             :         struct dirent *de;
     494             :         ListCell   *lc;
     495             :         TimeLineID  tli;
     496             : 
     497           2 :         pgstat_progress_update_param(PROGRESS_BASEBACKUP_PHASE,
     498             :                                      PROGRESS_BASEBACKUP_PHASE_TRANSFER_WAL);
     499             : 
     500             :         /*
     501             :          * I'd rather not worry about timelines here, so scan pg_wal and
     502             :          * include all WAL files in the range between 'startptr' and 'endptr',
     503             :          * regardless of the timeline the file is stamped with. If there are
     504             :          * some spurious WAL files belonging to timelines that don't belong in
     505             :          * this server's history, they will be included too. Normally there
     506             :          * shouldn't be such files, but if there are, there's little harm in
     507             :          * including them.
     508             :          */
     509           2 :         XLByteToSeg(startptr, startsegno, wal_segment_size);
     510           2 :         XLogFileName(firstoff, ThisTimeLineID, startsegno, wal_segment_size);
     511           2 :         XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
     512           2 :         XLogFileName(lastoff, ThisTimeLineID, endsegno, wal_segment_size);
     513             : 
     514           2 :         dir = AllocateDir("pg_wal");
     515          14 :         while ((de = ReadDir(dir, "pg_wal")) != NULL)
     516             :         {
     517             :             /* Does it look like a WAL segment, and is it in the range? */
     518          12 :             if (IsXLogFileName(de->d_name) &&
     519           6 :                 strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
     520           6 :                 strcmp(de->d_name + 8, lastoff + 8) <= 0)
     521             :             {
     522           2 :                 walFileList = lappend(walFileList, pstrdup(de->d_name));
     523             :             }
     524             :             /* Does it look like a timeline history file? */
     525          10 :             else if (IsTLHistoryFileName(de->d_name))
     526             :             {
     527           0 :                 historyFileList = lappend(historyFileList, pstrdup(de->d_name));
     528             :             }
     529             :         }
     530           2 :         FreeDir(dir);
     531             : 
     532             :         /*
     533             :          * Before we go any further, check that none of the WAL segments we
     534             :          * need were removed.
     535             :          */
     536           2 :         CheckXLogRemoved(startsegno, ThisTimeLineID);
     537             : 
     538             :         /*
     539             :          * Sort the WAL filenames.  We want to send the files in order from
     540             :          * oldest to newest, to reduce the chance that a file is recycled
     541             :          * before we get a chance to send it over.
     542             :          */
     543           2 :         list_sort(walFileList, compareWalFileNames);
     544             : 
     545             :         /*
     546             :          * There must be at least one xlog file in the pg_wal directory, since
     547             :          * we are doing backup-including-xlog.
     548             :          */
     549           2 :         if (walFileList == NIL)
     550           0 :             ereport(ERROR,
     551             :                     (errmsg("could not find any WAL files")));
     552             : 
     553             :         /*
     554             :          * Sanity check: the first and last segment should cover startptr and
     555             :          * endptr, with no gaps in between.
     556             :          */
     557           2 :         XLogFromFileName((char *) linitial(walFileList),
     558             :                          &tli, &segno, wal_segment_size);
     559           2 :         if (segno != startsegno)
     560             :         {
     561             :             char        startfname[MAXFNAMELEN];
     562             : 
     563           0 :             XLogFileName(startfname, ThisTimeLineID, startsegno,
     564             :                          wal_segment_size);
     565           0 :             ereport(ERROR,
     566             :                     (errmsg("could not find WAL file \"%s\"", startfname)));
     567             :         }
     568           4 :         foreach(lc, walFileList)
     569             :         {
     570           2 :             char       *walFileName = (char *) lfirst(lc);
     571           2 :             XLogSegNo   currsegno = segno;
     572           2 :             XLogSegNo   nextsegno = segno + 1;
     573             : 
     574           2 :             XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
     575           2 :             if (!(nextsegno == segno || currsegno == segno))
     576             :             {
     577             :                 char        nextfname[MAXFNAMELEN];
     578             : 
     579           0 :                 XLogFileName(nextfname, ThisTimeLineID, nextsegno,
     580             :                              wal_segment_size);
     581           0 :                 ereport(ERROR,
     582             :                         (errmsg("could not find WAL file \"%s\"", nextfname)));
     583             :             }
     584             :         }
     585           2 :         if (segno != endsegno)
     586             :         {
     587             :             char        endfname[MAXFNAMELEN];
     588             : 
     589           0 :             XLogFileName(endfname, ThisTimeLineID, endsegno, wal_segment_size);
     590           0 :             ereport(ERROR,
     591             :                     (errmsg("could not find WAL file \"%s\"", endfname)));
     592             :         }
     593             : 
     594             :         /* Ok, we have everything we need. Send the WAL files. */
     595           4 :         foreach(lc, walFileList)
     596             :         {
     597           2 :             char       *walFileName = (char *) lfirst(lc);
     598             :             FILE       *fp;
     599             :             char        buf[TAR_SEND_SIZE];
     600             :             size_t      cnt;
     601           2 :             pgoff_t     len = 0;
     602             : 
     603           2 :             snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
     604           2 :             XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
     605             : 
     606           2 :             fp = AllocateFile(pathbuf, "rb");
     607           2 :             if (fp == NULL)
     608             :             {
     609           0 :                 int         save_errno = errno;
     610             : 
     611             :                 /*
     612             :                  * Most likely reason for this is that the file was already
     613             :                  * removed by a checkpoint, so check for that to get a better
     614             :                  * error message.
     615             :                  */
     616           0 :                 CheckXLogRemoved(segno, tli);
     617             : 
     618           0 :                 errno = save_errno;
     619           0 :                 ereport(ERROR,
     620             :                         (errcode_for_file_access(),
     621             :                          errmsg("could not open file \"%s\": %m", pathbuf)));
     622             :             }
     623             : 
     624           2 :             if (fstat(fileno(fp), &statbuf) != 0)
     625           0 :                 ereport(ERROR,
     626             :                         (errcode_for_file_access(),
     627             :                          errmsg("could not stat file \"%s\": %m",
     628             :                                 pathbuf)));
     629           2 :             if (statbuf.st_size != wal_segment_size)
     630             :             {
     631           0 :                 CheckXLogRemoved(segno, tli);
     632           0 :                 ereport(ERROR,
     633             :                         (errcode_for_file_access(),
     634             :                          errmsg("unexpected WAL file size \"%s\"", walFileName)));
     635             :             }
     636             : 
     637             :             /* send the WAL file itself */
     638           2 :             _tarWriteHeader(pathbuf, NULL, &statbuf, false);
     639             : 
     640        1026 :             while ((cnt = fread(buf, 1,
     641        1024 :                                 Min(sizeof(buf), wal_segment_size - len),
     642             :                                 fp)) > 0)
     643             :             {
     644        1024 :                 CheckXLogRemoved(segno, tli);
     645             :                 /* Send the chunk as a CopyData message */
     646        1024 :                 if (pq_putmessage('d', buf, cnt))
     647           0 :                     ereport(ERROR,
     648             :                             (errmsg("base backup could not send data, aborting backup")));
     649        1024 :                 update_basebackup_progress(cnt);
     650             : 
     651        1024 :                 len += cnt;
     652        1024 :                 throttle(cnt);
     653             : 
     654        1024 :                 if (len == wal_segment_size)
     655           2 :                     break;
     656             :             }
     657             : 
     658           2 :             CHECK_FREAD_ERROR(fp, pathbuf);
     659             : 
     660           2 :             if (len != wal_segment_size)
     661             :             {
     662           0 :                 CheckXLogRemoved(segno, tli);
     663           0 :                 ereport(ERROR,
     664             :                         (errcode_for_file_access(),
     665             :                          errmsg("unexpected WAL file size \"%s\"", walFileName)));
     666             :             }
     667             : 
     668             :             /* wal_segment_size is a multiple of 512, so no need for padding */
     669             : 
     670           2 :             FreeFile(fp);
     671             : 
     672             :             /*
     673             :              * Mark file as archived, otherwise files can get archived again
     674             :              * after promotion of a new node. This is in line with
     675             :              * walreceiver.c always doing an XLogArchiveForceDone() after a
     676             :              * complete segment.
     677             :              */
     678           2 :             StatusFilePath(pathbuf, walFileName, ".done");
     679           2 :             sendFileWithContent(pathbuf, "", &manifest);
     680             :         }
     681             : 
     682             :         /*
     683             :          * Send timeline history files too. Only the latest timeline history
     684             :          * file is required for recovery, and even that only if there happens
     685             :          * to be a timeline switch in the first WAL segment that contains the
     686             :          * checkpoint record, or if we're taking a base backup from a standby
     687             :          * server and the target timeline changes while the backup is taken.
     688             :          * But they are small and highly useful for debugging purposes, so
     689             :          * better include them all, always.
     690             :          */
     691           2 :         foreach(lc, historyFileList)
     692             :         {
     693           0 :             char       *fname = lfirst(lc);
     694             : 
     695           0 :             snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
     696             : 
     697           0 :             if (lstat(pathbuf, &statbuf) != 0)
     698           0 :                 ereport(ERROR,
     699             :                         (errcode_for_file_access(),
     700             :                          errmsg("could not stat file \"%s\": %m", pathbuf)));
     701             : 
     702           0 :             sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid,
     703             :                      &manifest, NULL);
     704             : 
     705             :             /* unconditionally mark file as archived */
     706           0 :             StatusFilePath(pathbuf, fname, ".done");
     707           0 :             sendFileWithContent(pathbuf, "", &manifest);
     708             :         }
     709             : 
     710             :         /* Send CopyDone message for the last tar file */
     711           2 :         pq_putemptymessage('c');
     712             :     }
     713             : 
     714         138 :     AddWALInfoToBackupManifest(&manifest, startptr, starttli, endptr, endtli);
     715             : 
     716         138 :     SendBackupManifest(&manifest);
     717             : 
     718         138 :     SendXlogRecPtrResult(endptr, endtli);
     719             : 
     720         138 :     if (total_checksum_failures)
     721             :     {
     722           6 :         if (total_checksum_failures > 1)
     723           4 :             ereport(WARNING,
     724             :                     (errmsg("%lld total checksum verification failures", total_checksum_failures)));
     725             : 
     726           6 :         ereport(ERROR,
     727             :                 (errcode(ERRCODE_DATA_CORRUPTED),
     728             :                  errmsg("checksum verification failure during base backup")));
     729             :     }
     730             : 
     731             :     /* clean up the resource owner we created */
     732         132 :     WalSndResourceCleanup(true);
     733             : 
     734         132 :     pgstat_progress_end_command();
     735         132 : }
     736             : 
     737             : /*
     738             :  * list_sort comparison function, to compare log/seg portion of WAL segment
     739             :  * filenames, ignoring the timeline portion.
     740             :  */
     741             : static int
     742           0 : compareWalFileNames(const ListCell *a, const ListCell *b)
     743             : {
     744           0 :     char       *fna = (char *) lfirst(a);
     745           0 :     char       *fnb = (char *) lfirst(b);
     746             : 
     747           0 :     return strcmp(fna + 8, fnb + 8);
     748             : }
     749             : 
     750             : /*
     751             :  * Parse the base backup options passed down by the parser
     752             :  */
     753             : static void
     754         146 : parse_basebackup_options(List *options, basebackup_options *opt)
     755             : {
     756             :     ListCell   *lopt;
     757         146 :     bool        o_label = false;
     758         146 :     bool        o_progress = false;
     759         146 :     bool        o_fast = false;
     760         146 :     bool        o_nowait = false;
     761         146 :     bool        o_wal = false;
     762         146 :     bool        o_maxrate = false;
     763         146 :     bool        o_tablespace_map = false;
     764         146 :     bool        o_noverify_checksums = false;
     765         146 :     bool        o_manifest = false;
     766         146 :     bool        o_manifest_checksums = false;
     767             : 
     768         730 :     MemSet(opt, 0, sizeof(*opt));
     769         146 :     opt->manifest = MANIFEST_OPTION_NO;
     770         146 :     opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
     771             : 
     772         752 :     foreach(lopt, options)
     773             :     {
     774         608 :         DefElem    *defel = (DefElem *) lfirst(lopt);
     775             : 
     776         608 :         if (strcmp(defel->defname, "label") == 0)
     777             :         {
     778         146 :             if (o_label)
     779           0 :                 ereport(ERROR,
     780             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     781             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     782         146 :             opt->label = strVal(defel->arg);
     783         146 :             o_label = true;
     784             :         }
     785         462 :         else if (strcmp(defel->defname, "progress") == 0)
     786             :         {
     787         146 :             if (o_progress)
     788           0 :                 ereport(ERROR,
     789             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     790             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     791         146 :             opt->progress = true;
     792         146 :             o_progress = true;
     793             :         }
     794         316 :         else if (strcmp(defel->defname, "fast") == 0)
     795             :         {
     796           0 :             if (o_fast)
     797           0 :                 ereport(ERROR,
     798             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     799             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     800           0 :             opt->fastcheckpoint = true;
     801           0 :             o_fast = true;
     802             :         }
     803         316 :         else if (strcmp(defel->defname, "nowait") == 0)
     804             :         {
     805         144 :             if (o_nowait)
     806           0 :                 ereport(ERROR,
     807             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     808             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     809         144 :             opt->nowait = true;
     810         144 :             o_nowait = true;
     811             :         }
     812         172 :         else if (strcmp(defel->defname, "wal") == 0)
     813             :         {
     814           2 :             if (o_wal)
     815           0 :                 ereport(ERROR,
     816             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     817             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     818           2 :             opt->includewal = true;
     819           2 :             o_wal = true;
     820             :         }
     821         170 :         else if (strcmp(defel->defname, "max_rate") == 0)
     822             :         {
     823             :             long        maxrate;
     824             : 
     825           0 :             if (o_maxrate)
     826           0 :                 ereport(ERROR,
     827             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     828             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     829             : 
     830           0 :             maxrate = intVal(defel->arg);
     831           0 :             if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
     832           0 :                 ereport(ERROR,
     833             :                         (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
     834             :                          errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
     835             :                                 (int) maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
     836             : 
     837           0 :             opt->maxrate = (uint32) maxrate;
     838           0 :             o_maxrate = true;
     839             :         }
     840         170 :         else if (strcmp(defel->defname, "tablespace_map") == 0)
     841             :         {
     842          10 :             if (o_tablespace_map)
     843           0 :                 ereport(ERROR,
     844             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     845             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     846          10 :             opt->sendtblspcmapfile = true;
     847          10 :             o_tablespace_map = true;
     848             :         }
     849         160 :         else if (strcmp(defel->defname, "noverify_checksums") == 0)
     850             :         {
     851           2 :             if (o_noverify_checksums)
     852           0 :                 ereport(ERROR,
     853             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     854             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     855           2 :             noverify_checksums = true;
     856           2 :             o_noverify_checksums = true;
     857             :         }
     858         158 :         else if (strcmp(defel->defname, "manifest") == 0)
     859             :         {
     860         144 :             char       *optval = strVal(defel->arg);
     861             :             bool        manifest_bool;
     862             : 
     863         144 :             if (o_manifest)
     864           0 :                 ereport(ERROR,
     865             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     866             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     867         144 :             if (parse_bool(optval, &manifest_bool))
     868             :             {
     869         142 :                 if (manifest_bool)
     870         142 :                     opt->manifest = MANIFEST_OPTION_YES;
     871             :                 else
     872           0 :                     opt->manifest = MANIFEST_OPTION_NO;
     873             :             }
     874           2 :             else if (pg_strcasecmp(optval, "force-encode") == 0)
     875           2 :                 opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
     876             :             else
     877           0 :                 ereport(ERROR,
     878             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     879             :                          errmsg("unrecognized manifest option: \"%s\"",
     880             :                                 optval)));
     881         144 :             o_manifest = true;
     882             :         }
     883          14 :         else if (strcmp(defel->defname, "manifest_checksums") == 0)
     884             :         {
     885          14 :             char       *optval = strVal(defel->arg);
     886             : 
     887          14 :             if (o_manifest_checksums)
     888           0 :                 ereport(ERROR,
     889             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     890             :                          errmsg("duplicate option \"%s\"", defel->defname)));
     891          14 :             if (!pg_checksum_parse_type(optval,
     892             :                                         &opt->manifest_checksum_type))
     893           2 :                 ereport(ERROR,
     894             :                         (errcode(ERRCODE_SYNTAX_ERROR),
     895             :                          errmsg("unrecognized checksum algorithm: \"%s\"",
     896             :                                 optval)));
     897          12 :             o_manifest_checksums = true;
     898             :         }
     899             :         else
     900           0 :             elog(ERROR, "option \"%s\" not recognized",
     901             :                  defel->defname);
     902             :     }
     903         144 :     if (opt->label == NULL)
     904           0 :         opt->label = "base backup";
     905         144 :     if (opt->manifest == MANIFEST_OPTION_NO)
     906             :     {
     907           2 :         if (o_manifest_checksums)
     908           0 :             ereport(ERROR,
     909             :                     (errcode(ERRCODE_SYNTAX_ERROR),
     910             :                      errmsg("manifest checksums require a backup manifest")));
     911           2 :         opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
     912             :     }
     913         144 : }
     914             : 
     915             : 
     916             : /*
     917             :  * SendBaseBackup() - send a complete base backup.
     918             :  *
     919             :  * The function will put the system into backup mode like pg_start_backup()
     920             :  * does, so that the backup is consistent even though we read directly from
     921             :  * the filesystem, bypassing the buffer cache.
     922             :  */
     923             : void
     924         146 : SendBaseBackup(BaseBackupCmd *cmd)
     925             : {
     926             :     basebackup_options opt;
     927             : 
     928         146 :     parse_basebackup_options(cmd->options, &opt);
     929             : 
     930         144 :     WalSndSetState(WALSNDSTATE_BACKUP);
     931             : 
     932         144 :     if (update_process_title)
     933             :     {
     934             :         char        activitymsg[50];
     935             : 
     936         144 :         snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
     937             :                  opt.label);
     938         144 :         set_ps_display(activitymsg);
     939             :     }
     940             : 
     941         144 :     perform_base_backup(&opt);
     942         132 : }
     943             : 
     944             : static void
     945         176 : send_int8_string(StringInfoData *buf, int64 intval)
     946             : {
     947             :     char        is[32];
     948             : 
     949         176 :     sprintf(is, INT64_FORMAT, intval);
     950         176 :     pq_sendint32(buf, strlen(is));
     951         176 :     pq_sendbytes(buf, is, strlen(is));
     952         176 : }
     953             : 
     954             : static void
     955         144 : SendBackupHeader(List *tablespaces)
     956             : {
     957             :     StringInfoData buf;
     958             :     ListCell   *lc;
     959             : 
     960             :     /* Construct and send the directory information */
     961         144 :     pq_beginmessage(&buf, 'T'); /* RowDescription */
     962         144 :     pq_sendint16(&buf, 3);      /* 3 fields */
     963             : 
     964             :     /* First field - spcoid */
     965         144 :     pq_sendstring(&buf, "spcoid");
     966         144 :     pq_sendint32(&buf, 0);      /* table oid */
     967         144 :     pq_sendint16(&buf, 0);      /* attnum */
     968         144 :     pq_sendint32(&buf, OIDOID); /* type oid */
     969         144 :     pq_sendint16(&buf, 4);      /* typlen */
     970         144 :     pq_sendint32(&buf, 0);      /* typmod */
     971         144 :     pq_sendint16(&buf, 0);      /* format code */
     972             : 
     973             :     /* Second field - spclocation */
     974         144 :     pq_sendstring(&buf, "spclocation");
     975         144 :     pq_sendint32(&buf, 0);
     976         144 :     pq_sendint16(&buf, 0);
     977         144 :     pq_sendint32(&buf, TEXTOID);
     978         144 :     pq_sendint16(&buf, -1);
     979         144 :     pq_sendint32(&buf, 0);
     980         144 :     pq_sendint16(&buf, 0);
     981             : 
     982             :     /* Third field - size */
     983         144 :     pq_sendstring(&buf, "size");
     984         144 :     pq_sendint32(&buf, 0);
     985         144 :     pq_sendint16(&buf, 0);
     986         144 :     pq_sendint32(&buf, INT8OID);
     987         144 :     pq_sendint16(&buf, 8);
     988         144 :     pq_sendint32(&buf, 0);
     989         144 :     pq_sendint16(&buf, 0);
     990         144 :     pq_endmessage(&buf);
     991             : 
     992         320 :     foreach(lc, tablespaces)
     993             :     {
     994         176 :         tablespaceinfo *ti = lfirst(lc);
     995             : 
     996             :         /* Send one datarow message */
     997         176 :         pq_beginmessage(&buf, 'D');
     998         176 :         pq_sendint16(&buf, 3);  /* number of columns */
     999         176 :         if (ti->path == NULL)
    1000             :         {
    1001         144 :             pq_sendint32(&buf, -1); /* Length = -1 ==> NULL */
    1002         144 :             pq_sendint32(&buf, -1);
    1003             :         }
    1004             :         else
    1005             :         {
    1006             :             Size        len;
    1007             : 
    1008          32 :             len = strlen(ti->oid);
    1009          32 :             pq_sendint32(&buf, len);
    1010          32 :             pq_sendbytes(&buf, ti->oid, len);
    1011             : 
    1012          32 :             len = strlen(ti->path);
    1013          32 :             pq_sendint32(&buf, len);
    1014          32 :             pq_sendbytes(&buf, ti->path, len);
    1015             :         }
    1016         176 :         if (ti->size >= 0)
    1017         176 :             send_int8_string(&buf, ti->size / 1024);
    1018             :         else
    1019           0 :             pq_sendint32(&buf, -1); /* NULL */
    1020             : 
    1021         176 :         pq_endmessage(&buf);
    1022             :     }
    1023             : 
    1024             :     /* Send a CommandComplete message */
    1025         144 :     pq_puttextmessage('C', "SELECT");
    1026         144 : }
    1027             : 
    1028             : /*
    1029             :  * Send a single resultset containing just a single
    1030             :  * XLogRecPtr record (in text format)
    1031             :  */
    1032             : static void
    1033         282 : SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli)
    1034             : {
    1035             :     StringInfoData buf;
    1036             :     char        str[MAXFNAMELEN];
    1037             :     Size        len;
    1038             : 
    1039         282 :     pq_beginmessage(&buf, 'T'); /* RowDescription */
    1040         282 :     pq_sendint16(&buf, 2);      /* 2 fields */
    1041             : 
    1042             :     /* Field headers */
    1043         282 :     pq_sendstring(&buf, "recptr");
    1044         282 :     pq_sendint32(&buf, 0);      /* table oid */
    1045         282 :     pq_sendint16(&buf, 0);      /* attnum */
    1046         282 :     pq_sendint32(&buf, TEXTOID);    /* type oid */
    1047         282 :     pq_sendint16(&buf, -1);
    1048         282 :     pq_sendint32(&buf, 0);
    1049         282 :     pq_sendint16(&buf, 0);
    1050             : 
    1051         282 :     pq_sendstring(&buf, "tli");
    1052         282 :     pq_sendint32(&buf, 0);      /* table oid */
    1053         282 :     pq_sendint16(&buf, 0);      /* attnum */
    1054             : 
    1055             :     /*
    1056             :      * int8 may seem like a surprising data type for this, but in theory int4
    1057             :      * would not be wide enough for this, as TimeLineID is unsigned.
    1058             :      */
    1059         282 :     pq_sendint32(&buf, INT8OID);    /* type oid */
    1060         282 :     pq_sendint16(&buf, -1);
    1061         282 :     pq_sendint32(&buf, 0);
    1062         282 :     pq_sendint16(&buf, 0);
    1063         282 :     pq_endmessage(&buf);
    1064             : 
    1065             :     /* Data row */
    1066         282 :     pq_beginmessage(&buf, 'D');
    1067         282 :     pq_sendint16(&buf, 2);      /* number of columns */
    1068             : 
    1069         846 :     len = snprintf(str, sizeof(str),
    1070         282 :                    "%X/%X", (uint32) (ptr >> 32), (uint32) ptr);
    1071         282 :     pq_sendint32(&buf, len);
    1072         282 :     pq_sendbytes(&buf, str, len);
    1073             : 
    1074         282 :     len = snprintf(str, sizeof(str), "%u", tli);
    1075         282 :     pq_sendint32(&buf, len);
    1076         282 :     pq_sendbytes(&buf, str, len);
    1077             : 
    1078         282 :     pq_endmessage(&buf);
    1079             : 
    1080             :     /* Send a CommandComplete message */
    1081         282 :     pq_puttextmessage('C', "SELECT");
    1082         282 : }
    1083             : 
    1084             : /*
    1085             :  * Inject a file with given name and content in the output tar stream.
    1086             :  */
    1087             : static void
    1088         156 : sendFileWithContent(const char *filename, const char *content,
    1089             :                     backup_manifest_info *manifest)
    1090             : {
    1091             :     struct stat statbuf;
    1092             :     int         pad,
    1093             :                 len;
    1094             :     pg_checksum_context checksum_ctx;
    1095             : 
    1096         156 :     pg_checksum_init(&checksum_ctx, manifest->checksum_type);
    1097             : 
    1098         156 :     len = strlen(content);
    1099             : 
    1100             :     /*
    1101             :      * Construct a stat struct for the backup_label file we're injecting in
    1102             :      * the tar.
    1103             :      */
    1104             :     /* Windows doesn't have the concept of uid and gid */
    1105             : #ifdef WIN32
    1106             :     statbuf.st_uid = 0;
    1107             :     statbuf.st_gid = 0;
    1108             : #else
    1109         156 :     statbuf.st_uid = geteuid();
    1110         156 :     statbuf.st_gid = getegid();
    1111             : #endif
    1112         156 :     statbuf.st_mtime = time(NULL);
    1113         156 :     statbuf.st_mode = pg_file_create_mode;
    1114         156 :     statbuf.st_size = len;
    1115             : 
    1116         156 :     _tarWriteHeader(filename, NULL, &statbuf, false);
    1117             :     /* Send the contents as a CopyData message */
    1118         156 :     pq_putmessage('d', content, len);
    1119         156 :     update_basebackup_progress(len);
    1120             : 
    1121             :     /* Pad to 512 byte boundary, per tar format requirements */
    1122         156 :     pad = ((len + 511) & ~511) - len;
    1123         156 :     if (pad > 0)
    1124             :     {
    1125             :         char        buf[512];
    1126             : 
    1127        4108 :         MemSet(buf, 0, pad);
    1128         148 :         pq_putmessage('d', buf, pad);
    1129         148 :         update_basebackup_progress(pad);
    1130             :     }
    1131             : 
    1132         156 :     pg_checksum_update(&checksum_ctx, (uint8 *) content, len);
    1133         156 :     AddFileToBackupManifest(manifest, NULL, filename, len,
    1134         156 :                             (pg_time_t) statbuf.st_mtime, &checksum_ctx);
    1135         156 : }
    1136             : 
    1137             : /*
    1138             :  * Include the tablespace directory pointed to by 'path' in the output tar
    1139             :  * stream.  If 'sizeonly' is true, we just calculate a total length and return
    1140             :  * it, without actually sending anything.
    1141             :  *
    1142             :  * Only used to send auxiliary tablespaces, not PGDATA.
    1143             :  */
    1144             : int64
    1145          64 : sendTablespace(char *path, char *spcoid, bool sizeonly,
    1146             :                backup_manifest_info *manifest)
    1147             : {
    1148             :     int64       size;
    1149             :     char        pathbuf[MAXPGPATH];
    1150             :     struct stat statbuf;
    1151             : 
    1152             :     /*
    1153             :      * 'path' points to the tablespace location, but we only want to include
    1154             :      * the version directory in it that belongs to us.
    1155             :      */
    1156          64 :     snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
    1157             :              TABLESPACE_VERSION_DIRECTORY);
    1158             : 
    1159             :     /*
    1160             :      * Store a directory entry in the tar file so we get the permissions
    1161             :      * right.
    1162             :      */
    1163          64 :     if (lstat(pathbuf, &statbuf) != 0)
    1164             :     {
    1165           0 :         if (errno != ENOENT)
    1166           0 :             ereport(ERROR,
    1167             :                     (errcode_for_file_access(),
    1168             :                      errmsg("could not stat file or directory \"%s\": %m",
    1169             :                             pathbuf)));
    1170             : 
    1171             :         /* If the tablespace went away while scanning, it's no error. */
    1172           0 :         return 0;
    1173             :     }
    1174             : 
    1175          64 :     size = _tarWriteHeader(TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
    1176             :                            sizeonly);
    1177             : 
    1178             :     /* Send all the files in the tablespace version directory */
    1179          64 :     size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true, manifest,
    1180             :                     spcoid);
    1181             : 
    1182          64 :     return size;
    1183             : }
    1184             : 
    1185             : /*
    1186             :  * Include all files from the given directory in the output tar stream. If
    1187             :  * 'sizeonly' is true, we just calculate a total length and return it, without
    1188             :  * actually sending anything.
    1189             :  *
    1190             :  * Omit any directory in the tablespaces list, to avoid backing up
    1191             :  * tablespaces twice when they were created inside PGDATA.
    1192             :  *
    1193             :  * If sendtblspclinks is true, we need to include symlink
    1194             :  * information in the tar file. If not, we can skip that
    1195             :  * as it will be sent separately in the tablespace_map file.
    1196             :  */
    1197             : static int64
    1198        4984 : sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces,
    1199             :         bool sendtblspclinks, backup_manifest_info *manifest,
    1200             :         const char *spcoid)
    1201             : {
    1202             :     DIR        *dir;
    1203             :     struct dirent *de;
    1204             :     char        pathbuf[MAXPGPATH * 2];
    1205             :     struct stat statbuf;
    1206        4984 :     int64       size = 0;
    1207             :     const char *lastDir;        /* Split last dir from parent path. */
    1208        4984 :     bool        isDbDir = false;    /* Does this directory contain relations? */
    1209             : 
    1210             :     /*
    1211             :      * Determine if the current path is a database directory that can contain
    1212             :      * relations.
    1213             :      *
    1214             :      * Start by finding the location of the delimiter between the parent path
    1215             :      * and the current path.
    1216             :      */
    1217        4984 :     lastDir = last_dir_separator(path);
    1218             : 
    1219             :     /* Does this path look like a database path (i.e. all digits)? */
    1220        4984 :     if (lastDir != NULL &&
    1221        4696 :         strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
    1222             :     {
    1223             :         /* Part of path that contains the parent directory. */
    1224         910 :         int         parentPathLen = lastDir - path;
    1225             : 
    1226             :         /*
    1227             :          * Mark path as a database directory if the parent path is either
    1228             :          * $PGDATA/base or a tablespace version path.
    1229             :          */
    1230         910 :         if (strncmp(path, "./base", parentPathLen) == 0 ||
    1231          56 :             (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
    1232          56 :              strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
    1233             :                      TABLESPACE_VERSION_DIRECTORY,
    1234             :                      sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
    1235         910 :             isDbDir = true;
    1236             :     }
    1237             : 
    1238        4984 :     dir = AllocateDir(path);
    1239      289634 :     while ((de = ReadDir(dir, path)) != NULL)
    1240             :     {
    1241             :         int         excludeIdx;
    1242             :         bool        excludeFound;
    1243             :         ForkNumber  relForkNum; /* Type of fork if file is a relation */
    1244             :         int         relOidChars;    /* Chars in filename that are the rel oid */
    1245             : 
    1246             :         /* Skip special stuff */
    1247      284662 :         if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
    1248       14180 :             continue;
    1249             : 
    1250             :         /* Skip temporary files */
    1251      274710 :         if (strncmp(de->d_name,
    1252             :                     PG_TEMP_FILE_PREFIX,
    1253             :                     strlen(PG_TEMP_FILE_PREFIX)) == 0)
    1254         282 :             continue;
    1255             : 
    1256             :         /*
    1257             :          * Check if the postmaster has signaled us to exit, and abort with an
    1258             :          * error in that case. The error handler further up will call
    1259             :          * do_pg_abort_backup() for us. Also check that if the backup was
    1260             :          * started while still in recovery, the server wasn't promoted.
    1261             :          * do_pg_stop_backup() will check that too, but it's better to stop
    1262             :          * the backup early than continue to the end and fail there.
    1263             :          */
    1264      274428 :         CHECK_FOR_INTERRUPTS();
    1265      274428 :         if (RecoveryInProgress() != backup_started_in_recovery)
    1266           0 :             ereport(ERROR,
    1267             :                     (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
    1268             :                      errmsg("the standby was promoted during online backup"),
    1269             :                      errhint("This means that the backup being taken is corrupt "
    1270             :                              "and should not be used. "
    1271             :                              "Try taking another online backup.")));
    1272             : 
    1273             :         /* Scan for files that should be excluded */
    1274      274428 :         excludeFound = false;
    1275     2464820 :         for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
    1276             :         {
    1277     2191620 :             int         cmplen = strlen(excludeFiles[excludeIdx].name);
    1278             : 
    1279     2191620 :             if (!excludeFiles[excludeIdx].match_prefix)
    1280     1917372 :                 cmplen++;
    1281     2191620 :             if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
    1282             :             {
    1283        1228 :                 elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
    1284        1228 :                 excludeFound = true;
    1285        1228 :                 break;
    1286             :             }
    1287             :         }
    1288             : 
    1289      274428 :         if (excludeFound)
    1290        1228 :             continue;
    1291             : 
    1292             :         /* Exclude all forks for unlogged tables except the init fork */
    1293      520554 :         if (isDbDir &&
    1294      247354 :             parse_filename_for_nontemp_relation(de->d_name, &relOidChars,
    1295             :                                                 &relForkNum))
    1296             :         {
    1297             :             /* Never exclude init forks */
    1298      245574 :             if (relForkNum != INIT_FORKNUM)
    1299             :             {
    1300             :                 char        initForkFile[MAXPGPATH];
    1301             :                 char        relOid[OIDCHARS + 1];
    1302             : 
    1303             :                 /*
    1304             :                  * If any other type of fork, check if there is an init fork
    1305             :                  * with the same OID. If so, the file can be excluded.
    1306             :                  */
    1307      245476 :                 memcpy(relOid, de->d_name, relOidChars);
    1308      245476 :                 relOid[relOidChars] = '\0';
    1309      245476 :                 snprintf(initForkFile, sizeof(initForkFile), "%s/%s_init",
    1310             :                          path, relOid);
    1311             : 
    1312      245476 :                 if (lstat(initForkFile, &statbuf) == 0)
    1313             :                 {
    1314          98 :                     elog(DEBUG2,
    1315             :                          "unlogged relation file \"%s\" excluded from backup",
    1316             :                          de->d_name);
    1317             : 
    1318          98 :                     continue;
    1319             :                 }
    1320             :             }
    1321             :         }
    1322             : 
    1323             :         /* Exclude temporary relations */
    1324      273102 :         if (isDbDir && looks_like_temp_rel_name(de->d_name))
    1325             :         {
    1326          72 :             elog(DEBUG2,
    1327             :                  "temporary relation file \"%s\" excluded from backup",
    1328             :                  de->d_name);
    1329             : 
    1330          72 :             continue;
    1331             :         }
    1332             : 
    1333      273030 :         snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
    1334             : 
    1335             :         /* Skip pg_control here to back up it last */
    1336      273030 :         if (strcmp(pathbuf, "./global/pg_control") == 0)
    1337         282 :             continue;
    1338             : 
    1339      272748 :         if (lstat(pathbuf, &statbuf) != 0)
    1340             :         {
    1341           0 :             if (errno != ENOENT)
    1342           0 :                 ereport(ERROR,
    1343             :                         (errcode_for_file_access(),
    1344             :                          errmsg("could not stat file or directory \"%s\": %m",
    1345             :                                 pathbuf)));
    1346             : 
    1347             :             /* If the file went away while scanning, it's not an error. */
    1348           0 :             continue;
    1349             :         }
    1350             : 
    1351             :         /* Scan for directories whose contents should be excluded */
    1352      272748 :         excludeFound = false;
    1353     2174032 :         for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
    1354             :         {
    1355     1903268 :             if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
    1356             :             {
    1357        1984 :                 elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
    1358        1984 :                 size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
    1359        1984 :                 excludeFound = true;
    1360        1984 :                 break;
    1361             :             }
    1362             :         }
    1363             : 
    1364      272748 :         if (excludeFound)
    1365        1984 :             continue;
    1366             : 
    1367             :         /*
    1368             :          * Exclude contents of directory specified by statrelpath if not set
    1369             :          * to the default (pg_stat_tmp) which is caught in the loop above.
    1370             :          */
    1371      270764 :         if (statrelpath != NULL && strcmp(pathbuf, statrelpath) == 0)
    1372             :         {
    1373           0 :             elog(DEBUG1, "contents of directory \"%s\" excluded from backup", statrelpath);
    1374           0 :             size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
    1375           0 :             continue;
    1376             :         }
    1377             : 
    1378             :         /*
    1379             :          * We can skip pg_wal, the WAL segments need to be fetched from the
    1380             :          * WAL archive anyway. But include it as an empty directory anyway, so
    1381             :          * we get permissions right.
    1382             :          */
    1383      270764 :         if (strcmp(pathbuf, "./pg_wal") == 0)
    1384             :         {
    1385             :             /* If pg_wal is a symlink, write it as a directory anyway */
    1386         282 :             size += _tarWriteDir(pathbuf, basepathlen, &statbuf, sizeonly);
    1387             : 
    1388             :             /*
    1389             :              * Also send archive_status directory (by hackishly reusing
    1390             :              * statbuf from above ...).
    1391             :              */
    1392         282 :             size += _tarWriteHeader("./pg_wal/archive_status", NULL, &statbuf,
    1393             :                                     sizeonly);
    1394             : 
    1395         282 :             continue;           /* don't recurse into pg_wal */
    1396             :         }
    1397             : 
    1398             :         /* Allow symbolic links in pg_tblspc only */
    1399      270482 :         if (strcmp(path, "./pg_tblspc") == 0 &&
    1400             : #ifndef WIN32
    1401          58 :             S_ISLNK(statbuf.st_mode)
    1402             : #else
    1403             :             pgwin32_is_junction(pathbuf)
    1404             : #endif
    1405             :             )
    1406          58 :         {
    1407             : #if defined(HAVE_READLINK) || defined(WIN32)
    1408             :             char        linkpath[MAXPGPATH];
    1409             :             int         rllen;
    1410             : 
    1411          58 :             rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
    1412          58 :             if (rllen < 0)
    1413           0 :                 ereport(ERROR,
    1414             :                         (errcode_for_file_access(),
    1415             :                          errmsg("could not read symbolic link \"%s\": %m",
    1416             :                                 pathbuf)));
    1417          58 :             if (rllen >= sizeof(linkpath))
    1418           0 :                 ereport(ERROR,
    1419             :                         (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
    1420             :                          errmsg("symbolic link \"%s\" target is too long",
    1421             :                                 pathbuf)));
    1422          58 :             linkpath[rllen] = '\0';
    1423             : 
    1424          58 :             size += _tarWriteHeader(pathbuf + basepathlen + 1, linkpath,
    1425             :                                     &statbuf, sizeonly);
    1426             : #else
    1427             : 
    1428             :             /*
    1429             :              * If the platform does not have symbolic links, it should not be
    1430             :              * possible to have tablespaces - clearly somebody else created
    1431             :              * them. Warn about it and ignore.
    1432             :              */
    1433             :             ereport(WARNING,
    1434             :                     (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
    1435             :                      errmsg("tablespaces are not supported on this platform")));
    1436             :             continue;
    1437             : #endif                          /* HAVE_READLINK */
    1438             :         }
    1439      270424 :         else if (S_ISDIR(statbuf.st_mode))
    1440             :         {
    1441        4640 :             bool        skip_this_dir = false;
    1442             :             ListCell   *lc;
    1443             : 
    1444             :             /*
    1445             :              * Store a directory entry in the tar file so we can get the
    1446             :              * permissions right.
    1447             :              */
    1448        4640 :             size += _tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf,
    1449             :                                     sizeonly);
    1450             : 
    1451             :             /*
    1452             :              * Call ourselves recursively for a directory, unless it happens
    1453             :              * to be a separate tablespace located within PGDATA.
    1454             :              */
    1455        7910 :             foreach(lc, tablespaces)
    1456             :             {
    1457        3270 :                 tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
    1458             : 
    1459             :                 /*
    1460             :                  * ti->rpath is the tablespace relative path within PGDATA, or
    1461             :                  * NULL if the tablespace has been properly located somewhere
    1462             :                  * else.
    1463             :                  *
    1464             :                  * Skip past the leading "./" in pathbuf when comparing.
    1465             :                  */
    1466        3270 :                 if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
    1467             :                 {
    1468           0 :                     skip_this_dir = true;
    1469           0 :                     break;
    1470             :                 }
    1471             :             }
    1472             : 
    1473             :             /*
    1474             :              * skip sending directories inside pg_tblspc, if not required.
    1475             :              */
    1476        4640 :             if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
    1477           8 :                 skip_this_dir = true;
    1478             : 
    1479        4640 :             if (!skip_this_dir)
    1480        4632 :                 size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces,
    1481             :                                 sendtblspclinks, manifest, spcoid);
    1482             :         }
    1483      265784 :         else if (S_ISREG(statbuf.st_mode))
    1484             :         {
    1485      265784 :             bool        sent = false;
    1486             : 
    1487      265784 :             if (!sizeonly)
    1488      251110 :                 sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf,
    1489      120988 :                                 true, isDbDir ? atooid(lastDir + 1) : InvalidOid,
    1490             :                                 manifest, spcoid);
    1491             : 
    1492      265778 :             if (sent || sizeonly)
    1493             :             {
    1494             :                 /* Add size, rounded up to 512byte block */
    1495      265778 :                 size += ((statbuf.st_size + 511) & ~511);
    1496      265778 :                 size += 512;    /* Size of the header of the file */
    1497             :             }
    1498             :         }
    1499             :         else
    1500           0 :             ereport(WARNING,
    1501             :                     (errmsg("skipping special file \"%s\"", pathbuf)));
    1502             :     }
    1503        4972 :     FreeDir(dir);
    1504        4972 :     return size;
    1505             : }
    1506             : 
    1507             : /*
    1508             :  * Check if a file should have its checksum validated.
    1509             :  * We validate checksums on files in regular tablespaces
    1510             :  * (including global and default) only, and in those there
    1511             :  * are some files that are explicitly excluded.
    1512             :  */
    1513             : static bool
    1514       37516 : is_checksummed_file(const char *fullpath, const char *filename)
    1515             : {
    1516             :     /* Check that the file is in a tablespace */
    1517       37516 :     if (strncmp(fullpath, "./global/", 9) == 0 ||
    1518       35244 :         strncmp(fullpath, "./base/", 7) == 0 ||
    1519         512 :         strncmp(fullpath, "/", 1) == 0)
    1520             :     {
    1521             :         int         excludeIdx;
    1522             : 
    1523             :         /* Compare file against noChecksumFiles skip list */
    1524      184470 :         for (excludeIdx = 0; noChecksumFiles[excludeIdx].name != NULL; excludeIdx++)
    1525             :         {
    1526      147736 :             int         cmplen = strlen(noChecksumFiles[excludeIdx].name);
    1527             : 
    1528      147736 :             if (!noChecksumFiles[excludeIdx].match_prefix)
    1529      110882 :                 cmplen++;
    1530      147736 :             if (strncmp(filename, noChecksumFiles[excludeIdx].name,
    1531             :                         cmplen) == 0)
    1532         280 :                 return false;
    1533             :         }
    1534             : 
    1535       36734 :         return true;
    1536             :     }
    1537             :     else
    1538         502 :         return false;
    1539             : }
    1540             : 
    1541             : /*****
    1542             :  * Functions for handling tar file format
    1543             :  *
    1544             :  * Copied from pg_dump, but modified to work with libpq for sending
    1545             :  */
    1546             : 
    1547             : 
    1548             : /*
    1549             :  * Given the member, write the TAR header & send the file.
    1550             :  *
    1551             :  * If 'missing_ok' is true, will not throw an error if the file is not found.
    1552             :  *
    1553             :  * If dboid is anything other than InvalidOid then any checksum failures detected
    1554             :  * will get reported to the stats collector.
    1555             :  *
    1556             :  * Returns true if the file was successfully sent, false if 'missing_ok',
    1557             :  * and the file did not exist.
    1558             :  */
    1559             : static bool
    1560      130260 : sendFile(const char *readfilename, const char *tarfilename,
    1561             :          struct stat *statbuf, bool missing_ok, Oid dboid,
    1562             :          backup_manifest_info *manifest, const char *spcoid)
    1563             : {
    1564             :     FILE       *fp;
    1565      130260 :     BlockNumber blkno = 0;
    1566      130260 :     bool        block_retry = false;
    1567             :     char        buf[TAR_SEND_SIZE];
    1568             :     uint16      checksum;
    1569      130260 :     int         checksum_failures = 0;
    1570             :     off_t       cnt;
    1571             :     int         i;
    1572      130260 :     pgoff_t     len = 0;
    1573             :     char       *page;
    1574             :     size_t      pad;
    1575             :     PageHeader  phdr;
    1576      130260 :     int         segmentno = 0;
    1577             :     char       *segmentpath;
    1578      130260 :     bool        verify_checksum = false;
    1579             :     pg_checksum_context checksum_ctx;
    1580             : 
    1581      130260 :     pg_checksum_init(&checksum_ctx, manifest->checksum_type);
    1582             : 
    1583      130260 :     fp = AllocateFile(readfilename, "rb");
    1584      130260 :     if (fp == NULL)
    1585             :     {
    1586           0 :         if (errno == ENOENT && missing_ok)
    1587           0 :             return false;
    1588           0 :         ereport(ERROR,
    1589             :                 (errcode_for_file_access(),
    1590             :                  errmsg("could not open file \"%s\": %m", readfilename)));
    1591             :     }
    1592             : 
    1593      130260 :     _tarWriteHeader(tarfilename, NULL, statbuf, false);
    1594             : 
    1595      130258 :     if (!noverify_checksums && DataChecksumsEnabled())
    1596             :     {
    1597             :         char       *filename;
    1598             : 
    1599             :         /*
    1600             :          * Get the filename (excluding path).  As last_dir_separator()
    1601             :          * includes the last directory separator, we chop that off by
    1602             :          * incrementing the pointer.
    1603             :          */
    1604       37516 :         filename = last_dir_separator(readfilename) + 1;
    1605             : 
    1606       37516 :         if (is_checksummed_file(readfilename, filename))
    1607             :         {
    1608       36734 :             verify_checksum = true;
    1609             : 
    1610             :             /*
    1611             :              * Cut off at the segment boundary (".") to get the segment number
    1612             :              * in order to mix it into the checksum.
    1613             :              */
    1614       36734 :             segmentpath = strstr(filename, ".");
    1615       36734 :             if (segmentpath != NULL)
    1616             :             {
    1617           0 :                 segmentno = atoi(segmentpath + 1);
    1618           0 :                 if (segmentno == 0)
    1619           0 :                     ereport(ERROR,
    1620             :                             (errmsg("invalid segment number %d in file \"%s\"",
    1621             :                                     segmentno, filename)));
    1622             :             }
    1623             :         }
    1624             :     }
    1625             : 
    1626      184270 :     while ((cnt = fread(buf, 1, Min(sizeof(buf), statbuf->st_size - len), fp)) > 0)
    1627             :     {
    1628             :         /*
    1629             :          * The checksums are verified at block level, so we iterate over the
    1630             :          * buffer in chunks of BLCKSZ, after making sure that
    1631             :          * TAR_SEND_SIZE/buf is divisible by BLCKSZ and we read a multiple of
    1632             :          * BLCKSZ bytes.
    1633             :          */
    1634             :         Assert(TAR_SEND_SIZE % BLCKSZ == 0);
    1635             : 
    1636      161606 :         if (verify_checksum && (cnt % BLCKSZ != 0))
    1637             :         {
    1638           0 :             ereport(WARNING,
    1639             :                     (errmsg("could not verify checksum in file \"%s\", block "
    1640             :                             "%d: read buffer size %d and page size %d "
    1641             :                             "differ",
    1642             :                             readfilename, blkno, (int) cnt, BLCKSZ)));
    1643           0 :             verify_checksum = false;
    1644             :         }
    1645             : 
    1646      161606 :         if (verify_checksum)
    1647             :         {
    1648      160202 :             for (i = 0; i < cnt / BLCKSZ; i++)
    1649             :             {
    1650      114460 :                 page = buf + BLCKSZ * i;
    1651             : 
    1652             :                 /*
    1653             :                  * Only check pages which have not been modified since the
    1654             :                  * start of the base backup. Otherwise, they might have been
    1655             :                  * written only halfway and the checksum would not be valid.
    1656             :                  * However, replaying WAL would reinstate the correct page in
    1657             :                  * this case. We also skip completely new pages, since they
    1658             :                  * don't have a checksum yet.
    1659             :                  */
    1660      114460 :                 if (!PageIsNew(page) && PageGetLSN(page) < startptr)
    1661             :                 {
    1662      114460 :                     checksum = pg_checksum_page((char *) page, blkno + segmentno * RELSEG_SIZE);
    1663      114460 :                     phdr = (PageHeader) page;
    1664      114460 :                     if (phdr->pd_checksum != checksum)
    1665             :                     {
    1666             :                         /*
    1667             :                          * Retry the block on the first failure.  It's
    1668             :                          * possible that we read the first 4K page of the
    1669             :                          * block just before postgres updated the entire block
    1670             :                          * so it ends up looking torn to us.  We only need to
    1671             :                          * retry once because the LSN should be updated to
    1672             :                          * something we can ignore on the next pass.  If the
    1673             :                          * error happens again then it is a true validation
    1674             :                          * failure.
    1675             :                          */
    1676          56 :                         if (block_retry == false)
    1677             :                         {
    1678             :                             /* Reread the failed block */
    1679          28 :                             if (fseek(fp, -(cnt - BLCKSZ * i), SEEK_CUR) == -1)
    1680             :                             {
    1681           0 :                                 ereport(ERROR,
    1682             :                                         (errcode_for_file_access(),
    1683             :                                          errmsg("could not fseek in file \"%s\": %m",
    1684             :                                                 readfilename)));
    1685             :                             }
    1686             : 
    1687          28 :                             if (fread(buf + BLCKSZ * i, 1, BLCKSZ, fp) != BLCKSZ)
    1688             :                             {
    1689             :                                 /*
    1690             :                                  * If we hit end-of-file, a concurrent
    1691             :                                  * truncation must have occurred, so break out
    1692             :                                  * of this loop just as if the initial fread()
    1693             :                                  * returned 0. We'll drop through to the same
    1694             :                                  * code that handles that case. (We must fix
    1695             :                                  * up cnt first, though.)
    1696             :                                  */
    1697           0 :                                 if (feof(fp))
    1698             :                                 {
    1699           0 :                                     cnt = BLCKSZ * i;
    1700           0 :                                     break;
    1701             :                                 }
    1702             : 
    1703           0 :                                 ereport(ERROR,
    1704             :                                         (errcode_for_file_access(),
    1705             :                                          errmsg("could not reread block %d of file \"%s\": %m",
    1706             :                                                 blkno, readfilename)));
    1707             :                             }
    1708             : 
    1709          28 :                             if (fseek(fp, cnt - BLCKSZ * i - BLCKSZ, SEEK_CUR) == -1)
    1710             :                             {
    1711           0 :                                 ereport(ERROR,
    1712             :                                         (errcode_for_file_access(),
    1713             :                                          errmsg("could not fseek in file \"%s\": %m",
    1714             :                                                 readfilename)));
    1715             :                             }
    1716             : 
    1717             :                             /* Set flag so we know a retry was attempted */
    1718          28 :                             block_retry = true;
    1719             : 
    1720             :                             /* Reset loop to validate the block again */
    1721          28 :                             i--;
    1722          28 :                             continue;
    1723             :                         }
    1724             : 
    1725          28 :                         checksum_failures++;
    1726             : 
    1727          28 :                         if (checksum_failures <= 5)
    1728          24 :                             ereport(WARNING,
    1729             :                                     (errmsg("checksum verification failed in "
    1730             :                                             "file \"%s\", block %d: calculated "
    1731             :                                             "%X but expected %X",
    1732             :                                             readfilename, blkno, checksum,
    1733             :                                             phdr->pd_checksum)));
    1734          28 :                         if (checksum_failures == 5)
    1735           4 :                             ereport(WARNING,
    1736             :                                     (errmsg("further checksum verification "
    1737             :                                             "failures in file \"%s\" will not "
    1738             :                                             "be reported", readfilename)));
    1739             :                     }
    1740             :                 }
    1741      114432 :                 block_retry = false;
    1742      114432 :                 blkno++;
    1743             :             }
    1744             :         }
    1745             : 
    1746             :         /* Send the chunk as a CopyData message */
    1747      161606 :         if (pq_putmessage('d', buf, cnt))
    1748           4 :             ereport(ERROR,
    1749             :                     (errmsg("base backup could not send data, aborting backup")));
    1750      161602 :         update_basebackup_progress(cnt);
    1751             : 
    1752             :         /* Also feed it to the checksum machinery. */
    1753      161602 :         pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
    1754             : 
    1755      161602 :         len += cnt;
    1756      161602 :         throttle(cnt);
    1757             : 
    1758      161602 :         if (feof(fp) || len >= statbuf->st_size)
    1759             :         {
    1760             :             /*
    1761             :              * Reached end of file. The file could be longer, if it was
    1762             :              * extended while we were sending it, but for a base backup we can
    1763             :              * ignore such extended data. It will be restored from WAL.
    1764             :              */
    1765             :             break;
    1766             :         }
    1767             :     }
    1768             : 
    1769      130254 :     CHECK_FREAD_ERROR(fp, readfilename);
    1770             : 
    1771             :     /* If the file was truncated while we were sending it, pad it with zeros */
    1772      130254 :     if (len < statbuf->st_size)
    1773             :     {
    1774           0 :         MemSet(buf, 0, sizeof(buf));
    1775           0 :         while (len < statbuf->st_size)
    1776             :         {
    1777           0 :             cnt = Min(sizeof(buf), statbuf->st_size - len);
    1778           0 :             pq_putmessage('d', buf, cnt);
    1779           0 :             pg_checksum_update(&checksum_ctx, (uint8 *) buf, cnt);
    1780           0 :             update_basebackup_progress(cnt);
    1781           0 :             len += cnt;
    1782           0 :             throttle(cnt);
    1783             :         }
    1784             :     }
    1785             : 
    1786             :     /*
    1787             :      * Pad to 512 byte boundary, per tar format requirements. (This small
    1788             :      * piece of data is probably not worth throttling, and is not checksummed
    1789             :      * because it's not actually part of the file.)
    1790             :      */
    1791      130254 :     pad = ((len + 511) & ~511) - len;
    1792      130254 :     if (pad > 0)
    1793             :     {
    1794       18644 :         MemSet(buf, 0, pad);
    1795        1358 :         pq_putmessage('d', buf, pad);
    1796        1358 :         update_basebackup_progress(pad);
    1797             :     }
    1798             : 
    1799      130254 :     FreeFile(fp);
    1800             : 
    1801      130254 :     if (checksum_failures > 1)
    1802             :     {
    1803           4 :         ereport(WARNING,
    1804             :                 (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
    1805             :                                "file \"%s\" has a total of %d checksum verification failures",
    1806             :                                checksum_failures,
    1807             :                                readfilename, checksum_failures)));
    1808             : 
    1809           4 :         pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
    1810             :     }
    1811             : 
    1812      130254 :     total_checksum_failures += checksum_failures;
    1813             : 
    1814      130254 :     AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
    1815      130254 :                             (pg_time_t) statbuf->st_mtime, &checksum_ctx);
    1816             : 
    1817      130254 :     return true;
    1818             : }
    1819             : 
    1820             : 
    1821             : static int64
    1822      137728 : _tarWriteHeader(const char *filename, const char *linktarget,
    1823             :                 struct stat *statbuf, bool sizeonly)
    1824             : {
    1825             :     char        h[512];
    1826             :     enum tarError rc;
    1827             : 
    1828      137728 :     if (!sizeonly)
    1829             :     {
    1830      134024 :         rc = tarCreateHeader(h, filename, linktarget, statbuf->st_size,
    1831             :                              statbuf->st_mode, statbuf->st_uid, statbuf->st_gid,
    1832             :                              statbuf->st_mtime);
    1833             : 
    1834      134024 :         switch (rc)
    1835             :         {
    1836      134022 :             case TAR_OK:
    1837      134022 :                 break;
    1838           2 :             case TAR_NAME_TOO_LONG:
    1839           2 :                 ereport(ERROR,
    1840             :                         (errmsg("file name too long for tar format: \"%s\"",
    1841             :                                 filename)));
    1842             :                 break;
    1843           0 :             case TAR_SYMLINK_TOO_LONG:
    1844           0 :                 ereport(ERROR,
    1845             :                         (errmsg("symbolic link target too long for tar format: "
    1846             :                                 "file name \"%s\", target \"%s\"",
    1847             :                                 filename, linktarget)));
    1848             :                 break;
    1849           0 :             default:
    1850           0 :                 elog(ERROR, "unrecognized tar error: %d", rc);
    1851             :         }
    1852             : 
    1853      134022 :         pq_putmessage('d', h, sizeof(h));
    1854      134022 :         update_basebackup_progress(sizeof(h));
    1855             :     }
    1856             : 
    1857      137726 :     return sizeof(h);
    1858             : }
    1859             : 
    1860             : /*
    1861             :  * Write tar header for a directory.  If the entry in statbuf is a link then
    1862             :  * write it as a directory anyway.
    1863             :  */
    1864             : static int64
    1865        2266 : _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf,
    1866             :              bool sizeonly)
    1867             : {
    1868             :     /* If symlink, write it as a directory anyway */
    1869             : #ifndef WIN32
    1870        2266 :     if (S_ISLNK(statbuf->st_mode))
    1871             : #else
    1872             :     if (pgwin32_is_junction(pathbuf))
    1873             : #endif
    1874          88 :         statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
    1875             : 
    1876        2266 :     return _tarWriteHeader(pathbuf + basepathlen + 1, NULL, statbuf, sizeonly);
    1877             : }
    1878             : 
    1879             : /*
    1880             :  * Increment the network transfer counter by the given number of bytes,
    1881             :  * and sleep if necessary to comply with the requested network transfer
    1882             :  * rate.
    1883             :  */
    1884             : static void
    1885      162626 : throttle(size_t increment)
    1886             : {
    1887             :     TimeOffset  elapsed_min;
    1888             : 
    1889      162626 :     if (throttling_counter < 0)
    1890      162626 :         return;
    1891             : 
    1892           0 :     throttling_counter += increment;
    1893           0 :     if (throttling_counter < throttling_sample)
    1894           0 :         return;
    1895             : 
    1896             :     /* How much time should have elapsed at minimum? */
    1897           0 :     elapsed_min = elapsed_min_unit *
    1898           0 :         (throttling_counter / throttling_sample);
    1899             : 
    1900             :     /*
    1901             :      * Since the latch could be set repeatedly because of concurrently WAL
    1902             :      * activity, sleep in a loop to ensure enough time has passed.
    1903             :      */
    1904             :     for (;;)
    1905           0 :     {
    1906             :         TimeOffset  elapsed,
    1907             :                     sleep;
    1908             :         int         wait_result;
    1909             : 
    1910             :         /* Time elapsed since the last measurement (and possible wake up). */
    1911           0 :         elapsed = GetCurrentTimestamp() - throttled_last;
    1912             : 
    1913             :         /* sleep if the transfer is faster than it should be */
    1914           0 :         sleep = elapsed_min - elapsed;
    1915           0 :         if (sleep <= 0)
    1916           0 :             break;
    1917             : 
    1918           0 :         ResetLatch(MyLatch);
    1919             : 
    1920             :         /* We're eating a potentially set latch, so check for interrupts */
    1921           0 :         CHECK_FOR_INTERRUPTS();
    1922             : 
    1923             :         /*
    1924             :          * (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
    1925             :          * the maximum time to sleep. Thus the cast to long is safe.
    1926             :          */
    1927           0 :         wait_result = WaitLatch(MyLatch,
    1928             :                                 WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
    1929           0 :                                 (long) (sleep / 1000),
    1930             :                                 WAIT_EVENT_BASE_BACKUP_THROTTLE);
    1931             : 
    1932           0 :         if (wait_result & WL_LATCH_SET)
    1933           0 :             CHECK_FOR_INTERRUPTS();
    1934             : 
    1935             :         /* Done waiting? */
    1936           0 :         if (wait_result & WL_TIMEOUT)
    1937           0 :             break;
    1938             :     }
    1939             : 
    1940             :     /*
    1941             :      * As we work with integers, only whole multiple of throttling_sample was
    1942             :      * processed. The rest will be done during the next call of this function.
    1943             :      */
    1944           0 :     throttling_counter %= throttling_sample;
    1945             : 
    1946             :     /*
    1947             :      * Time interval for the remaining amount and possible next increments
    1948             :      * starts now.
    1949             :      */
    1950           0 :     throttled_last = GetCurrentTimestamp();
    1951             : }
    1952             : 
    1953             : /*
    1954             :  * Increment the counter for the amount of data already streamed
    1955             :  * by the given number of bytes, and update the progress report for
    1956             :  * pg_stat_progress_basebackup.
    1957             :  */
    1958             : static void
    1959      298310 : update_basebackup_progress(int64 delta)
    1960             : {
    1961      298310 :     const int   index[] = {
    1962             :         PROGRESS_BASEBACKUP_BACKUP_STREAMED,
    1963             :         PROGRESS_BASEBACKUP_BACKUP_TOTAL
    1964             :     };
    1965             :     int64       val[2];
    1966      298310 :     int         nparam = 0;
    1967             : 
    1968      298310 :     backup_streamed += delta;
    1969      298310 :     val[nparam++] = backup_streamed;
    1970             : 
    1971             :     /*
    1972             :      * Avoid overflowing past 100% or the full size. This may make the total
    1973             :      * size number change as we approach the end of the backup (the estimate
    1974             :      * will always be wrong if WAL is included), but that's better than having
    1975             :      * the done column be bigger than the total.
    1976             :      */
    1977      298310 :     if (backup_total > -1 && backup_streamed > backup_total)
    1978             :     {
    1979        1710 :         backup_total = backup_streamed;
    1980        1710 :         val[nparam++] = backup_total;
    1981             :     }
    1982             : 
    1983      298310 :     pgstat_progress_update_multi_param(nparam, index, val);
    1984      298310 : }

Generated by: LCOV version 1.13