Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * basebackup.c
4 : : * code for taking a base backup and streaming it to a standby
5 : : *
6 : : * Portions Copyright (c) 2010-2026, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/backend/backup/basebackup.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres.h"
14 : :
15 : : #include <sys/stat.h>
16 : : #include <unistd.h>
17 : : #include <time.h>
18 : :
19 : : #include "access/xlog_internal.h"
20 : : #include "access/xlogbackup.h"
21 : : #include "backup/backup_manifest.h"
22 : : #include "backup/basebackup.h"
23 : : #include "backup/basebackup_incremental.h"
24 : : #include "backup/basebackup_sink.h"
25 : : #include "backup/basebackup_target.h"
26 : : #include "catalog/pg_tablespace_d.h"
27 : : #include "commands/defrem.h"
28 : : #include "common/compression.h"
29 : : #include "common/file_perm.h"
30 : : #include "common/file_utils.h"
31 : : #include "lib/stringinfo.h"
32 : : #include "miscadmin.h"
33 : : #include "nodes/pg_list.h"
34 : : #include "pgstat.h"
35 : : #include "pgtar.h"
36 : : #include "postmaster/syslogger.h"
37 : : #include "postmaster/walsummarizer.h"
38 : : #include "replication/slot.h"
39 : : #include "replication/walsender.h"
40 : : #include "replication/walsender_private.h"
41 : : #include "storage/bufpage.h"
42 : : #include "storage/checksum.h"
43 : : #include "storage/dsm_impl.h"
44 : : #include "storage/ipc.h"
45 : : #include "storage/reinit.h"
46 : : #include "utils/builtins.h"
47 : : #include "utils/guc.h"
48 : : #include "utils/injection_point.h"
49 : : #include "utils/ps_status.h"
50 : : #include "utils/relcache.h"
51 : : #include "utils/resowner.h"
52 : : #include "utils/wait_event.h"
53 : :
54 : : /*
55 : : * How much data do we want to send in one CopyData message? Note that
56 : : * this may also result in reading the underlying files in chunks of this
57 : : * size.
58 : : *
59 : : * NB: The buffer size is required to be a multiple of the system block
60 : : * size, so use that value instead if it's bigger than our preference.
61 : : */
62 : : #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
63 : :
64 : : typedef struct
65 : : {
66 : : const char *label;
67 : : bool progress;
68 : : bool fastcheckpoint;
69 : : bool nowait;
70 : : bool includewal;
71 : : bool incremental;
72 : : uint32 maxrate;
73 : : bool sendtblspcmapfile;
74 : : bool send_to_client;
75 : : bool use_copytblspc;
76 : : BaseBackupTargetHandle *target_handle;
77 : : backup_manifest_option manifest;
78 : : pg_compress_algorithm compression;
79 : : pg_compress_specification compression_specification;
80 : : pg_checksum_type manifest_checksum_type;
81 : : } basebackup_options;
82 : :
83 : : #define TAR_NUM_TERMINATION_BLOCKS 2
84 : :
85 : : StaticAssertDecl(TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE <= BLCKSZ,
86 : : "BLCKSZ too small for " CppAsString2(TAR_NUM_TERMINATION_BLOCKS) " tar termination blocks");
87 : :
88 : : static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
89 : : struct backup_manifest_info *manifest,
90 : : IncrementalBackupInfo *ib);
91 : : static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
92 : : List *tablespaces, bool sendtblspclinks,
93 : : backup_manifest_info *manifest, Oid spcoid,
94 : : IncrementalBackupInfo *ib);
95 : : static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
96 : : struct stat *statbuf, bool missing_ok,
97 : : Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
98 : : unsigned segno,
99 : : backup_manifest_info *manifest,
100 : : unsigned num_incremental_blocks,
101 : : BlockNumber *incremental_blocks,
102 : : unsigned truncation_block_length);
103 : : static off_t read_file_data_into_buffer(bbsink *sink,
104 : : const char *readfilename, int fd,
105 : : off_t offset, size_t length,
106 : : BlockNumber blkno,
107 : : bool verify_checksum,
108 : : int *checksum_failures);
109 : : static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
110 : : size_t *bytes_done, void *data, size_t length);
111 : : static bool backup_checksums_verifiable(XLogRecPtr start_lsn);
112 : : static bool verify_page_checksum(Page page, XLogRecPtr start_lsn,
113 : : BlockNumber blkno,
114 : : uint16 *expected_checksum);
115 : : static void sendFileWithContent(bbsink *sink, const char *filename,
116 : : const char *content, int len,
117 : : backup_manifest_info *manifest);
118 : : static int64 _tarWriteHeader(bbsink *sink, const char *filename,
119 : : const char *linktarget, struct stat *statbuf,
120 : : bool sizeonly);
121 : : static void _tarWritePadding(bbsink *sink, int len);
122 : : static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
123 : : static void perform_base_backup(basebackup_options *opt, bbsink *sink,
124 : : IncrementalBackupInfo *ib);
125 : : static void parse_basebackup_options(List *options, basebackup_options *opt);
126 : : static int compareWalFileNames(const ListCell *a, const ListCell *b);
127 : : static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
128 : : const char *filename, bool partial_read_ok);
129 : :
130 : : /* Was the backup currently in-progress initiated in recovery mode? */
131 : : static bool backup_started_in_recovery = false;
132 : :
133 : : /* Total number of checksum failures during base backup. */
134 : : static long long int total_checksum_failures;
135 : :
136 : : /* Do not verify checksums. */
137 : : static bool noverify_checksums = false;
138 : :
139 : : /*
140 : : * Definition of one element part of an exclusion list, used for paths part
141 : : * of checksum validation or base backups. "name" is the name of the file
142 : : * or path to check for exclusion. If "match_prefix" is true, any items
143 : : * matching the name as prefix are excluded.
144 : : */
145 : : struct exclude_list_item
146 : : {
147 : : const char *name;
148 : : bool match_prefix;
149 : : };
150 : :
151 : : /*
152 : : * The contents of these directories are removed or recreated during server
153 : : * start so they are not included in backups. The directories themselves are
154 : : * kept and included as empty to preserve access permissions.
155 : : *
156 : : * Note: this list should be kept in sync with the filter lists in pg_rewind's
157 : : * filemap.c.
158 : : */
159 : : static const char *const excludeDirContents[] =
160 : : {
161 : : /*
162 : : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
163 : : * because extensions like pg_stat_statements store data there.
164 : : */
165 : : PG_STAT_TMP_DIR,
166 : :
167 : : /*
168 : : * It is generally not useful to backup the contents of this directory
169 : : * even if the intention is to restore to another primary. See backup.sgml
170 : : * for a more detailed description.
171 : : */
172 : : PG_REPLSLOT_DIR,
173 : :
174 : : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
175 : : PG_DYNSHMEM_DIR,
176 : :
177 : : /* Contents removed on startup, see AsyncShmemInit(). */
178 : : "pg_notify",
179 : :
180 : : /*
181 : : * Old contents are loaded for possible debugging but are not required for
182 : : * normal operation, see SerialInit().
183 : : */
184 : : "pg_serial",
185 : :
186 : : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
187 : : "pg_snapshots",
188 : :
189 : : /* Contents zeroed on startup, see StartupSUBTRANS(). */
190 : : "pg_subtrans",
191 : :
192 : : /* end of list */
193 : : NULL
194 : : };
195 : :
196 : : /*
197 : : * List of files excluded from backups.
198 : : */
199 : : static const struct exclude_list_item excludeFiles[] =
200 : : {
201 : : /* Skip auto conf temporary file. */
202 : : {PG_AUTOCONF_FILENAME ".tmp", false},
203 : :
204 : : /* Skip current log file temporary file */
205 : : {LOG_METAINFO_DATAFILE_TMP, false},
206 : :
207 : : /*
208 : : * Skip relation cache because it is rebuilt on startup. This includes
209 : : * temporary files.
210 : : */
211 : : {RELCACHE_INIT_FILENAME, true},
212 : :
213 : : /*
214 : : * backup_label and tablespace_map should not exist in a running cluster
215 : : * capable of doing an online backup, but exclude them just in case.
216 : : */
217 : : {BACKUP_LABEL_FILE, false},
218 : : {TABLESPACE_MAP, false},
219 : :
220 : : /*
221 : : * If there's a backup_manifest, it belongs to a backup that was used to
222 : : * start this server. It is *not* correct for this backup. Our
223 : : * backup_manifest is injected into the backup separately if users want
224 : : * it.
225 : : */
226 : : {"backup_manifest", false},
227 : :
228 : : {"postmaster.pid", false},
229 : : {"postmaster.opts", false},
230 : :
231 : : /* end of list */
232 : : {NULL, false}
233 : : };
234 : :
235 : : /*
236 : : * Actually do a base backup for the specified tablespaces.
237 : : *
238 : : * This is split out mainly to avoid complaints about "variable might be
239 : : * clobbered by longjmp" from stupider versions of gcc.
240 : : */
241 : : static void
981 rhaas@postgresql.org 242 :CBC 186 : perform_base_backup(basebackup_options *opt, bbsink *sink,
243 : : IncrementalBackupInfo *ib)
244 : : {
245 : : bbsink_state state;
246 : : XLogRecPtr endptr;
247 : : TimeLineID endtli;
248 : : backup_manifest_info manifest;
249 : : BackupState *backup_state;
250 : : StringInfoData tablespace_map;
251 : :
252 : : /* Initial backup state, insofar as we know it now. */
1756 253 : 186 : state.tablespaces = NIL;
254 : 186 : state.tablespace_num = 0;
255 : 186 : state.bytes_done = 0;
256 : 186 : state.bytes_total = 0;
257 : 186 : state.bytes_total_is_valid = false;
258 : :
259 : : /* we're going to use a BufFile, so we need a ResourceOwner */
688 andres@anarazel.de 260 [ - + ]: 186 : Assert(AuxProcessResourceOwner != NULL);
261 [ + + - + ]: 186 : Assert(CurrentResourceOwner == AuxProcessResourceOwner ||
262 : : CurrentResourceOwner == NULL);
263 : 186 : CurrentResourceOwner = AuxProcessResourceOwner;
264 : :
5005 heikki.linnakangas@i 265 : 186 : backup_started_in_recovery = RecoveryInProgress();
266 : :
2317 rhaas@postgresql.org 267 : 186 : InitializeBackupManifest(&manifest, opt->manifest,
268 : : opt->manifest_checksum_type);
269 : :
3068 magnus@hagander.net 270 : 186 : total_checksum_failures = 0;
271 : :
272 : : /* Allocate backup related variables. */
260 michael@paquier.xyz 273 : 186 : backup_state = palloc0_object(BackupState);
294 drowley@postgresql.o 274 : 186 : initStringInfo(&tablespace_map);
275 : :
1756 rhaas@postgresql.org 276 : 186 : basebackup_progress_wait_checkpoint();
1431 michael@paquier.xyz 277 : 186 : do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
278 : : backup_state, &tablespace_map);
279 : :
280 : 186 : state.startptr = backup_state->startpoint;
281 : 186 : state.starttli = backup_state->starttli;
282 : :
283 : : /*
284 : : * Once do_pg_backup_start has been called, ensure that any failure causes
285 : : * us to abort the backup so we don't "leak" a backup counter. For this
286 : : * reason, *all* functionality between do_pg_backup_start() and the end of
287 : : * do_pg_backup_stop() should be inside the error cleanup block!
288 : : */
289 : :
2443 rhaas@postgresql.org 290 [ + + ]: 186 : PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
291 : : {
292 : : ListCell *lc;
293 : : tablespaceinfo *newti;
294 : :
295 : : /* If this is an incremental backup, execute preparatory steps. */
981 296 [ + + ]: 186 : if (ib != NULL)
297 : 12 : PrepareForIncrementalBackup(ib, backup_state);
298 : :
299 : : /* Add a node for the base directory at the end */
260 michael@paquier.xyz 300 : 186 : newti = palloc0_object(tablespaceinfo);
1422 drowley@postgresql.o 301 : 186 : newti->size = -1;
302 : 186 : state.tablespaces = lappend(state.tablespaces, newti);
303 : :
304 : : /*
305 : : * Calculate the total backup size by summing up the size of each
306 : : * tablespace
307 : : */
2368 fujii@postgresql.org 308 [ + - ]: 186 : if (opt->progress)
309 : : {
1756 rhaas@postgresql.org 310 : 186 : basebackup_progress_estimate_backup_size();
311 : :
312 [ + - + + : 409 : foreach(lc, state.tablespaces)
+ + ]
313 : : {
2368 fujii@postgresql.org 314 : 223 : tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
315 : :
2262 rhaas@postgresql.org 316 [ + + ]: 223 : if (tmp->path == NULL)
1756 317 : 186 : tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
318 : : true, NULL, InvalidOid, NULL);
319 : : else
320 : 37 : tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
321 : : NULL, NULL);
322 : 223 : state.bytes_total += tmp->size;
323 : : }
324 : 186 : state.bytes_total_is_valid = true;
325 : : }
326 : :
327 : : /* notify basebackup sink about start of backup */
328 : 186 : bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
329 : :
330 : : /*
331 : : * Allow tests to hold the backup after the starting checkpoint but
332 : : * before any file data is sent.
333 : : */
9 dgustafsson@postgres 334 : 186 : INJECTION_POINT("basebackup-before-send-files", NULL);
335 : :
336 : : /* Send off our tablespaces one by one */
1756 rhaas@postgresql.org 337 [ + - + + : 404 : foreach(lc, state.tablespaces)
+ + ]
338 : : {
5707 tgl@sss.pgh.pa.us 339 : 223 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
340 : :
5328 simon@2ndQuadrant.co 341 [ + + ]: 223 : if (ti->path == NULL)
342 : : {
343 : : struct stat statbuf;
1933 tgl@sss.pgh.pa.us 344 : 186 : bool sendtblspclinks = true;
345 : : char *backup_label;
346 : :
1756 rhaas@postgresql.org 347 : 186 : bbsink_begin_archive(sink, "base.tar");
348 : :
349 : : /* In the main tar, include the backup_label first... */
1431 michael@paquier.xyz 350 : 186 : backup_label = build_backup_content(backup_state, false);
351 : 186 : sendFileWithContent(sink, BACKUP_LABEL_FILE,
352 : : backup_label, -1, &manifest);
353 : 186 : pfree(backup_label);
354 : :
355 : : /* Then the tablespace_map file, if required... */
2262 rhaas@postgresql.org 356 [ + + ]: 186 : if (opt->sendtblspcmapfile)
357 : : {
1431 michael@paquier.xyz 358 : 28 : sendFileWithContent(sink, TABLESPACE_MAP,
294 drowley@postgresql.o 359 : 28 : tablespace_map.data, -1, &manifest);
2262 rhaas@postgresql.org 360 : 28 : sendtblspclinks = false;
361 : : }
362 : :
363 : : /* Then the bulk of the files... */
1756 364 : 186 : sendDir(sink, ".", 1, false, state.tablespaces,
365 : : sendtblspclinks, &manifest, InvalidOid, ib);
366 : :
367 : : /* ... and pg_control after everything else. */
5328 simon@2ndQuadrant.co 368 [ - + ]: 181 : if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0)
5328 simon@2ndQuadrant.co 369 [ # # ]:UBC 0 : ereport(ERROR,
370 : : (errcode_for_file_access(),
371 : : errmsg("could not stat file \"%s\": %m",
372 : : XLOG_CONTROL_FILE)));
1756 rhaas@postgresql.org 373 :CBC 181 : sendFile(sink, XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf,
374 : : false, InvalidOid, InvalidOid,
375 : : InvalidRelFileNumber, 0, &manifest, 0, NULL, 0);
376 : : }
377 : : else
378 : : {
1039 379 : 37 : char *archive_name = psprintf("%u.tar", ti->oid);
380 : :
1756 381 : 37 : bbsink_begin_archive(sink, archive_name);
382 : :
981 383 : 37 : sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib);
384 : : }
385 : :
386 : : /*
387 : : * If we're including WAL, and this is the main data directory we
388 : : * don't treat this as the end of the tablespace. Instead, we will
389 : : * include the xlog files below and stop afterwards. This is safe
390 : : * since the main data directory is always sent *last*.
391 : : */
5688 magnus@hagander.net 392 [ + + + + ]: 218 : if (opt->includewal && ti->path == NULL)
393 : : {
1756 rhaas@postgresql.org 394 [ - + ]: 15 : Assert(lnext(state.tablespaces, lc) == NULL);
395 : : }
396 : : else
397 : : {
398 : : /* Properly terminate the tarfile. */
192 peter@eisentraut.org 399 : 203 : memset(sink->bbs_buffer, 0, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
400 : 203 : bbsink_archive_contents(sink, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
401 : :
402 : : /* OK, that's the end of the archive. */
1756 rhaas@postgresql.org 403 : 203 : bbsink_end_archive(sink);
404 : : }
405 : : }
406 : :
407 : 181 : basebackup_progress_wait_wal_archive(&state);
1431 michael@paquier.xyz 408 : 181 : do_pg_backup_stop(backup_state, !opt->nowait);
409 : :
410 : 181 : endptr = backup_state->stoppoint;
411 : 181 : endtli = backup_state->stoptli;
412 : :
413 : : /* Deallocate backup-related variables. */
294 drowley@postgresql.o 414 : 181 : pfree(tablespace_map.data);
1431 michael@paquier.xyz 415 : 181 : pfree(backup_state);
416 : : }
2443 rhaas@postgresql.org 417 [ - + ]: 182 : PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
418 : :
419 : :
5688 magnus@hagander.net 420 [ + + ]: 181 : if (opt->includewal)
421 : : {
422 : : /*
423 : : * We've left the last tar file "open", so we can now append the
424 : : * required WAL files to it.
425 : : */
426 : : char pathbuf[MAXPGPATH];
427 : : XLogSegNo segno;
428 : : XLogSegNo startsegno;
429 : : XLogSegNo endsegno;
430 : : struct stat statbuf;
4984 heikki.linnakangas@i 431 : 15 : List *historyFileList = NIL;
432 : 15 : List *walFileList = NIL;
433 : : char firstoff[MAXFNAMELEN];
434 : : char lastoff[MAXFNAMELEN];
435 : : DIR *dir;
436 : : struct dirent *de;
437 : : ListCell *lc;
438 : : TimeLineID tli;
439 : :
1756 rhaas@postgresql.org 440 : 15 : basebackup_progress_transfer_wal();
441 : :
442 : : /*
443 : : * I'd rather not worry about timelines here, so scan pg_wal and
444 : : * include all WAL files in the range between 'startptr' and 'endptr',
445 : : * regardless of the timeline the file is stamped with. If there are
446 : : * some spurious WAL files belonging to timelines that don't belong in
447 : : * this server's history, they will be included too. Normally there
448 : : * shouldn't be such files, but if there are, there's little harm in
449 : : * including them.
450 : : */
451 : 15 : XLByteToSeg(state.startptr, startsegno, wal_segment_size);
452 : 15 : XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
3264 andres@anarazel.de 453 : 15 : XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
1763 rhaas@postgresql.org 454 : 15 : XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
455 : :
3598 456 : 15 : dir = AllocateDir("pg_wal");
457 [ + + ]: 108 : while ((de = ReadDir(dir, "pg_wal")) != NULL)
458 : : {
459 : : /* Does it look like a WAL segment, and is it in the range? */
4129 heikki.linnakangas@i 460 [ + + ]: 93 : if (IsXLogFileName(de->d_name) &&
4984 461 [ + - ]: 33 : strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
462 [ + + ]: 33 : strcmp(de->d_name + 8, lastoff + 8) <= 0)
463 : : {
464 : 15 : walFileList = lappend(walFileList, pstrdup(de->d_name));
465 : : }
466 : : /* Does it look like a timeline history file? */
4129 467 [ - + ]: 78 : else if (IsTLHistoryFileName(de->d_name))
468 : : {
4984 heikki.linnakangas@i 469 :UBC 0 : historyFileList = lappend(historyFileList, pstrdup(de->d_name));
470 : : }
471 : : }
4984 heikki.linnakangas@i 472 :CBC 15 : FreeDir(dir);
473 : :
474 : : /*
475 : : * Before we go any further, check that none of the WAL segments we
476 : : * need were removed.
477 : : */
1756 rhaas@postgresql.org 478 : 15 : CheckXLogRemoved(startsegno, state.starttli);
479 : :
480 : : /*
481 : : * Sort the WAL filenames. We want to send the files in order from
482 : : * oldest to newest, to reduce the chance that a file is recycled
483 : : * before we get a chance to send it over.
484 : : */
2599 tgl@sss.pgh.pa.us 485 : 15 : list_sort(walFileList, compareWalFileNames);
486 : :
487 : : /*
488 : : * There must be at least one xlog file in the pg_wal directory, since
489 : : * we are doing backup-including-xlog.
490 : : */
491 [ - + ]: 15 : if (walFileList == NIL)
4751 magnus@hagander.net 492 [ # # ]:UBC 0 : ereport(ERROR,
493 : : (errmsg("could not find any WAL files")));
494 : :
495 : : /*
496 : : * Sanity check: the first and last segment should cover startptr and
497 : : * endptr, with no gaps in between.
498 : : */
2599 tgl@sss.pgh.pa.us 499 :CBC 15 : XLogFromFileName((char *) linitial(walFileList),
500 : : &tli, &segno, wal_segment_size);
4984 heikki.linnakangas@i 501 [ - + ]: 15 : if (segno != startsegno)
502 : : {
503 : : char startfname[MAXFNAMELEN];
504 : :
1756 rhaas@postgresql.org 505 :UBC 0 : XLogFileName(startfname, state.starttli, startsegno,
506 : : wal_segment_size);
4984 heikki.linnakangas@i 507 [ # # ]: 0 : ereport(ERROR,
508 : : (errmsg("could not find WAL file \"%s\"", startfname)));
509 : : }
2599 tgl@sss.pgh.pa.us 510 [ + - + + :CBC 30 : foreach(lc, walFileList)
+ + ]
511 : : {
512 : 15 : char *walFileName = (char *) lfirst(lc);
4838 bruce@momjian.us 513 : 15 : XLogSegNo currsegno = segno;
514 : 15 : XLogSegNo nextsegno = segno + 1;
515 : :
2599 tgl@sss.pgh.pa.us 516 : 15 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
4984 heikki.linnakangas@i 517 [ + - - + ]: 15 : if (!(nextsegno == segno || currsegno == segno))
518 : : {
519 : : char nextfname[MAXFNAMELEN];
520 : :
1763 rhaas@postgresql.org 521 :UBC 0 : XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
4984 heikki.linnakangas@i 522 [ # # ]: 0 : ereport(ERROR,
523 : : (errmsg("could not find WAL file \"%s\"", nextfname)));
524 : : }
525 : : }
4984 heikki.linnakangas@i 526 [ - + ]:CBC 15 : if (segno != endsegno)
527 : : {
528 : : char endfname[MAXFNAMELEN];
529 : :
1763 rhaas@postgresql.org 530 :UBC 0 : XLogFileName(endfname, endtli, endsegno, wal_segment_size);
4984 heikki.linnakangas@i 531 [ # # ]: 0 : ereport(ERROR,
532 : : (errmsg("could not find WAL file \"%s\"", endfname)));
533 : : }
534 : :
535 : : /* Ok, we have everything we need. Send the WAL files. */
2599 tgl@sss.pgh.pa.us 536 [ + - + + :CBC 30 : foreach(lc, walFileList)
+ + ]
537 : : {
538 : 15 : char *walFileName = (char *) lfirst(lc);
539 : : int fd;
540 : : ssize_t cnt;
4984 heikki.linnakangas@i 541 : 15 : pgoff_t len = 0;
542 : :
2599 tgl@sss.pgh.pa.us 543 : 15 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
544 : 15 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
545 : :
2262 rhaas@postgresql.org 546 : 15 : fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
547 [ - + ]: 15 : if (fd < 0)
548 : : {
2985 michael@paquier.xyz 549 :UBC 0 : int save_errno = errno;
550 : :
551 : : /*
552 : : * Most likely reason for this is that the file was already
553 : : * removed by a checkpoint, so check for that to get a better
554 : : * error message.
555 : : */
4984 heikki.linnakangas@i 556 : 0 : CheckXLogRemoved(segno, tli);
557 : :
2985 michael@paquier.xyz 558 : 0 : errno = save_errno;
4984 heikki.linnakangas@i 559 [ # # ]: 0 : ereport(ERROR,
560 : : (errcode_for_file_access(),
561 : : errmsg("could not open file \"%s\": %m", pathbuf)));
562 : : }
563 : :
2262 rhaas@postgresql.org 564 [ - + ]:CBC 15 : if (fstat(fd, &statbuf) != 0)
4984 heikki.linnakangas@i 565 [ # # ]:UBC 0 : ereport(ERROR,
566 : : (errcode_for_file_access(),
567 : : errmsg("could not stat file \"%s\": %m",
568 : : pathbuf)));
3264 andres@anarazel.de 569 [ - + ]:CBC 15 : if (statbuf.st_size != wal_segment_size)
570 : : {
4984 heikki.linnakangas@i 571 :UBC 0 : CheckXLogRemoved(segno, tli);
572 [ # # ]: 0 : ereport(ERROR,
573 : : (errcode_for_file_access(),
574 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
575 : : }
576 : :
577 : : /* send the WAL file itself */
1756 rhaas@postgresql.org 578 :CBC 15 : _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
579 : :
580 : 15 : while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
581 : 7680 : Min(sink->bbs_buffer_length,
582 : : wal_segment_size - len),
2262 583 [ + - ]: 7680 : len, pathbuf, true)) > 0)
584 : : {
4984 heikki.linnakangas@i 585 : 7680 : CheckXLogRemoved(segno, tli);
1756 rhaas@postgresql.org 586 : 7680 : bbsink_archive_contents(sink, cnt);
587 : :
4984 heikki.linnakangas@i 588 : 7680 : len += cnt;
589 : :
3264 andres@anarazel.de 590 [ + + ]: 7680 : if (len == wal_segment_size)
4984 heikki.linnakangas@i 591 : 15 : break;
592 : : }
593 : :
3264 andres@anarazel.de 594 [ - + ]: 15 : if (len != wal_segment_size)
595 : : {
4984 heikki.linnakangas@i 596 :UBC 0 : CheckXLogRemoved(segno, tli);
597 [ # # ]: 0 : ereport(ERROR,
598 : : (errcode_for_file_access(),
599 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
600 : : }
601 : :
602 : : /*
603 : : * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
604 : : * for padding.
605 : : */
2316 rhaas@postgresql.org 606 [ - + ]:CBC 15 : Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
607 : :
2262 608 : 15 : CloseTransientFile(fd);
609 : :
610 : : /*
611 : : * Mark file as archived, otherwise files can get archived again
612 : : * after promotion of a new node. This is in line with
613 : : * walreceiver.c always doing an XLogArchiveForceDone() after a
614 : : * complete segment.
615 : : */
2599 tgl@sss.pgh.pa.us 616 : 15 : StatusFilePath(pathbuf, walFileName, ".done");
1018 michael@paquier.xyz 617 : 15 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
618 : : }
619 : :
620 : : /*
621 : : * Send timeline history files too. Only the latest timeline history
622 : : * file is required for recovery, and even that only if there happens
623 : : * to be a timeline switch in the first WAL segment that contains the
624 : : * checkpoint record, or if we're taking a base backup from a standby
625 : : * server and the target timeline changes while the backup is taken.
626 : : * But they are small and highly useful for debugging purposes, so
627 : : * better include them all, always.
628 : : */
4984 heikki.linnakangas@i 629 [ - + - - : 15 : foreach(lc, historyFileList)
- + ]
630 : : {
4838 bruce@momjian.us 631 :UBC 0 : char *fname = lfirst(lc);
632 : :
4984 heikki.linnakangas@i 633 : 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
634 : :
635 [ # # ]: 0 : if (lstat(pathbuf, &statbuf) != 0)
636 [ # # ]: 0 : ereport(ERROR,
637 : : (errcode_for_file_access(),
638 : : errmsg("could not stat file \"%s\": %m", pathbuf)));
639 : :
1039 rhaas@postgresql.org 640 : 0 : sendFile(sink, pathbuf, pathbuf, &statbuf, false,
641 : : InvalidOid, InvalidOid, InvalidRelFileNumber, 0,
642 : : &manifest, 0, NULL, 0);
643 : :
644 : : /* unconditionally mark file as archived */
4254 andres@anarazel.de 645 : 0 : StatusFilePath(pathbuf, fname, ".done");
1018 michael@paquier.xyz 646 : 0 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
647 : : }
648 : :
649 : : /* Properly terminate the tar file. */
192 peter@eisentraut.org 650 :CBC 15 : memset(sink->bbs_buffer, 0, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
651 : 15 : bbsink_archive_contents(sink, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
652 : :
653 : : /* OK, that's the end of the archive. */
1756 rhaas@postgresql.org 654 : 15 : bbsink_end_archive(sink);
655 : : }
656 : :
657 : 181 : AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
658 : : endptr, endtli);
659 : :
660 : 181 : SendBackupManifest(&manifest, sink);
661 : :
662 : 181 : bbsink_end_backup(sink, endptr, endtli);
663 : :
3068 magnus@hagander.net 664 [ + + ]: 181 : if (total_checksum_failures)
665 : : {
8 michael@paquier.xyz 666 [ + - ]: 3 : ereport(WARNING,
667 : : (errmsg_plural("%lld total checksum verification failure",
668 : : "%lld total checksum verification failures",
669 : : total_checksum_failures,
670 : : total_checksum_failures)));
671 : :
3068 magnus@hagander.net 672 [ + - ]: 3 : ereport(ERROR,
673 : : (errcode(ERRCODE_DATA_CORRUPTED),
674 : : errmsg("checksum verification failure during base backup")));
675 : : }
676 : :
677 : : /*
678 : : * Make sure to free the manifest before the resource owners as manifests
679 : : * use cryptohash contexts that may depend on resource owners (like
680 : : * OpenSSL).
681 : : */
2092 michael@paquier.xyz 682 : 178 : FreeBackupManifest(&manifest);
683 : :
684 : : /* clean up the resource owner we created */
688 andres@anarazel.de 685 : 178 : ReleaseAuxProcessResources(true);
5707 tgl@sss.pgh.pa.us 686 : 178 : }
687 : :
688 : : /*
689 : : * list_sort comparison function, to compare log/seg portion of WAL segment
690 : : * filenames, ignoring the timeline portion.
691 : : */
692 : : static int
2599 tgl@sss.pgh.pa.us 693 :UBC 0 : compareWalFileNames(const ListCell *a, const ListCell *b)
694 : : {
695 : 0 : char *fna = (char *) lfirst(a);
696 : 0 : char *fnb = (char *) lfirst(b);
697 : :
4984 heikki.linnakangas@i 698 : 0 : return strcmp(fna + 8, fnb + 8);
699 : : }
700 : :
701 : : /*
702 : : * Parse the base backup options passed down by the parser
703 : : */
704 : : static void
5695 magnus@hagander.net 705 :CBC 203 : parse_basebackup_options(List *options, basebackup_options *opt)
706 : : {
707 : : ListCell *lopt;
708 : 203 : bool o_label = false;
709 : 203 : bool o_progress = false;
1787 rhaas@postgresql.org 710 : 203 : bool o_checkpoint = false;
5678 magnus@hagander.net 711 : 203 : bool o_nowait = false;
5688 712 : 203 : bool o_wal = false;
981 rhaas@postgresql.org 713 : 203 : bool o_incremental = false;
4564 alvherre@alvh.no-ip. 714 : 203 : bool o_maxrate = false;
4125 andrew@dunslane.net 715 : 203 : bool o_tablespace_map = false;
3068 magnus@hagander.net 716 : 203 : bool o_noverify_checksums = false;
2337 rhaas@postgresql.org 717 : 203 : bool o_manifest = false;
718 : 203 : bool o_manifest_checksums = false;
1682 719 : 203 : bool o_target = false;
1745 720 : 203 : bool o_target_detail = false;
1626 721 : 203 : char *target_str = NULL;
722 : 203 : char *target_detail_str = NULL;
1676 723 : 203 : bool o_compression = false;
1618 724 : 203 : bool o_compression_detail = false;
725 : 203 : char *compression_detail_str = NULL;
726 : :
5693 magnus@hagander.net 727 [ + - + - : 2233 : MemSet(opt, 0, sizeof(*opt));
+ - + - +
+ ]
2337 rhaas@postgresql.org 728 : 203 : opt->manifest = MANIFEST_OPTION_NO;
729 : 203 : opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
1598 michael@paquier.xyz 730 : 203 : opt->compression = PG_COMPRESSION_NONE;
731 : 203 : opt->compression_specification.algorithm = PG_COMPRESSION_NONE;
732 : :
5695 magnus@hagander.net 733 [ + - + + : 1523 : foreach(lopt, options)
+ + ]
734 : : {
735 : 1323 : DefElem *defel = (DefElem *) lfirst(lopt);
736 : :
737 [ + + ]: 1323 : if (strcmp(defel->defname, "label") == 0)
738 : : {
739 [ - + ]: 203 : if (o_label)
5695 magnus@hagander.net 740 [ # # ]:UBC 0 : ereport(ERROR,
741 : : (errcode(ERRCODE_SYNTAX_ERROR),
742 : : errmsg("duplicate option \"%s\"", defel->defname)));
1787 rhaas@postgresql.org 743 :CBC 203 : opt->label = defGetString(defel);
5695 magnus@hagander.net 744 : 203 : o_label = true;
745 : : }
746 [ + + ]: 1120 : else if (strcmp(defel->defname, "progress") == 0)
747 : : {
748 [ - + ]: 203 : if (o_progress)
5695 magnus@hagander.net 749 [ # # ]:UBC 0 : ereport(ERROR,
750 : : (errcode(ERRCODE_SYNTAX_ERROR),
751 : : errmsg("duplicate option \"%s\"", defel->defname)));
1787 rhaas@postgresql.org 752 :CBC 203 : opt->progress = defGetBoolean(defel);
5695 magnus@hagander.net 753 : 203 : o_progress = true;
754 : : }
1787 rhaas@postgresql.org 755 [ + + ]: 917 : else if (strcmp(defel->defname, "checkpoint") == 0)
756 : : {
757 : 193 : char *optval = defGetString(defel);
758 : :
759 [ - + ]: 193 : if (o_checkpoint)
5695 magnus@hagander.net 760 [ # # ]:UBC 0 : ereport(ERROR,
761 : : (errcode(ERRCODE_SYNTAX_ERROR),
762 : : errmsg("duplicate option \"%s\"", defel->defname)));
1787 rhaas@postgresql.org 763 [ + - ]:CBC 193 : if (pg_strcasecmp(optval, "fast") == 0)
764 : 193 : opt->fastcheckpoint = true;
1787 rhaas@postgresql.org 765 [ # # ]:UBC 0 : else if (pg_strcasecmp(optval, "spread") == 0)
766 : 0 : opt->fastcheckpoint = false;
767 : : else
768 [ # # ]: 0 : ereport(ERROR,
769 : : (errcode(ERRCODE_SYNTAX_ERROR),
770 : : errmsg("unrecognized checkpoint type: \"%s\"",
771 : : optval)));
1787 rhaas@postgresql.org 772 :CBC 193 : o_checkpoint = true;
773 : : }
774 [ + + ]: 724 : else if (strcmp(defel->defname, "wait") == 0)
775 : : {
5678 magnus@hagander.net 776 [ - + ]: 194 : if (o_nowait)
5678 magnus@hagander.net 777 [ # # ]:UBC 0 : ereport(ERROR,
778 : : (errcode(ERRCODE_SYNTAX_ERROR),
779 : : errmsg("duplicate option \"%s\"", defel->defname)));
1787 rhaas@postgresql.org 780 :CBC 194 : opt->nowait = !defGetBoolean(defel);
5678 magnus@hagander.net 781 : 194 : o_nowait = true;
782 : : }
5688 783 [ + + ]: 530 : else if (strcmp(defel->defname, "wal") == 0)
784 : : {
785 [ - + ]: 19 : if (o_wal)
5688 magnus@hagander.net 786 [ # # ]:UBC 0 : ereport(ERROR,
787 : : (errcode(ERRCODE_SYNTAX_ERROR),
788 : : errmsg("duplicate option \"%s\"", defel->defname)));
1787 rhaas@postgresql.org 789 :CBC 19 : opt->includewal = defGetBoolean(defel);
5688 magnus@hagander.net 790 : 19 : o_wal = true;
791 : : }
981 rhaas@postgresql.org 792 [ + + ]: 511 : else if (strcmp(defel->defname, "incremental") == 0)
793 : : {
794 [ - + ]: 12 : if (o_incremental)
981 rhaas@postgresql.org 795 [ # # ]:UBC 0 : ereport(ERROR,
796 : : (errcode(ERRCODE_SYNTAX_ERROR),
797 : : errmsg("duplicate option \"%s\"", defel->defname)));
981 rhaas@postgresql.org 798 :CBC 12 : opt->incremental = defGetBoolean(defel);
799 [ + - - + ]: 12 : if (opt->incremental && !summarize_wal)
981 rhaas@postgresql.org 800 [ # # ]:UBC 0 : ereport(ERROR,
801 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
802 : : errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));
981 rhaas@postgresql.org 803 :CBC 12 : o_incremental = true;
804 : : }
4564 alvherre@alvh.no-ip. 805 [ + + ]: 499 : else if (strcmp(defel->defname, "max_rate") == 0)
806 : : {
807 : : int64 maxrate;
808 : :
809 [ - + ]: 1 : if (o_maxrate)
4564 alvherre@alvh.no-ip. 810 [ # # ]:UBC 0 : ereport(ERROR,
811 : : (errcode(ERRCODE_SYNTAX_ERROR),
812 : : errmsg("duplicate option \"%s\"", defel->defname)));
813 : :
1787 rhaas@postgresql.org 814 :CBC 1 : maxrate = defGetInt64(defel);
4564 alvherre@alvh.no-ip. 815 [ + - - + ]: 1 : if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
4564 alvherre@alvh.no-ip. 816 [ # # ]:UBC 0 : ereport(ERROR,
817 : : (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
818 : : errmsg("%" PRId64 " is outside the valid range for parameter \"%s\" (%d .. %d)",
819 : : maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
820 : :
4564 alvherre@alvh.no-ip. 821 :CBC 1 : opt->maxrate = (uint32) maxrate;
822 : 1 : o_maxrate = true;
823 : : }
4125 andrew@dunslane.net 824 [ + + ]: 498 : else if (strcmp(defel->defname, "tablespace_map") == 0)
825 : : {
826 [ - + ]: 34 : if (o_tablespace_map)
4125 andrew@dunslane.net 827 [ # # ]:UBC 0 : ereport(ERROR,
828 : : (errcode(ERRCODE_SYNTAX_ERROR),
829 : : errmsg("duplicate option \"%s\"", defel->defname)));
1787 rhaas@postgresql.org 830 :CBC 34 : opt->sendtblspcmapfile = defGetBoolean(defel);
4125 andrew@dunslane.net 831 : 34 : o_tablespace_map = true;
832 : : }
1787 rhaas@postgresql.org 833 [ + + ]: 464 : else if (strcmp(defel->defname, "verify_checksums") == 0)
834 : : {
3068 magnus@hagander.net 835 [ - + ]: 1 : if (o_noverify_checksums)
3068 magnus@hagander.net 836 [ # # ]:UBC 0 : ereport(ERROR,
837 : : (errcode(ERRCODE_SYNTAX_ERROR),
838 : : errmsg("duplicate option \"%s\"", defel->defname)));
1787 rhaas@postgresql.org 839 :CBC 1 : noverify_checksums = !defGetBoolean(defel);
3068 magnus@hagander.net 840 : 1 : o_noverify_checksums = true;
841 : : }
2337 rhaas@postgresql.org 842 [ + + ]: 463 : else if (strcmp(defel->defname, "manifest") == 0)
843 : : {
1787 844 : 202 : char *optval = defGetString(defel);
845 : : bool manifest_bool;
846 : :
2337 847 [ - + ]: 202 : if (o_manifest)
2337 rhaas@postgresql.org 848 [ # # ]:UBC 0 : ereport(ERROR,
849 : : (errcode(ERRCODE_SYNTAX_ERROR),
850 : : errmsg("duplicate option \"%s\"", defel->defname)));
2337 rhaas@postgresql.org 851 [ + + ]:CBC 202 : if (parse_bool(optval, &manifest_bool))
852 : : {
853 [ + - ]: 201 : if (manifest_bool)
854 : 201 : opt->manifest = MANIFEST_OPTION_YES;
855 : : else
2337 rhaas@postgresql.org 856 :UBC 0 : opt->manifest = MANIFEST_OPTION_NO;
857 : : }
2337 rhaas@postgresql.org 858 [ + - ]:CBC 1 : else if (pg_strcasecmp(optval, "force-encode") == 0)
859 : 1 : opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
860 : : else
2337 rhaas@postgresql.org 861 [ # # ]:UBC 0 : ereport(ERROR,
862 : : (errcode(ERRCODE_SYNTAX_ERROR),
863 : : errmsg("unrecognized manifest option: \"%s\"",
864 : : optval)));
2337 rhaas@postgresql.org 865 :CBC 202 : o_manifest = true;
866 : : }
867 [ + + ]: 261 : else if (strcmp(defel->defname, "manifest_checksums") == 0)
868 : : {
1787 869 : 14 : char *optval = defGetString(defel);
870 : :
2337 871 [ - + ]: 14 : if (o_manifest_checksums)
2337 rhaas@postgresql.org 872 [ # # ]:UBC 0 : ereport(ERROR,
873 : : (errcode(ERRCODE_SYNTAX_ERROR),
874 : : errmsg("duplicate option \"%s\"", defel->defname)));
2337 rhaas@postgresql.org 875 [ + + ]:CBC 14 : if (!pg_checksum_parse_type(optval,
876 : : &opt->manifest_checksum_type))
877 [ + - ]: 2 : ereport(ERROR,
878 : : (errcode(ERRCODE_SYNTAX_ERROR),
879 : : errmsg("unrecognized checksum algorithm: \"%s\"",
880 : : optval)));
881 : 12 : o_manifest_checksums = true;
882 : : }
1682 883 [ + + ]: 247 : else if (strcmp(defel->defname, "target") == 0)
884 : : {
885 [ - + ]: 201 : if (o_target)
1682 rhaas@postgresql.org 886 [ # # ]:UBC 0 : ereport(ERROR,
887 : : (errcode(ERRCODE_SYNTAX_ERROR),
888 : : errmsg("duplicate option \"%s\"", defel->defname)));
1626 rhaas@postgresql.org 889 :CBC 201 : target_str = defGetString(defel);
1682 890 : 201 : o_target = true;
891 : : }
1745 892 [ + + ]: 46 : else if (strcmp(defel->defname, "target_detail") == 0)
893 : : {
894 : 8 : char *optval = defGetString(defel);
895 : :
896 [ - + ]: 8 : if (o_target_detail)
1745 rhaas@postgresql.org 897 [ # # ]:UBC 0 : ereport(ERROR,
898 : : (errcode(ERRCODE_SYNTAX_ERROR),
899 : : errmsg("duplicate option \"%s\"", defel->defname)));
1626 rhaas@postgresql.org 900 :CBC 8 : target_detail_str = optval;
1745 901 : 8 : o_target_detail = true;
902 : : }
1676 903 [ + + ]: 38 : else if (strcmp(defel->defname, "compression") == 0)
904 : : {
905 : 26 : char *optval = defGetString(defel);
906 : :
907 [ - + ]: 26 : if (o_compression)
1676 rhaas@postgresql.org 908 [ # # ]:UBC 0 : ereport(ERROR,
909 : : (errcode(ERRCODE_SYNTAX_ERROR),
910 : : errmsg("duplicate option \"%s\"", defel->defname)));
1598 michael@paquier.xyz 911 [ + + ]:CBC 26 : if (!parse_compress_algorithm(optval, &opt->compression))
1676 rhaas@postgresql.org 912 [ + - ]: 1 : ereport(ERROR,
913 : : (errcode(ERRCODE_SYNTAX_ERROR),
914 : : errmsg("unrecognized compression algorithm: \"%s\"",
915 : : optval)));
916 : 25 : o_compression = true;
917 : : }
1618 918 [ + - ]: 12 : else if (strcmp(defel->defname, "compression_detail") == 0)
919 : : {
920 [ - + ]: 12 : if (o_compression_detail)
1676 rhaas@postgresql.org 921 [ # # ]:UBC 0 : ereport(ERROR,
922 : : (errcode(ERRCODE_SYNTAX_ERROR),
923 : : errmsg("duplicate option \"%s\"", defel->defname)));
1618 rhaas@postgresql.org 924 :CBC 12 : compression_detail_str = defGetString(defel);
925 : 12 : o_compression_detail = true;
926 : : }
927 : : else
1676 rhaas@postgresql.org 928 [ # # ]:UBC 0 : ereport(ERROR,
929 : : (errcode(ERRCODE_SYNTAX_ERROR),
930 : : errmsg("unrecognized base backup option: \"%s\"",
931 : : defel->defname)));
932 : : }
933 : :
5695 magnus@hagander.net 934 [ - + ]:CBC 200 : if (opt->label == NULL)
5695 magnus@hagander.net 935 :UBC 0 : opt->label = "base backup";
2337 rhaas@postgresql.org 936 [ + + ]:CBC 200 : if (opt->manifest == MANIFEST_OPTION_NO)
937 : : {
938 [ - + ]: 1 : if (o_manifest_checksums)
2337 rhaas@postgresql.org 939 [ # # ]:UBC 0 : ereport(ERROR,
940 : : (errcode(ERRCODE_SYNTAX_ERROR),
941 : : errmsg("manifest checksums require a backup manifest")));
2337 rhaas@postgresql.org 942 :CBC 1 : opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
943 : : }
944 : :
1626 945 [ - + ]: 200 : if (target_str == NULL)
946 : : {
1626 rhaas@postgresql.org 947 [ # # ]:UBC 0 : if (target_detail_str != NULL)
1745 948 [ # # ]: 0 : ereport(ERROR,
949 : : (errcode(ERRCODE_SYNTAX_ERROR),
950 : : errmsg("target detail cannot be used without target")));
1626 951 : 0 : opt->use_copytblspc = true;
952 : 0 : opt->send_to_client = true;
953 : : }
1626 rhaas@postgresql.org 954 [ + + ]:CBC 200 : else if (strcmp(target_str, "client") == 0)
955 : : {
956 [ - + ]: 186 : if (target_detail_str != NULL)
1745 rhaas@postgresql.org 957 [ # # ]:UBC 0 : ereport(ERROR,
958 : : (errcode(ERRCODE_SYNTAX_ERROR),
959 : : errmsg("target \"%s\" does not accept a target detail",
960 : : target_str)));
1626 rhaas@postgresql.org 961 :CBC 186 : opt->send_to_client = true;
962 : : }
963 : : else
964 : 12 : opt->target_handle =
965 : 14 : BaseBackupGetTargetHandle(target_str, target_detail_str);
966 : :
1618 967 [ + + - + ]: 198 : if (o_compression_detail && !o_compression)
1676 rhaas@postgresql.org 968 [ # # ]:UBC 0 : ereport(ERROR,
969 : : (errcode(ERRCODE_SYNTAX_ERROR),
970 : : errmsg("compression detail cannot be specified unless compression is enabled")));
971 : :
1618 rhaas@postgresql.org 972 [ + + ]:CBC 198 : if (o_compression)
973 : : {
974 : : char *error_detail;
975 : :
1598 michael@paquier.xyz 976 : 23 : parse_compress_specification(opt->compression, compression_detail_str,
977 : : &opt->compression_specification);
978 : : error_detail =
979 : 23 : validate_compress_specification(&opt->compression_specification);
1618 rhaas@postgresql.org 980 [ + + ]: 23 : if (error_detail != NULL)
981 [ + - ]: 9 : ereport(ERROR,
982 : : errcode(ERRCODE_SYNTAX_ERROR),
983 : : errmsg("invalid compression specification: %s",
984 : : error_detail));
985 : : }
5695 magnus@hagander.net 986 : 189 : }
987 : :
988 : :
989 : : /*
990 : : * SendBaseBackup() - send a complete base backup.
991 : : *
992 : : * The function will put the system into backup mode like pg_backup_start()
993 : : * does, so that the backup is consistent even though we read directly from
994 : : * the filesystem, bypassing the buffer cache.
995 : : */
996 : : void
981 rhaas@postgresql.org 997 : 204 : SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)
998 : : {
999 : : basebackup_options opt;
1000 : : bbsink *sink;
1507 fujii@postgresql.org 1001 : 204 : SessionBackupState status = get_backup_status();
1002 : :
1003 [ + + ]: 204 : if (status == SESSION_BACKUP_RUNNING)
1004 [ + - ]: 1 : ereport(ERROR,
1005 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1006 : : errmsg("a backup is already in progress in this session")));
1007 : :
5695 magnus@hagander.net 1008 : 203 : parse_basebackup_options(cmd->options, &opt);
1009 : :
5707 1010 : 189 : WalSndSetState(WALSNDSTATE_BACKUP);
1011 : :
5708 1012 [ + - ]: 189 : if (update_process_title)
1013 : : {
1014 : : char activitymsg[50];
1015 : :
1016 : 189 : snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
1017 : : opt.label);
2360 peter@eisentraut.org 1018 : 189 : set_ps_display(activitymsg);
1019 : : }
1020 : :
1021 : : /*
1022 : : * If we're asked to perform an incremental backup and the user has not
1023 : : * supplied a manifest, that's an ERROR.
1024 : : *
1025 : : * If we're asked to perform a full backup and the user did supply a
1026 : : * manifest, just ignore it.
1027 : : */
981 rhaas@postgresql.org 1028 [ + + ]: 189 : if (!opt.incremental)
1029 : 177 : ib = NULL;
1030 [ - + ]: 12 : else if (ib == NULL)
981 rhaas@postgresql.org 1031 [ # # ]:UBC 0 : ereport(ERROR,
1032 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1033 : : errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));
1034 : :
1035 : : /*
1036 : : * If the target is specifically 'client' then set up to stream the backup
1037 : : * to the client; otherwise, it's being sent someplace else and should not
1038 : : * be sent to the client. BaseBackupGetSink has the job of setting up a
1039 : : * sink to send the backup data wherever it needs to go.
1040 : : */
1626 rhaas@postgresql.org 1041 :CBC 189 : sink = bbsink_copystream_new(opt.send_to_client);
1042 [ + + ]: 189 : if (opt.target_handle != NULL)
1043 : 12 : sink = BaseBackupGetSink(opt.target_handle, sink);
1044 : :
1045 : : /* Set up network throttling, if client requested it */
1756 1046 [ + + ]: 186 : if (opt.maxrate > 0)
1047 : 1 : sink = bbsink_throttle_new(sink, opt.maxrate);
1048 : :
1049 : : /* Set up server-side compression, if client requested it */
1598 michael@paquier.xyz 1050 [ + + ]: 186 : if (opt.compression == PG_COMPRESSION_GZIP)
1618 rhaas@postgresql.org 1051 : 2 : sink = bbsink_gzip_new(sink, &opt.compression_specification);
1598 michael@paquier.xyz 1052 [ + + ]: 184 : else if (opt.compression == PG_COMPRESSION_LZ4)
1618 rhaas@postgresql.org 1053 : 3 : sink = bbsink_lz4_new(sink, &opt.compression_specification);
1598 michael@paquier.xyz 1054 [ - + ]: 181 : else if (opt.compression == PG_COMPRESSION_ZSTD)
1618 rhaas@postgresql.org 1055 :UBC 0 : sink = bbsink_zstd_new(sink, &opt.compression_specification);
1056 : :
1057 : : /* Set up progress reporting. */
387 msawada@postgresql.o 1058 :CBC 186 : sink = bbsink_progress_new(sink, opt.progress, opt.incremental);
1059 : :
1060 : : /*
1061 : : * Perform the base backup, but make sure we clean up the bbsink even if
1062 : : * an error occurs.
1063 : : */
1756 rhaas@postgresql.org 1064 [ + + ]: 186 : PG_TRY();
1065 : : {
981 1066 : 186 : perform_base_backup(&opt, sink, ib);
1067 : : }
1756 1068 : 4 : PG_FINALLY();
1069 : : {
1070 : 182 : bbsink_cleanup(sink);
1071 : : }
1072 [ + + ]: 182 : PG_END_TRY();
5684 magnus@hagander.net 1073 : 178 : }
1074 : :
1075 : : /*
1076 : : * Inject a file with given name and content in the output tar stream.
1077 : : *
1078 : : * "len" can optionally be set to an arbitrary length of data sent. If set
1079 : : * to -1, the content sent is treated as a string with strlen() as length.
1080 : : */
1081 : : static void
1756 rhaas@postgresql.org 1082 : 229 : sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1083 : : int len, backup_manifest_info *manifest)
1084 : : {
1085 : : struct stat statbuf;
1018 michael@paquier.xyz 1086 : 229 : int bytes_done = 0;
1087 : : pg_checksum_context checksum_ctx;
1088 : :
2094 1089 [ - + ]: 229 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
2094 michael@paquier.xyz 1090 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1091 : : filename);
1092 : :
1018 michael@paquier.xyz 1093 [ + - ]:CBC 229 : if (len < 0)
1094 : 229 : len = strlen(content);
1095 : :
1096 : : /*
1097 : : * Construct a stat struct for the file we're injecting in the tar.
1098 : : */
1099 : :
1100 : : /* Windows doesn't have the concept of uid and gid */
1101 : : #ifdef WIN32
1102 : : statbuf.st_uid = 0;
1103 : : statbuf.st_gid = 0;
1104 : : #else
5687 heikki.linnakangas@i 1105 : 229 : statbuf.st_uid = geteuid();
1106 : 229 : statbuf.st_gid = getegid();
1107 : : #endif
1108 : 229 : statbuf.st_mtime = time(NULL);
3064 sfrost@snowman.net 1109 : 229 : statbuf.st_mode = pg_file_create_mode;
5687 heikki.linnakangas@i 1110 : 229 : statbuf.st_size = len;
1111 : :
1756 rhaas@postgresql.org 1112 : 229 : _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1113 : :
213 peter@eisentraut.org 1114 [ - + ]: 229 : if (pg_checksum_update(&checksum_ctx, (const uint8 *) content, len) < 0)
1756 rhaas@postgresql.org 1115 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of file \"%s\"",
1116 : : filename);
1117 : :
1756 rhaas@postgresql.org 1118 [ + + ]:CBC 421 : while (bytes_done < len)
1119 : : {
1120 : 192 : size_t remaining = len - bytes_done;
1121 : 192 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1122 : :
1123 : 192 : memcpy(sink->bbs_buffer, content, nbytes);
1124 : 192 : bbsink_archive_contents(sink, nbytes);
1125 : 192 : bytes_done += nbytes;
1354 1126 : 192 : content += nbytes;
1127 : : }
1128 : :
1756 1129 : 229 : _tarWritePadding(sink, len);
1130 : :
1039 1131 : 229 : AddFileToBackupManifest(manifest, InvalidOid, filename, len,
2317 1132 : 229 : (pg_time_t) statbuf.st_mtime, &checksum_ctx);
5687 heikki.linnakangas@i 1133 : 229 : }
1134 : :
1135 : : /*
1136 : : * Include the tablespace directory pointed to by 'path' in the output tar
1137 : : * stream. If 'sizeonly' is true, we just calculate a total length and return
1138 : : * it, without actually sending anything.
1139 : : *
1140 : : * Only used to send auxiliary tablespaces, not PGDATA.
1141 : : */
1142 : : static int64
1039 rhaas@postgresql.org 1143 : 74 : sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
1144 : : backup_manifest_info *manifest, IncrementalBackupInfo *ib)
1145 : : {
1146 : : int64 size;
1147 : : char pathbuf[MAXPGPATH];
1148 : : struct stat statbuf;
1149 : :
1150 : : /*
1151 : : * 'path' points to the tablespace location, but we only want to include
1152 : : * the version directory in it that belongs to us.
1153 : : */
4903 heikki.linnakangas@i 1154 : 74 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1155 : : TABLESPACE_VERSION_DIRECTORY);
1156 : :
1157 : : /*
1158 : : * Store a directory entry in the tar file so we get the permissions
1159 : : * right.
1160 : : */
1161 [ - + ]: 74 : if (lstat(pathbuf, &statbuf) != 0)
1162 : : {
4903 heikki.linnakangas@i 1163 [ # # ]:UBC 0 : if (errno != ENOENT)
1164 [ # # ]: 0 : ereport(ERROR,
1165 : : (errcode_for_file_access(),
1166 : : errmsg("could not stat file or directory \"%s\": %m",
1167 : : pathbuf)));
1168 : :
1169 : : /* If the tablespace went away while scanning, it's no error. */
1170 : 0 : return 0;
1171 : : }
1172 : :
1756 rhaas@postgresql.org 1173 :CBC 74 : size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1174 : : sizeonly);
1175 : :
1176 : : /* Send all the files in the tablespace version directory */
1177 : 74 : size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1178 : : spcoid, ib);
1179 : :
4903 heikki.linnakangas@i 1180 : 74 : return size;
1181 : : }
1182 : :
1183 : : /*
1184 : : * Include all files from the given directory in the output tar stream. If
1185 : : * 'sizeonly' is true, we just calculate a total length and return it, without
1186 : : * actually sending anything.
1187 : : *
1188 : : * Omit any directory in the tablespaces list, to avoid backing up
1189 : : * tablespaces twice when they were created inside PGDATA.
1190 : : *
1191 : : * If sendtblspclinks is true, we need to include symlink
1192 : : * information in the tar file. If not, we can skip that
1193 : : * as it will be sent separately in the tablespace_map file.
1194 : : */
1195 : : static int64
1756 rhaas@postgresql.org 1196 : 6425 : sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1197 : : List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1198 : : Oid spcoid, IncrementalBackupInfo *ib)
1199 : : {
1200 : : DIR *dir;
1201 : : struct dirent *de;
1202 : : char pathbuf[MAXPGPATH * 2];
1203 : : struct stat statbuf;
5708 magnus@hagander.net 1204 : 6425 : int64 size = 0;
1205 : : const char *lastDir; /* Split last dir from parent path. */
1017 rhaas@postgresql.org 1206 : 6425 : bool isRelationDir = false; /* Does directory contain relations? */
981 1207 : 6425 : bool isGlobalDir = false;
1017 1208 : 6425 : Oid dboid = InvalidOid;
981 1209 : 6425 : BlockNumber *relative_block_numbers = NULL;
1210 : :
1211 : : /*
1212 : : * Since this array is relatively large, avoid putting it on the stack.
1213 : : * But we don't need it at all if this is not an incremental backup.
1214 : : */
1215 [ + + ]: 6425 : if (ib != NULL)
260 michael@paquier.xyz 1216 : 208 : relative_block_numbers = palloc_array(BlockNumber, RELSEG_SIZE);
1217 : :
1218 : : /*
1219 : : * Determine if the current path is a database directory that can contain
1220 : : * relations.
1221 : : *
1222 : : * Start by finding the location of the delimiter between the parent path
1223 : : * and the current path.
1224 : : */
3079 teodor@sigaev.ru 1225 : 6425 : lastDir = last_dir_separator(path);
1226 : :
1227 : : /* Does this path look like a database path (i.e. all digits)? */
1228 [ + + ]: 6425 : if (lastDir != NULL &&
1229 [ + + ]: 6053 : strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1230 : 1187 : {
1231 : : /* Part of path that contains the parent directory. */
3045 tgl@sss.pgh.pa.us 1232 : 1187 : int parentPathLen = lastDir - path;
1233 : :
1234 : : /*
1235 : : * Mark path as a database directory if the parent path is either
1236 : : * $PGDATA/base or a tablespace version path.
1237 : : */
3079 teodor@sigaev.ru 1238 [ + + ]: 1187 : if (strncmp(path, "./base", parentPathLen) == 0 ||
1239 [ + - ]: 52 : (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1240 [ + - ]: 52 : strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1241 : : TABLESPACE_VERSION_DIRECTORY,
1242 : : sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1243 : : {
1017 rhaas@postgresql.org 1244 : 1187 : isRelationDir = true;
1245 : 1187 : dboid = atooid(lastDir + 1);
1246 : : }
1247 : : }
1248 [ + + ]: 5238 : else if (strcmp(path, "./global") == 0)
1249 : : {
1250 : 369 : isRelationDir = true;
981 1251 : 369 : isGlobalDir = true;
1252 : : }
1253 : :
5708 magnus@hagander.net 1254 : 6425 : dir = AllocateDir(path);
1255 [ + + ]: 420001 : while ((de = ReadDir(dir, path)) != NULL)
1256 : : {
1257 : : int excludeIdx;
1258 : : bool excludeFound;
1017 rhaas@postgresql.org 1259 : 413584 : RelFileNumber relfilenumber = InvalidRelFileNumber;
1260 : 413584 : ForkNumber relForkNum = InvalidForkNumber;
1261 : 413584 : unsigned segno = 0;
1262 : 413584 : bool isRelationFile = false;
1263 : :
1264 : : /* Skip special stuff */
5708 magnus@hagander.net 1265 [ + + + + ]: 413584 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1266 : 18152 : continue;
1267 : :
1268 : : /* Skip temporary files */
heikki.linnakangas@i 1269 [ + + ]: 400746 : if (strncmp(de->d_name,
1270 : : PG_TEMP_FILE_PREFIX,
1271 : : strlen(PG_TEMP_FILE_PREFIX)) == 0)
1272 : 367 : continue;
1273 : :
1274 : : /* Skip macOS system files */
926 dgustafsson@postgres 1275 [ + + ]: 400379 : if (strcmp(de->d_name, ".DS_Store") == 0)
1276 : 68 : continue;
1277 : :
1278 : : /*
1279 : : * Check if the postmaster has signaled us to exit, and abort with an
1280 : : * error in that case. The error handler further up will call
1281 : : * do_pg_abort_backup() for us. Also check that if the backup was
1282 : : * started while still in recovery, the server wasn't promoted.
1283 : : * do_pg_backup_stop() will check that too, but it's better to stop
1284 : : * the backup early than continue to the end and fail there.
1285 : : */
5005 heikki.linnakangas@i 1286 [ + + ]: 400311 : CHECK_FOR_INTERRUPTS();
1287 [ - + ]: 400307 : if (RecoveryInProgress() != backup_started_in_recovery)
5704 magnus@hagander.net 1288 [ # # ]:UBC 0 : ereport(ERROR,
1289 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1290 : : errmsg("the standby was promoted during online backup"),
1291 : : errhint("This means that the backup being taken is corrupt "
1292 : : "and should not be used. "
1293 : : "Try taking another online backup.")));
1294 : :
1295 : : /* Scan for files that should be excluded */
3620 peter_e@gmx.net 1296 :CBC 400307 : excludeFound = false;
2376 michael@paquier.xyz 1297 [ + + ]: 3597277 : for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1298 : : {
1299 : 3198419 : int cmplen = strlen(excludeFiles[excludeIdx].name);
1300 : :
1301 [ + + ]: 3198419 : if (!excludeFiles[excludeIdx].match_prefix)
1302 : 2798242 : cmplen++;
1303 [ + + ]: 3198419 : if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1304 : : {
3620 peter_e@gmx.net 1305 [ + + ]: 1449 : elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1306 : 1449 : excludeFound = true;
1307 : 1449 : break;
1308 : : }
1309 : : }
1310 : :
1311 [ + + ]: 400307 : if (excludeFound)
5708 magnus@hagander.net 1312 : 1449 : continue;
1313 : :
1314 : : /*
1315 : : * If there could be non-temporary relation files in this directory,
1316 : : * try to parse the filename.
1317 : : */
1017 rhaas@postgresql.org 1318 [ + + ]: 398858 : if (isRelationDir)
1319 : : isRelationFile =
1320 : 385972 : parse_filename_for_nontemp_relation(de->d_name,
1321 : : &relfilenumber,
1322 : : &relForkNum, &segno);
1323 : :
1324 : : /* Exclude all forks for unlogged tables except the init fork */
1325 [ + + + + ]: 398858 : if (isRelationFile && relForkNum != INIT_FORKNUM)
1326 : : {
1327 : : char initForkFile[MAXPGPATH];
1328 : :
1329 : : /*
1330 : : * If any other type of fork, check if there is an init fork with
1331 : : * the same RelFileNumber. If so, the file can be excluded.
1332 : : */
1333 : 382847 : snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",
1334 : : path, relfilenumber);
1335 : :
1336 [ + + ]: 382847 : if (lstat(initForkFile, &statbuf) == 0)
1337 : : {
1338 [ - + ]: 81 : elog(DEBUG2,
1339 : : "unlogged relation file \"%s\" excluded from backup",
1340 : : de->d_name);
1341 : :
1342 : 81 : continue;
1343 : : }
1344 : : }
1345 : :
1346 : : /* Exclude temporary relations */
1347 [ + + + + ]: 398777 : if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name))
1348 : : {
3075 teodor@sigaev.ru 1349 [ - + ]: 36 : elog(DEBUG2,
1350 : : "temporary relation file \"%s\" excluded from backup",
1351 : : de->d_name);
1352 : :
1353 : 36 : continue;
1354 : : }
1355 : :
3425 peter_e@gmx.net 1356 : 398741 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1357 : :
1358 : : /* Skip pg_control here to back up it last */
507 fujii@postgresql.org 1359 [ + + ]: 398741 : if (strcmp(pathbuf, "./" XLOG_CONTROL_FILE) == 0)
5328 simon@2ndQuadrant.co 1360 : 368 : continue;
1361 : :
5708 magnus@hagander.net 1362 [ - + ]: 398373 : if (lstat(pathbuf, &statbuf) != 0)
1363 : : {
5708 magnus@hagander.net 1364 [ # # ]:UBC 0 : if (errno != ENOENT)
1365 [ # # ]: 0 : ereport(ERROR,
1366 : : (errcode_for_file_access(),
1367 : : errmsg("could not stat file or directory \"%s\": %m",
1368 : : pathbuf)));
1369 : :
1370 : : /* If the file went away while scanning, it's not an error. */
1371 : 0 : continue;
1372 : : }
1373 : :
1374 : : /* Scan for directories whose contents should be excluded */
3620 peter_e@gmx.net 1375 :CBC 398373 : excludeFound = false;
1376 [ + + ]: 3176658 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1377 : : {
1378 [ + + ]: 2780863 : if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1379 : : {
1380 [ + + ]: 2578 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1780 rhaas@postgresql.org 1381 : 2578 : convert_link_to_directory(pathbuf, &statbuf);
1756 1382 : 2578 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1383 : : &statbuf, sizeonly);
3620 peter_e@gmx.net 1384 : 2578 : excludeFound = true;
1385 : 2578 : break;
1386 : : }
1387 : : }
1388 : :
1389 [ + + ]: 398373 : if (excludeFound)
1390 : 2578 : continue;
1391 : :
1392 : : /*
1393 : : * We can skip pg_wal, the WAL segments need to be fetched from the
1394 : : * WAL archive anyway. But include it as an empty directory anyway, so
1395 : : * we get permissions right.
1396 : : */
3598 rhaas@postgresql.org 1397 [ + + ]: 395795 : if (strcmp(pathbuf, "./pg_wal") == 0)
1398 : : {
1399 : : /* If pg_wal is a symlink, write it as a directory anyway */
1780 1400 : 367 : convert_link_to_directory(pathbuf, &statbuf);
1756 1401 : 367 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1402 : : &statbuf, sizeonly);
1403 : :
1404 : : /*
1405 : : * Also send archive_status and summaries directories (by
1406 : : * hackishly reusing statbuf from above ...).
1407 : : */
1408 : 367 : size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1409 : : &statbuf, sizeonly);
981 1410 : 367 : size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,
1411 : : &statbuf, sizeonly);
1412 : :
3598 1413 : 367 : continue; /* don't recurse into pg_wal */
1414 : : }
1415 : :
1416 : : /* Allow symbolic links in pg_tblspc only */
1482 tmunro@postgresql.or 1417 [ + + + + ]: 395428 : if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
5708 magnus@hagander.net 1418 : 39 : {
1419 : : char linkpath[MAXPGPATH];
1420 : : ssize_t rllen;
1421 : :
5377 tgl@sss.pgh.pa.us 1422 : 39 : rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1423 [ - + ]: 39 : if (rllen < 0)
5708 magnus@hagander.net 1424 [ # # ]:UBC 0 : ereport(ERROR,
1425 : : (errcode_for_file_access(),
1426 : : errmsg("could not read symbolic link \"%s\": %m",
1427 : : pathbuf)));
5377 tgl@sss.pgh.pa.us 1428 [ - + ]:CBC 39 : if (rllen >= sizeof(linkpath))
5377 tgl@sss.pgh.pa.us 1429 [ # # ]:UBC 0 : ereport(ERROR,
1430 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1431 : : errmsg("symbolic link \"%s\" target is too long",
1432 : : pathbuf)));
5377 tgl@sss.pgh.pa.us 1433 :CBC 39 : linkpath[rllen] = '\0';
1434 : :
1756 rhaas@postgresql.org 1435 : 39 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1436 : : &statbuf, sizeonly);
1437 : : }
5708 magnus@hagander.net 1438 [ + + ]: 395389 : else if (S_ISDIR(statbuf.st_mode))
1439 : : {
4615 1440 : 6034 : bool skip_this_dir = false;
1441 : : ListCell *lc;
1442 : :
1443 : : /*
1444 : : * Store a directory entry in the tar file so we can get the
1445 : : * permissions right.
1446 : : */
1756 rhaas@postgresql.org 1447 : 6034 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1448 : : sizeonly);
1449 : :
1450 : : /*
1451 : : * Call ourselves recursively for a directory, unless it happens
1452 : : * to be a separate tablespace located within PGDATA.
1453 : : */
4615 magnus@hagander.net 1454 [ + + + + : 13205 : foreach(lc, tablespaces)
+ + ]
1455 : : {
1456 : 7199 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1457 : :
1458 : : /*
1459 : : * ti->rpath is the tablespace relative path within PGDATA, or
1460 : : * NULL if the tablespace has been properly located somewhere
1461 : : * else.
1462 : : *
1463 : : * Skip past the leading "./" in pathbuf when comparing.
1464 : : */
1465 [ + + + + ]: 7199 : if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1466 : : {
1467 : 28 : skip_this_dir = true;
1468 : 28 : break;
1469 : : }
1470 : : }
1471 : :
1472 : : /*
1473 : : * skip sending directories inside pg_tblspc, if not required.
1474 : : */
4125 andrew@dunslane.net 1475 [ + + + + ]: 6034 : if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1476 : 27 : skip_this_dir = true;
1477 : :
4615 magnus@hagander.net 1478 [ + + ]: 6034 : if (!skip_this_dir)
1756 rhaas@postgresql.org 1479 : 5979 : size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1480 : : sendtblspclinks, manifest, spcoid, ib);
1481 : : }
5708 magnus@hagander.net 1482 [ + - ]: 389355 : else if (S_ISREG(statbuf.st_mode))
1483 : : {
4838 bruce@momjian.us 1484 : 389355 : bool sent = false;
981 rhaas@postgresql.org 1485 : 389355 : unsigned num_blocks_required = 0;
1486 : 389355 : unsigned truncation_block_length = 0;
1487 : : char tarfilenamebuf[MAXPGPATH * 2];
1488 : 389355 : char *tarfilename = pathbuf + basepathlen + 1;
1489 : 389355 : FileBackupMethod method = BACK_UP_FILE_FULLY;
1490 : :
1491 [ + + + + ]: 389355 : if (ib != NULL && isRelationFile)
1492 : : {
1493 : : Oid relspcoid;
1494 : : char *lookup_path;
1495 : :
1496 [ + + ]: 12822 : if (OidIsValid(spcoid))
1497 : : {
1498 : 9 : relspcoid = spcoid;
723 michael@paquier.xyz 1499 : 9 : lookup_path = psprintf("%s/%u/%s", PG_TBLSPC_DIR, spcoid,
1500 : : tarfilename);
1501 : : }
1502 : : else
1503 : : {
981 rhaas@postgresql.org 1504 [ + + ]: 12813 : if (isGlobalDir)
1505 : 672 : relspcoid = GLOBALTABLESPACE_OID;
1506 : : else
1507 : 12141 : relspcoid = DEFAULTTABLESPACE_OID;
1508 : 12813 : lookup_path = pstrdup(tarfilename);
1509 : : }
1510 : :
1511 : 12822 : method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid,
1512 : : relfilenumber, relForkNum,
1513 : 12822 : segno, statbuf.st_size,
1514 : : &num_blocks_required,
1515 : : relative_block_numbers,
1516 : : &truncation_block_length);
1517 [ + + ]: 12822 : if (method == BACK_UP_FILE_INCREMENTALLY)
1518 : : {
1519 : 8462 : statbuf.st_size =
1520 : 8462 : GetIncrementalFileSize(num_blocks_required);
1521 : 8462 : snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),
1522 : : "%s/INCREMENTAL.%s",
1523 : 8462 : path + basepathlen + 1,
1524 : 8462 : de->d_name);
1525 : 8462 : tarfilename = tarfilenamebuf;
1526 : : }
1527 : :
1528 : 12822 : pfree(lookup_path);
1529 : : }
1530 : :
5708 magnus@hagander.net 1531 [ + + ]: 389355 : if (!sizeonly)
981 rhaas@postgresql.org 1532 [ + + ]: 192155 : sent = sendFile(sink, pathbuf, tarfilename, &statbuf,
1533 : : true, dboid, spcoid,
1534 : : relfilenumber, segno, manifest,
1535 : : num_blocks_required,
1536 : : method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL,
1537 : : truncation_block_length);
1538 : :
4997 heikki.linnakangas@i 1539 [ + + + - ]: 389354 : if (sent || sizeonly)
1540 : : {
1541 : : /* Add size. */
2316 rhaas@postgresql.org 1542 : 389354 : size += statbuf.st_size;
1543 : :
1544 : : /* Pad to a multiple of the tar block size. */
1545 : 389354 : size += tarPaddingBytesRequired(statbuf.st_size);
1546 : :
1547 : : /* Size of the header for the file. */
1548 : 389354 : size += TAR_BLOCK_SIZE;
1549 : : }
1550 : : }
1551 : : else
5708 magnus@hagander.net 1552 [ # # ]:UBC 0 : ereport(WARNING,
1553 : : (errmsg("skipping special file \"%s\"", pathbuf)));
1554 : : }
1555 : :
981 rhaas@postgresql.org 1556 [ + + ]:CBC 6417 : if (relative_block_numbers != NULL)
1557 : 208 : pfree(relative_block_numbers);
1558 : :
5708 magnus@hagander.net 1559 : 6417 : FreeDir(dir);
1560 : 6417 : return size;
1561 : : }
1562 : :
1563 : : /*
1564 : : * Given the member, write the TAR header & send the file.
1565 : : *
1566 : : * If 'missing_ok' is true, will not throw an error if the file is not found.
1567 : : *
1568 : : * If dboid is anything other than InvalidOid then any checksum failures
1569 : : * detected will get reported to the cumulative stats system.
1570 : : *
1571 : : * If the file is to be sent incrementally, then num_incremental_blocks
1572 : : * should be the number of blocks to be sent, and incremental_blocks
1573 : : * an array of block numbers relative to the start of the current segment.
1574 : : * If the whole file is to be sent, then incremental_blocks should be NULL,
1575 : : * and num_incremental_blocks can have any value, as it will be ignored.
1576 : : *
1577 : : * Returns true if the file was successfully sent, false if 'missing_ok',
1578 : : * and the file did not exist.
1579 : : */
1580 : : static bool
1756 rhaas@postgresql.org 1581 : 192336 : sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1582 : : struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,
1583 : : RelFileNumber relfilenumber, unsigned segno,
1584 : : backup_manifest_info *manifest, unsigned num_incremental_blocks,
1585 : : BlockNumber *incremental_blocks, unsigned truncation_block_length)
1586 : : {
1587 : : int fd;
3068 magnus@hagander.net 1588 : 192336 : BlockNumber blkno = 0;
1589 : 192336 : int checksum_failures = 0;
1590 : : off_t cnt;
1059 rhaas@postgresql.org 1591 : 192336 : pgoff_t bytes_done = 0;
3068 magnus@hagander.net 1592 : 192336 : bool verify_checksum = false;
1593 : : pg_checksum_context checksum_ctx;
981 rhaas@postgresql.org 1594 : 192336 : int ibindex = 0;
1595 : :
2094 michael@paquier.xyz 1596 [ - + ]: 192336 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
2094 michael@paquier.xyz 1597 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1598 : : readfilename);
1599 : :
2262 rhaas@postgresql.org 1600 :CBC 192336 : fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1601 [ - + ]: 192336 : if (fd < 0)
1602 : : {
4997 heikki.linnakangas@i 1603 [ # # # # ]:UBC 0 : if (errno == ENOENT && missing_ok)
1604 : 0 : return false;
5708 magnus@hagander.net 1605 [ # # ]: 0 : ereport(ERROR,
1606 : : (errcode_for_file_access(),
1607 : : errmsg("could not open file \"%s\": %m", readfilename)));
1608 : : }
1609 : :
1756 rhaas@postgresql.org 1610 :CBC 192336 : _tarWriteHeader(sink, tarfilename, NULL, statbuf, false);
1611 : :
1612 : : /*
1613 : : * Checksums are verified in multiples of BLCKSZ, so the buffer length
1614 : : * should be a multiple of the block size as well.
1615 : : */
1059 1616 [ - + ]: 192335 : Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1617 : :
1618 : : /*
1619 : : * Verify checksums unless the client requested otherwise, but only for
1620 : : * relation files, and only while checksums have been continuously enabled
1621 : : * since the checkpoint this backup started from. Checksums can still be
1622 : : * disabled while the backup runs, thus we check at each point where we
1623 : : * could be validating a checksum.
1624 : : */
9 dgustafsson@postgres 1625 [ + + + + : 380251 : if (!noverify_checksums && RelFileNumberIsValid(relfilenumber) &&
+ + ]
1626 : 187916 : backup_checksums_verifiable(sink->bbs_state->startptr))
1017 rhaas@postgresql.org 1627 : 185885 : verify_checksum = true;
1628 : :
1629 : : /*
1630 : : * If we're sending an incremental file, write the file header.
1631 : : */
981 1632 [ + + ]: 192335 : if (incremental_blocks != NULL)
1633 : : {
1634 : 8462 : unsigned magic = INCREMENTAL_MAGIC;
1635 : 8462 : size_t header_bytes_done = 0;
1636 : : char padding[BLCKSZ];
1637 : : size_t paddinglen;
1638 : :
1639 : : /* Emit header data. */
1640 : 8462 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1641 : : &magic, sizeof(magic));
1642 : 8462 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1643 : : &num_incremental_blocks, sizeof(num_incremental_blocks));
1644 : 8462 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1645 : : &truncation_block_length, sizeof(truncation_block_length));
1646 : 8462 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1647 : : incremental_blocks,
1648 : : sizeof(BlockNumber) * num_incremental_blocks);
1649 : :
1650 : : /*
1651 : : * Add padding to align header to a multiple of BLCKSZ, but only if
1652 : : * the incremental file has some blocks, and the alignment is actually
1653 : : * needed (i.e. header is not already a multiple of BLCKSZ). If there
1654 : : * are no blocks we don't want to make the file unnecessarily large,
1655 : : * as that might make some filesystem optimizations impossible.
1656 : : */
865 tomas.vondra@postgre 1657 [ + + + - ]: 8462 : if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))
1658 : : {
874 1659 : 27 : paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));
1660 : :
1661 : 27 : memset(padding, 0, paddinglen);
1662 : 27 : bytes_done += paddinglen;
1663 : :
1664 : 27 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1665 : : padding, paddinglen);
1666 : : }
1667 : :
1668 : : /* Flush out any data still in the buffer so it's again empty. */
981 rhaas@postgresql.org 1669 [ + - ]: 8462 : if (header_bytes_done > 0)
1670 : : {
1671 : 8462 : bbsink_archive_contents(sink, header_bytes_done);
1672 [ - + ]: 8462 : if (pg_checksum_update(&checksum_ctx,
1673 : 8462 : (uint8 *) sink->bbs_buffer,
1674 : : header_bytes_done) < 0)
981 rhaas@postgresql.org 1675 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1676 : : }
1677 : :
1678 : : /* Update our notion of file position. */
981 rhaas@postgresql.org 1679 :CBC 8462 : bytes_done += sizeof(magic);
1680 : 8462 : bytes_done += sizeof(num_incremental_blocks);
1681 : 8462 : bytes_done += sizeof(truncation_block_length);
1682 : 8462 : bytes_done += sizeof(BlockNumber) * num_incremental_blocks;
1683 : : }
1684 : :
1685 : : /*
1686 : : * Loop until we read the amount of data the caller told us to expect. The
1687 : : * file could be longer, if it was extended while we were sending it, but
1688 : : * for a base backup we can ignore such extended data. It will be restored
1689 : : * from WAL.
1690 : : */
1691 : : while (1)
1692 : : {
1693 : : /*
1694 : : * Determine whether we've read all the data that we need, and if not,
1695 : : * read some more.
1696 : : */
1697 [ + + ]: 409957 : if (incremental_blocks == NULL)
1698 : : {
1699 : 401456 : size_t remaining = statbuf->st_size - bytes_done;
1700 : :
1701 : : /*
1702 : : * If we've read the required number of bytes, then it's time to
1703 : : * stop.
1704 : : */
1705 [ + + ]: 401456 : if (bytes_done >= statbuf->st_size)
1706 : 183873 : break;
1707 : :
1708 : : /*
1709 : : * Read as many bytes as will fit in the buffer, or however many
1710 : : * are left to read, whichever is less.
1711 : : */
1712 : 217583 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1713 : : bytes_done, remaining,
1714 : 217583 : blkno + segno * RELSEG_SIZE,
1715 : : verify_checksum,
1716 : : &checksum_failures);
1717 : : }
1718 : : else
1719 : : {
1720 : : BlockNumber relative_blkno;
1721 : :
1722 : : /*
1723 : : * If we've read all the blocks, then it's time to stop.
1724 : : */
1725 [ + + ]: 8501 : if (ibindex >= num_incremental_blocks)
1726 : 8462 : break;
1727 : :
1728 : : /*
1729 : : * Read just one block, whichever one is the next that we're
1730 : : * supposed to include.
1731 : : */
1732 : 39 : relative_blkno = incremental_blocks[ibindex++];
1733 : 39 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1734 : 39 : relative_blkno * BLCKSZ,
1735 : : BLCKSZ,
1736 : 39 : relative_blkno + segno * RELSEG_SIZE,
1737 : : verify_checksum,
1738 : : &checksum_failures);
1739 : :
1740 : : /*
1741 : : * If we get a partial read, that must mean that the relation is
1742 : : * being truncated. Ultimately, it should be truncated to a
1743 : : * multiple of BLCKSZ, since this path should only be reached for
1744 : : * relation files, but we might transiently observe an
1745 : : * intermediate value.
1746 : : *
1747 : : * It should be fine to treat this just as if the entire block had
1748 : : * been truncated away - i.e. fill this and all later blocks with
1749 : : * zeroes. WAL replay will fix things up.
1750 : : */
1751 [ - + ]: 39 : if (cnt < BLCKSZ)
981 rhaas@postgresql.org 1752 :UBC 0 : break;
1753 : : }
1754 : :
1755 : : /*
1756 : : * If the amount of data we were able to read was not a multiple of
1757 : : * BLCKSZ, we cannot verify checksums, which are block-level.
1758 : : */
9 dgustafsson@postgres 1759 [ + + + - ]:CBC 427817 : if (verify_checksum &&
1760 : 210195 : backup_checksums_verifiable(sink->bbs_state->startptr) &&
1761 [ - + ]: 210195 : (cnt % BLCKSZ != 0))
1762 : : {
3068 magnus@hagander.net 1763 [ # # ]:UBC 0 : ereport(WARNING,
1764 : : (errmsg("could not verify checksum in file \"%s\", block "
1765 : : "%u: read buffer size %d and page size %d "
1766 : : "differ",
1767 : : readfilename, blkno, (int) cnt, BLCKSZ)));
1768 : 0 : verify_checksum = false;
1769 : : }
1770 : :
1771 : : /*
1772 : : * If we hit end-of-file, a concurrent truncation must have occurred.
1773 : : * That's not an error condition, because WAL replay will fix things
1774 : : * up.
1775 : : */
1302 rhaas@postgresql.org 1776 [ - + ]:CBC 217622 : if (cnt == 0)
1302 rhaas@postgresql.org 1777 :UBC 0 : break;
1778 : :
1779 : : /* Update block number and # of bytes done for next loop iteration. */
1059 rhaas@postgresql.org 1780 :CBC 217622 : blkno += cnt / BLCKSZ;
1781 : 217622 : bytes_done += cnt;
1782 : :
1783 : : /*
1784 : : * Make sure incremental files with block data are properly aligned
1785 : : * (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1786 : : */
874 tomas.vondra@postgre 1787 [ + + + - : 217622 : Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&
- + ]
1788 : : (bytes_done % BLCKSZ != 0)));
1789 : :
1790 : : /* Archive the data we just read. */
1756 rhaas@postgresql.org 1791 : 217622 : bbsink_archive_contents(sink, cnt);
1792 : :
1793 : : /* Also feed it to the checksum machinery. */
1794 [ - + ]: 217622 : if (pg_checksum_update(&checksum_ctx,
1795 : 217622 : (uint8 *) sink->bbs_buffer, cnt) < 0)
2094 michael@paquier.xyz 1796 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1797 : : }
1798 : :
1799 : : /* If the file was truncated while we were sending it, pad it with zeros */
1059 rhaas@postgresql.org 1800 [ - + ]:CBC 192335 : while (bytes_done < statbuf->st_size)
1801 : : {
1059 rhaas@postgresql.org 1802 :UBC 0 : size_t remaining = statbuf->st_size - bytes_done;
1756 1803 : 0 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1804 : :
1805 [ # # # # : 0 : MemSet(sink->bbs_buffer, 0, nbytes);
# # # # #
# ]
1806 [ # # ]: 0 : if (pg_checksum_update(&checksum_ctx,
1807 : 0 : (uint8 *) sink->bbs_buffer,
1808 : : nbytes) < 0)
1809 [ # # ]: 0 : elog(ERROR, "could not update checksum of base backup");
1810 : 0 : bbsink_archive_contents(sink, nbytes);
1059 1811 : 0 : bytes_done += nbytes;
1812 : : }
1813 : :
1814 : : /*
1815 : : * Pad to a block boundary, per tar format requirements. (This small piece
1816 : : * of data is probably not worth throttling, and is not checksummed
1817 : : * because it's not actually part of the file.)
1818 : : */
1059 rhaas@postgresql.org 1819 :CBC 192335 : _tarWritePadding(sink, bytes_done);
1820 : :
2262 1821 : 192335 : CloseTransientFile(fd);
1822 : :
8 michael@paquier.xyz 1823 [ + + ]: 192335 : if (checksum_failures > 0)
1824 : : {
3068 magnus@hagander.net 1825 [ + - ]: 4 : ereport(WARNING,
1826 : : (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1827 : : "file \"%s\" has a total of %d checksum verification failures",
1828 : : checksum_failures,
1829 : : readfilename, checksum_failures)));
1830 : :
515 andres@anarazel.de 1831 : 4 : pgstat_prepare_report_checksum_failure(dboid);
2694 magnus@hagander.net 1832 : 4 : pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1833 : : }
1834 : :
3068 1835 : 192335 : total_checksum_failures += checksum_failures;
1836 : :
2317 rhaas@postgresql.org 1837 : 192335 : AddFileToBackupManifest(manifest, spcoid, tarfilename, statbuf->st_size,
1838 : 192335 : (pg_time_t) statbuf->st_mtime, &checksum_ctx);
1839 : :
4997 heikki.linnakangas@i 1840 : 192335 : return true;
1841 : : }
1842 : :
1843 : : /*
1844 : : * Read some more data from the file into the bbsink's buffer, verifying
1845 : : * checksums as required.
1846 : : *
1847 : : * 'offset' is the file offset from which we should begin to read, and
1848 : : * 'length' is the amount of data that should be read. The actual amount
1849 : : * of data read will be less than the requested amount if the bbsink's
1850 : : * buffer isn't big enough to hold it all, or if the underlying file has
1851 : : * been truncated. The return value is the number of bytes actually read.
1852 : : *
1853 : : * 'blkno' is the block number of the first page in the bbsink's buffer
1854 : : * relative to the start of the relation.
1855 : : *
1856 : : * 'verify_checksum' determines if the user has asked to verify checksums, but
1857 : : * since data checksums can be disabled, or become disabled, we need to check
1858 : : * state before verifying individual pages. If we do this, we'll update
1859 : : * *checksum_failures and issue warnings as appropriate.
1860 : : */
1861 : : static off_t
1059 rhaas@postgresql.org 1862 : 217622 : read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd,
1863 : : off_t offset, size_t length, BlockNumber blkno,
1864 : : bool verify_checksum, int *checksum_failures)
1865 : : {
1866 : : off_t cnt;
1867 : : int i;
1868 : : char *page;
1869 : :
1870 : : /* Try to read some more data. */
1871 : 217622 : cnt = basebackup_read_file(fd, sink->bbs_buffer,
1872 : 217622 : Min(sink->bbs_buffer_length, length),
1873 : : offset, readfilename, true);
1874 : :
1875 : : /* Can't verify checksums if read length is not a multiple of BLCKSZ. */
1876 [ + + - + ]: 217622 : if (!verify_checksum || (cnt % BLCKSZ) != 0)
1877 : 7427 : return cnt;
1878 : :
1879 : : /* Verify checksum for each block. */
1880 [ + + ]: 725418 : for (i = 0; i < cnt / BLCKSZ; i++)
1881 : : {
1882 : : int reread_cnt;
1883 : : uint16 expected_checksum;
1884 : :
1885 : : /*
1886 : : * The data checksum state can change at any point, so we need to
1887 : : * re-check before each page.
1888 : : */
9 dgustafsson@postgres 1889 [ - + ]: 515223 : if (!backup_checksums_verifiable(sink->bbs_state->startptr))
146 dgustafsson@postgres 1890 :UBC 0 : return cnt;
1891 : :
1059 rhaas@postgresql.org 1892 :CBC 515223 : page = sink->bbs_buffer + BLCKSZ * i;
1893 : :
1894 : : /* If the page is OK, go on to the next one. */
1895 [ + + ]: 515223 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1896 : : &expected_checksum))
1897 : 515209 : continue;
1898 : :
1899 : : /*
1900 : : * Retry the block on the first failure. It's possible that we read
1901 : : * the first 4K page of the block just before postgres updated the
1902 : : * entire block so it ends up looking torn to us. If, before we retry
1903 : : * the read, the concurrent write of the block finishes, the page LSN
1904 : : * will be updated and we'll realize that we should ignore this block.
1905 : : *
1906 : : * There's no guarantee that this will actually happen, though: the
1907 : : * torn write could take an arbitrarily long time to complete.
1908 : : * Retrying multiple times wouldn't fix this problem, either, though
1909 : : * it would reduce the chances of it happening in practice. The only
1910 : : * real fix here seems to be to have some kind of interlock that
1911 : : * allows us to wait until we can be certain that no write to the
1912 : : * block is in progress. Since we don't have any such thing right now,
1913 : : * we just do this and hope for the best.
1914 : : *
1915 : : * The data checksum state may also have changed concurrently so check
1916 : : * again.
1917 : : */
9 dgustafsson@postgres 1918 [ - + ]: 14 : if (!backup_checksums_verifiable(sink->bbs_state->startptr))
146 dgustafsson@postgres 1919 :UBC 0 : return cnt;
1059 rhaas@postgresql.org 1920 :CBC 14 : reread_cnt =
1921 : 14 : basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i,
1922 : 14 : BLCKSZ, offset + BLCKSZ * i,
1923 : : readfilename, false);
1924 [ - + ]: 14 : if (reread_cnt == 0)
1925 : : {
1926 : : /*
1927 : : * If we hit end-of-file, a concurrent truncation must have
1928 : : * occurred, so reduce cnt to reflect only the blocks already
1929 : : * processed and break out of this loop.
1930 : : */
1059 rhaas@postgresql.org 1931 :UBC 0 : cnt = BLCKSZ * i;
1932 : 0 : break;
1933 : : }
1934 : :
1935 : : /* If the page now looks OK, go on to the next one. */
1059 rhaas@postgresql.org 1936 [ - + ]:CBC 14 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1937 : : &expected_checksum))
1059 rhaas@postgresql.org 1938 :UBC 0 : continue;
1939 : :
1940 : : /* Handle checksum failure. */
1059 rhaas@postgresql.org 1941 :CBC 14 : (*checksum_failures)++;
1942 [ + + ]: 14 : if (*checksum_failures <= 5)
1943 [ + - ]: 12 : ereport(WARNING,
1944 : : (errmsg("checksum verification failed in "
1945 : : "file \"%s\", block %u: calculated "
1946 : : "%X but expected %X",
1947 : : readfilename, blkno + i, expected_checksum,
1948 : : ((PageHeader) page)->pd_checksum)));
1949 [ + + ]: 14 : if (*checksum_failures == 5)
1950 [ + - ]: 2 : ereport(WARNING,
1951 : : (errmsg("further checksum verification "
1952 : : "failures in file \"%s\" will not "
1953 : : "be reported", readfilename)));
1954 : : }
1955 : :
1956 : 210195 : return cnt;
1957 : : }
1958 : :
1959 : : /*
1960 : : * Push data into a bbsink.
1961 : : *
1962 : : * It's better, when possible, to read data directly into the bbsink's buffer,
1963 : : * rather than using this function to copy it into the buffer; this function is
1964 : : * for cases where that approach is not practical.
1965 : : *
1966 : : * bytes_done should point to a count of the number of bytes that are
1967 : : * currently used in the bbsink's buffer. Upon return, the bytes identified by
1968 : : * data and length will have been copied into the bbsink's buffer, flushing
1969 : : * as required, and *bytes_done will have been updated accordingly. If the
1970 : : * buffer was flushed, the previous contents will also have been fed to
1971 : : * checksum_ctx.
1972 : : *
1973 : : * Note that after one or more calls to this function it is the caller's
1974 : : * responsibility to perform any required final flush.
1975 : : */
1976 : : static void
981 1977 : 33875 : push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
1978 : : size_t *bytes_done, void *data, size_t length)
1979 : : {
1980 [ + + ]: 33875 : while (length > 0)
1981 : : {
1982 : : size_t bytes_to_copy;
1983 : :
1984 : : /*
1985 : : * We use < here rather than <= so that if the data exactly fills the
1986 : : * remaining buffer space, we trigger a flush now.
1987 : : */
1988 [ + - ]: 25440 : if (length < sink->bbs_buffer_length - *bytes_done)
1989 : : {
1990 : : /* Append remaining data to buffer. */
1991 : 25440 : memcpy(sink->bbs_buffer + *bytes_done, data, length);
1992 : 25440 : *bytes_done += length;
1993 : 25440 : return;
1994 : : }
1995 : :
1996 : : /* Copy until buffer is full and flush it. */
981 rhaas@postgresql.org 1997 :UBC 0 : bytes_to_copy = sink->bbs_buffer_length - *bytes_done;
1998 : 0 : memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);
1999 : 0 : data = ((char *) data) + bytes_to_copy;
2000 : 0 : length -= bytes_to_copy;
2001 : 0 : bbsink_archive_contents(sink, sink->bbs_buffer_length);
2002 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer,
2003 : : sink->bbs_buffer_length) < 0)
2004 [ # # ]: 0 : elog(ERROR, "could not update checksum");
2005 : 0 : *bytes_done = 0;
2006 : : }
2007 : : }
2008 : :
2009 : : /*
2010 : : * Check whether data checksums can be verified for a backup started at
2011 : : * start_lsn.
2012 : : *
2013 : : * Checksums are verified only while they have been continuously enabled
2014 : : * since the checkpoint the backup started from: the state must be "on" and
2015 : : * the last state change must predate the backup start. Such a checkpoint
2016 : : * guarantees that every page flushed before it has a checksum written. Any
2017 : : * later state change ends verification for the rest of the backup: pages
2018 : : * written while checksums were off can lack checksums yet keep LSNs older
2019 : : * than the backup start, and re-enabling completes before the rewritten
2020 : : * pages are flushed, so observing the "on" state again is not enough to
2021 : : * resume.
2022 : : */
2023 : : static bool
9 dgustafsson@postgres 2024 :CBC 1427529 : backup_checksums_verifiable(XLogRecPtr start_lsn)
2025 : : {
2026 [ + + + - ]: 2853027 : return DataChecksumsNeedVerify() &&
2027 : 1425498 : GetLastChecksumChangeRecPtr() <= start_lsn;
2028 : : }
2029 : :
2030 : : /*
2031 : : * Try to verify the checksum for the provided page, if it seems appropriate
2032 : : * to do so.
2033 : : *
2034 : : * Returns true if verification succeeds or if we decide not to check it,
2035 : : * and false if verification fails. When return false, it also sets
2036 : : * *expected_checksum to the computed value.
2037 : : */
2038 : : static bool
1059 rhaas@postgresql.org 2039 : 515237 : verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno,
2040 : : uint16 *expected_checksum)
2041 : : {
2042 : : PageHeader phdr;
2043 : : uint16 checksum;
2044 : :
2045 : : /*
2046 : : * Only check pages which have not been modified since the start of the
2047 : : * base backup. Otherwise, they might have been written only halfway and
2048 : : * the checksum would not be valid. However, replaying WAL would
2049 : : * reinstate the correct page in this case. We also skip completely new
2050 : : * pages, since they don't have a checksum yet.
2051 : : */
2052 [ + + + + ]: 515237 : if (PageIsNew(page) || PageGetLSN(page) >= start_lsn)
2053 : 1056 : return true;
2054 : :
9 dgustafsson@postgres 2055 [ - + ]: 514181 : if (!backup_checksums_verifiable(start_lsn))
146 dgustafsson@postgres 2056 :UBC 0 : return true;
2057 : :
2058 : : /* Perform the actual checksum calculation. */
1059 rhaas@postgresql.org 2059 :CBC 514181 : checksum = pg_checksum_page(page, blkno);
2060 : :
2061 : : #ifdef USE_INJECTION_POINTS
2062 : : {
2063 : : /*
2064 : : * Make it possible to test checksum verification failure without
2065 : : * having to destroy data on disk. There is cap on how many times we
2066 : : * want to cause verification failure to make tests more interesting
2067 : : * and less log intensive. This makes it easy to test pg_basebackup
2068 : : * with the command_checks_all test function.
2069 : : */
2070 : : static int hit = 0;
2071 : :
9 dgustafsson@postgres 2072 [ - + ]: 514181 : if (IS_INJECTION_POINT_ATTACHED("basebackup-fail-checksum-verification"))
2073 : : {
9 dgustafsson@postgres 2074 [ # # ]:UBC 0 : if (hit++ < 5)
2075 : : {
2076 : 0 : checksum = 0;
2077 : 0 : INJECTION_POINT_CACHED("basebackup-fail-checksum-verification",
2078 : : NULL);
2079 : : }
2080 : : }
2081 : : }
2082 : : #endif
2083 : :
2084 : : /* See whether it matches the value from the page. */
1059 rhaas@postgresql.org 2085 :CBC 514181 : phdr = (PageHeader) page;
2086 [ + + ]: 514181 : if (phdr->pd_checksum == checksum)
2087 : 514153 : return true;
2088 : 28 : *expected_checksum = checksum;
2089 : 28 : return false;
2090 : : }
2091 : :
2092 : : static int64
1756 2093 : 202406 : _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
2094 : : struct stat *statbuf, bool sizeonly)
2095 : : {
2096 : : enum tarError rc;
2097 : :
3620 peter_e@gmx.net 2098 [ + + ]: 202406 : if (!sizeonly)
2099 : : {
2100 : : /*
2101 : : * As of this writing, the smallest supported block size is 1kB, which
2102 : : * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
2103 : : * multiple of BLCKSZ, it should be safe to assume that the buffer is
2104 : : * large enough to fit an entire tar block. We double-check by means
2105 : : * of these assertions.
2106 : : */
2107 : : StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
2108 : : "BLCKSZ too small for tar block");
1756 rhaas@postgresql.org 2109 [ - + ]: 197449 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2110 : :
2111 : 197449 : rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
2112 : : statbuf->st_size, statbuf->st_mode,
2113 : : statbuf->st_uid, statbuf->st_gid,
2114 : : statbuf->st_mtime);
2115 : :
3620 peter_e@gmx.net 2116 [ + + - - ]: 197449 : switch (rc)
2117 : : {
2118 : 197448 : case TAR_OK:
2119 : 197448 : break;
2120 : 1 : case TAR_NAME_TOO_LONG:
2121 [ + - ]: 1 : ereport(ERROR,
2122 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2123 : : errmsg("file name too long for tar format: \"%s\"",
2124 : : filename)));
2125 : : break;
3620 peter_e@gmx.net 2126 :UBC 0 : case TAR_SYMLINK_TOO_LONG:
2127 [ # # ]: 0 : ereport(ERROR,
2128 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2129 : : errmsg("symbolic link target too long for tar format: "
2130 : : "file name \"%s\", target \"%s\"",
2131 : : filename, linktarget)));
2132 : : break;
2133 : 0 : default:
2134 [ # # ]: 0 : elog(ERROR, "unrecognized tar error: %d", rc);
2135 : : }
2136 : :
1756 rhaas@postgresql.org 2137 :CBC 197448 : bbsink_archive_contents(sink, TAR_BLOCK_SIZE);
2138 : : }
2139 : :
2140 : 202405 : return TAR_BLOCK_SIZE;
2141 : : }
2142 : :
2143 : : /*
2144 : : * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
2145 : : */
2146 : : static void
2147 : 192564 : _tarWritePadding(bbsink *sink, int len)
2148 : : {
2149 : 192564 : int pad = tarPaddingBytesRequired(len);
2150 : :
2151 : : /*
2152 : : * As in _tarWriteHeader, it should be safe to assume that the buffer is
2153 : : * large enough that we don't need to do this in multiple chunks.
2154 : : */
2155 [ - + ]: 192564 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2156 [ - + ]: 192564 : Assert(pad <= TAR_BLOCK_SIZE);
2157 : :
2158 [ + + ]: 192564 : if (pad > 0)
2159 : : {
2160 [ + - + + : 33471 : MemSet(sink->bbs_buffer, 0, pad);
+ - + - +
+ ]
2161 : 11284 : bbsink_archive_contents(sink, pad);
2162 : : }
4564 alvherre@alvh.no-ip. 2163 : 192564 : }
2164 : :
2165 : : /*
2166 : : * If the entry in statbuf is a link, then adjust statbuf to make it look like a
2167 : : * directory, so that it will be written that way.
2168 : : */
2169 : : static void
1756 rhaas@postgresql.org 2170 : 2945 : convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
2171 : : {
2172 : : /* If symlink, write it as a directory anyway */
2173 [ + + ]: 2945 : if (S_ISLNK(statbuf->st_mode))
2174 : 66 : statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
2368 fujii@postgresql.org 2175 : 2945 : }
2176 : :
2177 : : /*
2178 : : * Read some data from a file, setting a wait event and reporting any error
2179 : : * encountered.
2180 : : *
2181 : : * If partial_read_ok is false, also report an error if the number of bytes
2182 : : * read is not equal to the number of bytes requested.
2183 : : *
2184 : : * Returns the number of bytes read.
2185 : : */
2186 : : static ssize_t
2262 rhaas@postgresql.org 2187 : 225316 : basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
2188 : : const char *filename, bool partial_read_ok)
2189 : : {
2190 : : ssize_t rc;
2191 : :
2192 : 225316 : pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
1428 tmunro@postgresql.or 2193 : 225316 : rc = pg_pread(fd, buf, nbytes, offset);
2262 rhaas@postgresql.org 2194 : 225316 : pgstat_report_wait_end();
2195 : :
2196 [ - + ]: 225316 : if (rc < 0)
2262 rhaas@postgresql.org 2197 [ # # ]:UBC 0 : ereport(ERROR,
2198 : : (errcode_for_file_access(),
2199 : : errmsg("could not read file \"%s\": %m", filename)));
2262 rhaas@postgresql.org 2200 [ + + + - :CBC 225316 : if (!partial_read_ok && rc > 0 && rc != nbytes)
- + ]
2262 rhaas@postgresql.org 2201 [ # # ]:UBC 0 : ereport(ERROR,
2202 : : (errcode_for_file_access(),
2203 : : errmsg("could not read file \"%s\": read %zd of %zu",
2204 : : filename, rc, nbytes)));
2205 : :
2262 rhaas@postgresql.org 2206 :CBC 225316 : return rc;
2207 : : }
|
