Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * basebackup.c
4 : : * code for taking a base backup and streaming it to a standby
5 : : *
6 : : * Portions Copyright (c) 2010-2026, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/backend/backup/basebackup.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres.h"
14 : :
15 : : #include <sys/stat.h>
16 : : #include <unistd.h>
17 : : #include <time.h>
18 : :
19 : : #include "access/xlog_internal.h"
20 : : #include "access/xlogbackup.h"
21 : : #include "backup/backup_manifest.h"
22 : : #include "backup/basebackup.h"
23 : : #include "backup/basebackup_incremental.h"
24 : : #include "backup/basebackup_sink.h"
25 : : #include "backup/basebackup_target.h"
26 : : #include "catalog/pg_control.h"
27 : : #include "catalog/pg_tablespace_d.h"
28 : : #include "commands/defrem.h"
29 : : #include "common/compression.h"
30 : : #include "common/controldata_utils.h"
31 : : #include "common/file_perm.h"
32 : : #include "common/file_utils.h"
33 : : #include "lib/stringinfo.h"
34 : : #include "miscadmin.h"
35 : : #include "nodes/pg_list.h"
36 : : #include "pgstat.h"
37 : : #include "pgtar.h"
38 : : #include "postmaster/syslogger.h"
39 : : #include "postmaster/walsummarizer.h"
40 : : #include "replication/slot.h"
41 : : #include "replication/walsender.h"
42 : : #include "replication/walsender_private.h"
43 : : #include "storage/bufpage.h"
44 : : #include "storage/checksum.h"
45 : : #include "storage/dsm_impl.h"
46 : : #include "storage/ipc.h"
47 : : #include "storage/lwlock.h"
48 : : #include "storage/reinit.h"
49 : : #include "utils/builtins.h"
50 : : #include "utils/guc.h"
51 : : #include "utils/injection_point.h"
52 : : #include "utils/ps_status.h"
53 : : #include "utils/relcache.h"
54 : : #include "utils/resowner.h"
55 : : #include "utils/wait_event.h"
56 : :
57 : : /*
58 : : * How much data do we want to send in one CopyData message? Note that
59 : : * this may also result in reading the underlying files in chunks of this
60 : : * size.
61 : : *
62 : : * NB: The buffer size is required to be a multiple of the system block
63 : : * size, so use that value instead if it's bigger than our preference.
64 : : */
65 : : #define SINK_BUFFER_LENGTH Max(32768, BLCKSZ)
66 : :
67 : : typedef struct
68 : : {
69 : : const char *label;
70 : : bool progress;
71 : : bool fastcheckpoint;
72 : : bool nowait;
73 : : bool includewal;
74 : : bool incremental;
75 : : uint32 maxrate;
76 : : bool sendtblspcmapfile;
77 : : bool send_to_client;
78 : : bool use_copytblspc;
79 : : BaseBackupTargetHandle *target_handle;
80 : : backup_manifest_option manifest;
81 : : pg_compress_algorithm compression;
82 : : pg_compress_specification compression_specification;
83 : : pg_checksum_type manifest_checksum_type;
84 : : } basebackup_options;
85 : :
86 : : #define TAR_NUM_TERMINATION_BLOCKS 2
87 : :
88 : : StaticAssertDecl(TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE <= BLCKSZ,
89 : : "BLCKSZ too small for " CppAsString2(TAR_NUM_TERMINATION_BLOCKS) " tar termination blocks");
90 : :
91 : : static int64 sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
92 : : struct backup_manifest_info *manifest,
93 : : IncrementalBackupInfo *ib);
94 : : static int64 sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
95 : : List *tablespaces, bool sendtblspclinks,
96 : : backup_manifest_info *manifest, Oid spcoid,
97 : : IncrementalBackupInfo *ib);
98 : : static bool sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
99 : : const struct stat *statbuf, bool missing_ok,
100 : : Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
101 : : unsigned segno,
102 : : backup_manifest_info *manifest,
103 : : unsigned num_incremental_blocks,
104 : : BlockNumber *incremental_blocks,
105 : : unsigned truncation_block_length);
106 : : static off_t read_file_data_into_buffer(bbsink *sink,
107 : : const char *readfilename, int fd,
108 : : off_t offset, size_t length,
109 : : BlockNumber blkno,
110 : : bool verify_checksum,
111 : : int *checksum_failures);
112 : : static void push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
113 : : size_t *bytes_done, void *data, size_t length);
114 : : static bool backup_checksums_verifiable(XLogRecPtr start_lsn);
115 : : static bool verify_page_checksum(Page page, XLogRecPtr start_lsn,
116 : : BlockNumber blkno,
117 : : uint16 *expected_checksum);
118 : : static void sendFileWithContent(bbsink *sink, const char *filename,
119 : : const char *content, int len,
120 : : backup_manifest_info *manifest);
121 : : static int64 _tarWriteHeader(bbsink *sink, const char *filename,
122 : : const char *linktarget, struct stat *statbuf,
123 : : bool sizeonly);
124 : : static void _tarWritePadding(bbsink *sink, int len);
125 : : static void convert_link_to_directory(const char *pathbuf, struct stat *statbuf);
126 : : static void perform_base_backup(basebackup_options *opt, bbsink *sink,
127 : : IncrementalBackupInfo *ib);
128 : : static void parse_basebackup_options(List *options, basebackup_options *opt);
129 : : static int compareWalFileNames(const ListCell *a, const ListCell *b);
130 : : static ssize_t basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
131 : : const char *filename, bool partial_read_ok);
132 : :
133 : : /* Was the backup currently in-progress initiated in recovery mode? */
134 : : static bool backup_started_in_recovery = false;
135 : :
136 : : /* Total number of checksum failures during base backup. */
137 : : static long long int total_checksum_failures;
138 : :
139 : : /* Do not verify checksums. */
140 : : static bool noverify_checksums = false;
141 : :
142 : : /*
143 : : * Definition of one element part of an exclusion list, used for paths part
144 : : * of checksum validation or base backups. "name" is the name of the file
145 : : * or path to check for exclusion. If "match_prefix" is true, any items
146 : : * matching the name as prefix are excluded.
147 : : */
148 : : struct exclude_list_item
149 : : {
150 : : const char *name;
151 : : bool match_prefix;
152 : : };
153 : :
154 : : /*
155 : : * The contents of these directories are removed or recreated during server
156 : : * start so they are not included in backups. The directories themselves are
157 : : * kept and included as empty to preserve access permissions.
158 : : *
159 : : * Note: this list should be kept in sync with the filter lists in pg_rewind's
160 : : * filemap.c.
161 : : */
162 : : static const char *const excludeDirContents[] =
163 : : {
164 : : /*
165 : : * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped
166 : : * because extensions like pg_stat_statements store data there.
167 : : */
168 : : PG_STAT_TMP_DIR,
169 : :
170 : : /*
171 : : * It is generally not useful to backup the contents of this directory
172 : : * even if the intention is to restore to another primary. See backup.sgml
173 : : * for a more detailed description.
174 : : */
175 : : PG_REPLSLOT_DIR,
176 : :
177 : : /* Contents removed on startup, see dsm_cleanup_for_mmap(). */
178 : : PG_DYNSHMEM_DIR,
179 : :
180 : : /* Contents removed on startup, see AsyncShmemInit(). */
181 : : "pg_notify",
182 : :
183 : : /*
184 : : * Old contents are loaded for possible debugging but are not required for
185 : : * normal operation, see SerialInit().
186 : : */
187 : : "pg_serial",
188 : :
189 : : /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */
190 : : "pg_snapshots",
191 : :
192 : : /* Contents zeroed on startup, see StartupSUBTRANS(). */
193 : : "pg_subtrans",
194 : :
195 : : /* end of list */
196 : : NULL
197 : : };
198 : :
199 : : /*
200 : : * List of files excluded from backups.
201 : : */
202 : : static const struct exclude_list_item excludeFiles[] =
203 : : {
204 : : /* Skip auto conf temporary file. */
205 : : {PG_AUTOCONF_FILENAME ".tmp", false},
206 : :
207 : : /* Skip current log file temporary file */
208 : : {LOG_METAINFO_DATAFILE_TMP, false},
209 : :
210 : : /*
211 : : * Skip relation cache because it is rebuilt on startup. This includes
212 : : * temporary files.
213 : : */
214 : : {RELCACHE_INIT_FILENAME, true},
215 : :
216 : : /*
217 : : * backup_label and tablespace_map should not exist in a running cluster
218 : : * capable of doing an online backup, but exclude them just in case.
219 : : */
220 : : {BACKUP_LABEL_FILE, false},
221 : : {TABLESPACE_MAP, false},
222 : :
223 : : /*
224 : : * If there's a backup_manifest, it belongs to a backup that was used to
225 : : * start this server. It is *not* correct for this backup. Our
226 : : * backup_manifest is injected into the backup separately if users want
227 : : * it.
228 : : */
229 : : {"backup_manifest", false},
230 : :
231 : : {"postmaster.pid", false},
232 : : {"postmaster.opts", false},
233 : :
234 : : /* end of list */
235 : : {NULL, false}
236 : : };
237 : :
238 : : /*
239 : : * Actually do a base backup for the specified tablespaces.
240 : : *
241 : : * This is split out mainly to avoid complaints about "variable might be
242 : : * clobbered by longjmp" from stupider versions of gcc.
243 : : */
244 : : static void
1051 rhaas@postgresql.org 245 :CBC 197 : perform_base_backup(basebackup_options *opt, bbsink *sink,
246 : : IncrementalBackupInfo *ib)
247 : : {
248 : : bbsink_state state;
249 : : XLogRecPtr endptr;
250 : : TimeLineID endtli;
251 : : backup_manifest_info manifest;
252 : : BackupState *backup_state;
253 : : StringInfoData tablespace_map;
254 : :
255 : : /* Initial backup state, insofar as we know it now. */
1826 256 : 197 : state.tablespaces = NIL;
257 : 197 : state.tablespace_num = 0;
258 : 197 : state.bytes_done = 0;
259 : 197 : state.bytes_total = 0;
260 : 197 : state.bytes_total_is_valid = false;
261 : :
262 : : /* we're going to use a BufFile, so we need a ResourceOwner */
758 andres@anarazel.de 263 [ - + ]: 197 : Assert(AuxProcessResourceOwner != NULL);
264 [ + + - + ]: 197 : Assert(CurrentResourceOwner == AuxProcessResourceOwner ||
265 : : CurrentResourceOwner == NULL);
266 : 197 : CurrentResourceOwner = AuxProcessResourceOwner;
267 : :
5075 heikki.linnakangas@i 268 : 197 : backup_started_in_recovery = RecoveryInProgress();
269 : :
2387 rhaas@postgresql.org 270 : 197 : InitializeBackupManifest(&manifest, opt->manifest,
271 : : opt->manifest_checksum_type);
272 : :
3138 magnus@hagander.net 273 : 197 : total_checksum_failures = 0;
274 : :
275 : : /* Allocate backup related variables. */
330 michael@paquier.xyz 276 : 197 : backup_state = palloc0_object(BackupState);
364 drowley@postgresql.o 277 : 197 : initStringInfo(&tablespace_map);
278 : :
1826 rhaas@postgresql.org 279 : 197 : basebackup_progress_wait_checkpoint();
1501 michael@paquier.xyz 280 : 197 : do_pg_backup_start(opt->label, opt->fastcheckpoint, &state.tablespaces,
281 : : backup_state, &tablespace_map);
282 : :
283 : 197 : state.startptr = backup_state->startpoint;
284 : 197 : state.starttli = backup_state->starttli;
285 : :
286 : : /*
287 : : * Once do_pg_backup_start has been called, ensure that any failure causes
288 : : * us to abort the backup so we don't "leak" a backup counter. For this
289 : : * reason, *all* functionality between do_pg_backup_start() and the end of
290 : : * do_pg_backup_stop() should be inside the error cleanup block!
291 : : */
292 : :
2513 rhaas@postgresql.org 293 [ + + ]: 197 : PG_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
294 : : {
295 : : ListCell *lc;
296 : : tablespaceinfo *newti;
297 : :
298 : : /* If this is an incremental backup, execute preparatory steps. */
1051 299 [ + + ]: 197 : if (ib != NULL)
300 : 15 : PrepareForIncrementalBackup(ib, backup_state);
301 : :
302 : : /* Add a node for the base directory at the end */
330 michael@paquier.xyz 303 : 196 : newti = palloc0_object(tablespaceinfo);
1492 drowley@postgresql.o 304 : 196 : newti->size = -1;
305 : 196 : state.tablespaces = lappend(state.tablespaces, newti);
306 : :
307 : : /*
308 : : * Calculate the total backup size by summing up the size of each
309 : : * tablespace
310 : : */
2438 fujii@postgresql.org 311 [ + - ]: 196 : if (opt->progress)
312 : : {
1826 rhaas@postgresql.org 313 : 196 : basebackup_progress_estimate_backup_size();
314 : :
315 [ + - + + : 430 : foreach(lc, state.tablespaces)
+ + ]
316 : : {
2438 fujii@postgresql.org 317 : 234 : tablespaceinfo *tmp = (tablespaceinfo *) lfirst(lc);
318 : :
2332 rhaas@postgresql.org 319 [ + + ]: 234 : if (tmp->path == NULL)
1826 320 : 196 : tmp->size = sendDir(sink, ".", 1, true, state.tablespaces,
321 : : true, NULL, InvalidOid, ib);
322 : : else
323 : 38 : tmp->size = sendTablespace(sink, tmp->path, tmp->oid, true,
324 : : NULL, ib);
325 : 234 : state.bytes_total += tmp->size;
326 : : }
327 : 196 : state.bytes_total_is_valid = true;
328 : : }
329 : :
330 : : /* notify basebackup sink about start of backup */
331 : 196 : bbsink_begin_backup(sink, &state, SINK_BUFFER_LENGTH);
332 : :
333 : : /*
334 : : * Allow tests to hold the backup after the starting checkpoint but
335 : : * before any file data is sent.
336 : : */
79 dgustafsson@postgres 337 :GNC 196 : INJECTION_POINT("basebackup-before-send-files", NULL);
338 : :
339 : : /* Send off our tablespaces one by one */
1826 rhaas@postgresql.org 340 [ + - + + :CBC 425 : foreach(lc, state.tablespaces)
+ + ]
341 : : {
5777 tgl@sss.pgh.pa.us 342 : 234 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
343 : :
5398 simon@2ndQuadrant.co 344 [ + + ]: 234 : if (ti->path == NULL)
345 : : {
2003 tgl@sss.pgh.pa.us 346 : 196 : bool sendtblspclinks = true;
347 : : char *backup_label;
348 : : ControlFileData *controlData;
349 : : bool crc_ok;
350 : : char *controlFile;
351 : :
1826 rhaas@postgresql.org 352 : 196 : bbsink_begin_archive(sink, "base.tar");
353 : :
354 : : /* In the main tar, include the backup_label first... */
1501 michael@paquier.xyz 355 : 196 : backup_label = build_backup_content(backup_state, false);
356 : 196 : sendFileWithContent(sink, BACKUP_LABEL_FILE,
357 : : backup_label, -1, &manifest);
358 : 196 : pfree(backup_label);
359 : :
360 : : /* Then the tablespace_map file, if required... */
2332 rhaas@postgresql.org 361 [ + + ]: 196 : if (opt->sendtblspcmapfile)
362 : : {
1501 michael@paquier.xyz 363 : 30 : sendFileWithContent(sink, TABLESPACE_MAP,
364 drowley@postgresql.o 364 : 30 : tablespace_map.data, -1, &manifest);
2332 rhaas@postgresql.org 365 : 30 : sendtblspclinks = false;
366 : : }
367 : :
368 : : /* Then the bulk of the files... */
1826 369 : 196 : sendDir(sink, ".", 1, false, state.tablespaces,
370 : : sendtblspclinks, &manifest, InvalidOid, ib);
371 : :
372 : : /* ... and pg_control after everything else. */
8 michael@paquier.xyz 373 : 191 : LWLockAcquire(ControlFileLock, LW_SHARED);
374 : 191 : controlData = get_controlfile(DataDir, &crc_ok);
375 : 191 : LWLockRelease(ControlFileLock);
376 : :
377 [ - + ]: 191 : if (!crc_ok)
5398 simon@2ndQuadrant.co 378 [ # # ]:UBC 0 : ereport(ERROR,
379 : : (errmsg("calculated CRC checksum does not match value stored in file")));
380 : :
381 : : /* Send the full file, zero-padded as it is on disk */
7 michael@paquier.xyz 382 :CBC 191 : controlFile = palloc0(PG_CONTROL_FILE_SIZE);
8 383 : 191 : memcpy(controlFile, controlData, sizeof(ControlFileData));
384 : 191 : pfree(controlData);
385 : :
386 : 191 : sendFileWithContent(sink, XLOG_CONTROL_FILE,
387 : : controlFile, PG_CONTROL_FILE_SIZE,
388 : : &manifest);
7 389 : 191 : pfree(controlFile);
390 : : }
391 : : else
392 : : {
1109 rhaas@postgresql.org 393 : 38 : char *archive_name = psprintf("%u.tar", ti->oid);
394 : :
1826 395 : 38 : bbsink_begin_archive(sink, archive_name);
396 : :
1051 397 : 38 : sendTablespace(sink, ti->path, ti->oid, false, &manifest, ib);
398 : : }
399 : :
400 : : /*
401 : : * If we're including WAL, and this is the main data directory we
402 : : * don't treat this as the end of the tablespace. Instead, we will
403 : : * include the xlog files below and stop afterwards. This is safe
404 : : * since the main data directory is always sent *last*.
405 : : */
5758 magnus@hagander.net 406 [ + + + + ]: 229 : if (opt->includewal && ti->path == NULL)
407 : : {
1826 rhaas@postgresql.org 408 [ - + ]: 15 : Assert(lnext(state.tablespaces, lc) == NULL);
409 : : }
410 : : else
411 : : {
412 : : /* Properly terminate the tarfile. */
262 peter@eisentraut.org 413 : 214 : memset(sink->bbs_buffer, 0, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
414 : 214 : bbsink_archive_contents(sink, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
415 : :
416 : : /* OK, that's the end of the archive. */
1826 rhaas@postgresql.org 417 : 214 : bbsink_end_archive(sink);
418 : : }
419 : : }
420 : :
421 : 191 : basebackup_progress_wait_wal_archive(&state);
1501 michael@paquier.xyz 422 : 191 : do_pg_backup_stop(backup_state, !opt->nowait);
423 : :
424 : 191 : endptr = backup_state->stoppoint;
425 : 191 : endtli = backup_state->stoptli;
426 : :
427 : : /* Deallocate backup-related variables. */
364 drowley@postgresql.o 428 : 191 : pfree(tablespace_map.data);
1501 michael@paquier.xyz 429 : 191 : pfree(backup_state);
430 : : }
2513 rhaas@postgresql.org 431 [ - + ]: 193 : PG_END_ENSURE_ERROR_CLEANUP(do_pg_abort_backup, BoolGetDatum(false));
432 : :
433 : :
5758 magnus@hagander.net 434 [ + + ]: 191 : if (opt->includewal)
435 : : {
436 : : /*
437 : : * We've left the last tar file "open", so we can now append the
438 : : * required WAL files to it.
439 : : */
440 : : char pathbuf[MAXPGPATH];
441 : : XLogSegNo segno;
442 : : XLogSegNo startsegno;
443 : : XLogSegNo endsegno;
444 : : struct stat statbuf;
5054 heikki.linnakangas@i 445 : 15 : List *historyFileList = NIL;
446 : 15 : List *walFileList = NIL;
447 : : char firstoff[MAXFNAMELEN];
448 : : char lastoff[MAXFNAMELEN];
449 : : DIR *dir;
450 : : struct dirent *de;
451 : : ListCell *lc;
452 : : TimeLineID tli;
453 : :
1826 rhaas@postgresql.org 454 : 15 : basebackup_progress_transfer_wal();
455 : :
456 : : /*
457 : : * I'd rather not worry about timelines here, so scan pg_wal and
458 : : * include all WAL files in the range between 'startptr' and 'endptr',
459 : : * regardless of the timeline the file is stamped with. If there are
460 : : * some spurious WAL files belonging to timelines that don't belong in
461 : : * this server's history, they will be included too. Normally there
462 : : * shouldn't be such files, but if there are, there's little harm in
463 : : * including them.
464 : : */
465 : 15 : XLByteToSeg(state.startptr, startsegno, wal_segment_size);
466 : 15 : XLogFileName(firstoff, state.starttli, startsegno, wal_segment_size);
3334 andres@anarazel.de 467 : 15 : XLByteToPrevSeg(endptr, endsegno, wal_segment_size);
1833 rhaas@postgresql.org 468 : 15 : XLogFileName(lastoff, endtli, endsegno, wal_segment_size);
469 : :
3668 470 : 15 : dir = AllocateDir("pg_wal");
471 [ + + ]: 108 : while ((de = ReadDir(dir, "pg_wal")) != NULL)
472 : : {
473 : : /* Does it look like a WAL segment, and is it in the range? */
4199 heikki.linnakangas@i 474 [ + + ]: 93 : if (IsXLogFileName(de->d_name) &&
5054 475 [ + - ]: 33 : strcmp(de->d_name + 8, firstoff + 8) >= 0 &&
476 [ + + ]: 33 : strcmp(de->d_name + 8, lastoff + 8) <= 0)
477 : : {
478 : 15 : walFileList = lappend(walFileList, pstrdup(de->d_name));
479 : : }
480 : : /* Does it look like a timeline history file? */
4199 481 [ - + ]: 78 : else if (IsTLHistoryFileName(de->d_name))
482 : : {
5054 heikki.linnakangas@i 483 :UBC 0 : historyFileList = lappend(historyFileList, pstrdup(de->d_name));
484 : : }
485 : : }
5054 heikki.linnakangas@i 486 :CBC 15 : FreeDir(dir);
487 : :
488 : : /*
489 : : * Before we go any further, check that none of the WAL segments we
490 : : * need were removed.
491 : : */
1826 rhaas@postgresql.org 492 : 15 : CheckXLogRemoved(startsegno, state.starttli);
493 : :
494 : : /*
495 : : * Sort the WAL filenames. We want to send the files in order from
496 : : * oldest to newest, to reduce the chance that a file is recycled
497 : : * before we get a chance to send it over.
498 : : */
2669 tgl@sss.pgh.pa.us 499 : 15 : list_sort(walFileList, compareWalFileNames);
500 : :
501 : : /*
502 : : * There must be at least one xlog file in the pg_wal directory, since
503 : : * we are doing backup-including-xlog.
504 : : */
505 [ - + ]: 15 : if (walFileList == NIL)
4821 magnus@hagander.net 506 [ # # ]:UBC 0 : ereport(ERROR,
507 : : (errmsg("could not find any WAL files")));
508 : :
509 : : /*
510 : : * Sanity check: the first and last segment should cover startptr and
511 : : * endptr, with no gaps in between.
512 : : */
2669 tgl@sss.pgh.pa.us 513 :CBC 15 : XLogFromFileName((char *) linitial(walFileList),
514 : : &tli, &segno, wal_segment_size);
5054 heikki.linnakangas@i 515 [ - + ]: 15 : if (segno != startsegno)
516 : : {
517 : : char startfname[MAXFNAMELEN];
518 : :
1826 rhaas@postgresql.org 519 :UBC 0 : XLogFileName(startfname, state.starttli, startsegno,
520 : : wal_segment_size);
5054 heikki.linnakangas@i 521 [ # # ]: 0 : ereport(ERROR,
522 : : (errmsg("could not find WAL file \"%s\"", startfname)));
523 : : }
2669 tgl@sss.pgh.pa.us 524 [ + - + + :CBC 30 : foreach(lc, walFileList)
+ + ]
525 : : {
526 : 15 : char *walFileName = (char *) lfirst(lc);
4908 bruce@momjian.us 527 : 15 : XLogSegNo currsegno = segno;
528 : 15 : XLogSegNo nextsegno = segno + 1;
529 : :
2669 tgl@sss.pgh.pa.us 530 : 15 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
5054 heikki.linnakangas@i 531 [ + - - + ]: 15 : if (!(nextsegno == segno || currsegno == segno))
532 : : {
533 : : char nextfname[MAXFNAMELEN];
534 : :
1833 rhaas@postgresql.org 535 :UBC 0 : XLogFileName(nextfname, tli, nextsegno, wal_segment_size);
5054 heikki.linnakangas@i 536 [ # # ]: 0 : ereport(ERROR,
537 : : (errmsg("could not find WAL file \"%s\"", nextfname)));
538 : : }
539 : : }
5054 heikki.linnakangas@i 540 [ - + ]:CBC 15 : if (segno != endsegno)
541 : : {
542 : : char endfname[MAXFNAMELEN];
543 : :
1833 rhaas@postgresql.org 544 :UBC 0 : XLogFileName(endfname, endtli, endsegno, wal_segment_size);
5054 heikki.linnakangas@i 545 [ # # ]: 0 : ereport(ERROR,
546 : : (errmsg("could not find WAL file \"%s\"", endfname)));
547 : : }
548 : :
549 : : /* Ok, we have everything we need. Send the WAL files. */
2669 tgl@sss.pgh.pa.us 550 [ + - + + :CBC 30 : foreach(lc, walFileList)
+ + ]
551 : : {
552 : 15 : char *walFileName = (char *) lfirst(lc);
553 : : int fd;
554 : : ssize_t cnt;
5054 heikki.linnakangas@i 555 : 15 : pgoff_t len = 0;
556 : :
2669 tgl@sss.pgh.pa.us 557 : 15 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", walFileName);
558 : 15 : XLogFromFileName(walFileName, &tli, &segno, wal_segment_size);
559 : :
2332 rhaas@postgresql.org 560 : 15 : fd = OpenTransientFile(pathbuf, O_RDONLY | PG_BINARY);
561 [ - + ]: 15 : if (fd < 0)
562 : : {
3055 michael@paquier.xyz 563 :UBC 0 : int save_errno = errno;
564 : :
565 : : /*
566 : : * Most likely reason for this is that the file was already
567 : : * removed by a checkpoint, so check for that to get a better
568 : : * error message.
569 : : */
5054 heikki.linnakangas@i 570 : 0 : CheckXLogRemoved(segno, tli);
571 : :
3055 michael@paquier.xyz 572 : 0 : errno = save_errno;
5054 heikki.linnakangas@i 573 [ # # ]: 0 : ereport(ERROR,
574 : : (errcode_for_file_access(),
575 : : errmsg("could not open file \"%s\": %m", pathbuf)));
576 : : }
577 : :
2332 rhaas@postgresql.org 578 [ - + ]:CBC 15 : if (fstat(fd, &statbuf) != 0)
5054 heikki.linnakangas@i 579 [ # # ]:UBC 0 : ereport(ERROR,
580 : : (errcode_for_file_access(),
581 : : errmsg("could not stat file \"%s\": %m",
582 : : pathbuf)));
3334 andres@anarazel.de 583 [ - + ]:CBC 15 : if (statbuf.st_size != wal_segment_size)
584 : : {
5054 heikki.linnakangas@i 585 :UBC 0 : CheckXLogRemoved(segno, tli);
586 [ # # ]: 0 : ereport(ERROR,
587 : : (errcode_for_file_access(),
588 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
589 : : }
590 : :
591 : : /* send the WAL file itself */
1826 rhaas@postgresql.org 592 :CBC 15 : _tarWriteHeader(sink, pathbuf, NULL, &statbuf, false);
593 : :
594 : 15 : while ((cnt = basebackup_read_file(fd, sink->bbs_buffer,
595 : 7680 : Min(sink->bbs_buffer_length,
596 : : wal_segment_size - len),
2332 597 [ + - ]: 7680 : len, pathbuf, true)) > 0)
598 : : {
5054 heikki.linnakangas@i 599 : 7680 : CheckXLogRemoved(segno, tli);
1826 rhaas@postgresql.org 600 : 7680 : bbsink_archive_contents(sink, cnt);
601 : :
5054 heikki.linnakangas@i 602 : 7680 : len += cnt;
603 : :
3334 andres@anarazel.de 604 [ + + ]: 7680 : if (len == wal_segment_size)
5054 heikki.linnakangas@i 605 : 15 : break;
606 : : }
607 : :
3334 andres@anarazel.de 608 [ - + ]: 15 : if (len != wal_segment_size)
609 : : {
5054 heikki.linnakangas@i 610 :UBC 0 : CheckXLogRemoved(segno, tli);
611 [ # # ]: 0 : ereport(ERROR,
612 : : (errcode_for_file_access(),
613 : : errmsg("unexpected WAL file size \"%s\"", walFileName)));
614 : : }
615 : :
616 : : /*
617 : : * wal_segment_size is a multiple of TAR_BLOCK_SIZE, so no need
618 : : * for padding.
619 : : */
2386 rhaas@postgresql.org 620 [ - + ]:CBC 15 : Assert(wal_segment_size % TAR_BLOCK_SIZE == 0);
621 : :
2332 622 : 15 : CloseTransientFile(fd);
623 : :
624 : : /*
625 : : * Mark file as archived, otherwise files can get archived again
626 : : * after promotion of a new node. This is in line with
627 : : * walreceiver.c always doing an XLogArchiveForceDone() after a
628 : : * complete segment.
629 : : */
2669 tgl@sss.pgh.pa.us 630 : 15 : StatusFilePath(pathbuf, walFileName, ".done");
1088 michael@paquier.xyz 631 : 15 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
632 : : }
633 : :
634 : : /*
635 : : * Send timeline history files too. Only the latest timeline history
636 : : * file is required for recovery, and even that only if there happens
637 : : * to be a timeline switch in the first WAL segment that contains the
638 : : * checkpoint record, or if we're taking a base backup from a standby
639 : : * server and the target timeline changes while the backup is taken.
640 : : * But they are small and highly useful for debugging purposes, so
641 : : * better include them all, always.
642 : : */
5054 heikki.linnakangas@i 643 [ - + - - : 15 : foreach(lc, historyFileList)
- + ]
644 : : {
4908 bruce@momjian.us 645 :UBC 0 : char *fname = lfirst(lc);
646 : :
5054 heikki.linnakangas@i 647 : 0 : snprintf(pathbuf, MAXPGPATH, XLOGDIR "/%s", fname);
648 : :
649 [ # # ]: 0 : if (lstat(pathbuf, &statbuf) != 0)
650 [ # # ]: 0 : ereport(ERROR,
651 : : (errcode_for_file_access(),
652 : : errmsg("could not stat file \"%s\": %m", pathbuf)));
653 : :
1109 rhaas@postgresql.org 654 : 0 : sendFile(sink, pathbuf, pathbuf, &statbuf, false,
655 : : InvalidOid, InvalidOid, InvalidRelFileNumber, 0,
656 : : &manifest, 0, NULL, 0);
657 : :
658 : : /* unconditionally mark file as archived */
4324 andres@anarazel.de 659 : 0 : StatusFilePath(pathbuf, fname, ".done");
1088 michael@paquier.xyz 660 : 0 : sendFileWithContent(sink, pathbuf, "", -1, &manifest);
661 : : }
662 : :
663 : : /* Properly terminate the tar file. */
262 peter@eisentraut.org 664 :CBC 15 : memset(sink->bbs_buffer, 0, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
665 : 15 : bbsink_archive_contents(sink, TAR_NUM_TERMINATION_BLOCKS * TAR_BLOCK_SIZE);
666 : :
667 : : /* OK, that's the end of the archive. */
1826 rhaas@postgresql.org 668 : 15 : bbsink_end_archive(sink);
669 : : }
670 : :
671 : 191 : AddWALInfoToBackupManifest(&manifest, state.startptr, state.starttli,
672 : : endptr, endtli);
673 : :
674 : 191 : SendBackupManifest(&manifest, sink);
675 : :
676 : 191 : bbsink_end_backup(sink, endptr, endtli);
677 : :
3138 magnus@hagander.net 678 [ + + ]: 191 : if (total_checksum_failures)
679 : : {
78 michael@paquier.xyz 680 [ + - ]: 3 : ereport(WARNING,
681 : : (errmsg_plural("%lld total checksum verification failure",
682 : : "%lld total checksum verification failures",
683 : : total_checksum_failures,
684 : : total_checksum_failures)));
685 : :
3138 magnus@hagander.net 686 [ + - ]: 3 : ereport(ERROR,
687 : : (errcode(ERRCODE_DATA_CORRUPTED),
688 : : errmsg("checksum verification failure during base backup")));
689 : : }
690 : :
691 : : /*
692 : : * Make sure to free the manifest before the resource owners as manifests
693 : : * use cryptohash contexts that may depend on resource owners (like
694 : : * OpenSSL).
695 : : */
2162 michael@paquier.xyz 696 : 188 : FreeBackupManifest(&manifest);
697 : :
698 : : /* clean up the resource owner we created */
758 andres@anarazel.de 699 : 188 : ReleaseAuxProcessResources(true);
5777 tgl@sss.pgh.pa.us 700 : 188 : }
701 : :
702 : : /*
703 : : * list_sort comparison function, to compare log/seg portion of WAL segment
704 : : * filenames, ignoring the timeline portion.
705 : : */
706 : : static int
2669 tgl@sss.pgh.pa.us 707 :UBC 0 : compareWalFileNames(const ListCell *a, const ListCell *b)
708 : : {
709 : 0 : char *fna = (char *) lfirst(a);
710 : 0 : char *fnb = (char *) lfirst(b);
711 : :
5054 heikki.linnakangas@i 712 : 0 : return strcmp(fna + 8, fnb + 8);
713 : : }
714 : :
715 : : /*
716 : : * Parse the base backup options passed down by the parser
717 : : */
718 : : static void
5765 magnus@hagander.net 719 :CBC 214 : parse_basebackup_options(List *options, basebackup_options *opt)
720 : : {
721 : : ListCell *lopt;
722 : 214 : bool o_label = false;
723 : 214 : bool o_progress = false;
1857 rhaas@postgresql.org 724 : 214 : bool o_checkpoint = false;
5748 magnus@hagander.net 725 : 214 : bool o_nowait = false;
5758 726 : 214 : bool o_wal = false;
1051 rhaas@postgresql.org 727 : 214 : bool o_incremental = false;
4634 alvherre@alvh.no-ip. 728 : 214 : bool o_maxrate = false;
4195 andrew@dunslane.net 729 : 214 : bool o_tablespace_map = false;
3138 magnus@hagander.net 730 : 214 : bool o_noverify_checksums = false;
2407 rhaas@postgresql.org 731 : 214 : bool o_manifest = false;
732 : 214 : bool o_manifest_checksums = false;
1752 733 : 214 : bool o_target = false;
1815 734 : 214 : bool o_target_detail = false;
1696 735 : 214 : char *target_str = NULL;
736 : 214 : char *target_detail_str = NULL;
1746 737 : 214 : bool o_compression = false;
1688 738 : 214 : bool o_compression_detail = false;
739 : 214 : char *compression_detail_str = NULL;
740 : :
5763 magnus@hagander.net 741 [ + - + - : 2354 : MemSet(opt, 0, sizeof(*opt));
+ - + - +
+ ]
2407 rhaas@postgresql.org 742 : 214 : opt->manifest = MANIFEST_OPTION_NO;
743 : 214 : opt->manifest_checksum_type = CHECKSUM_TYPE_CRC32C;
1668 michael@paquier.xyz 744 : 214 : opt->compression = PG_COMPRESSION_NONE;
745 : 214 : opt->compression_specification.algorithm = PG_COMPRESSION_NONE;
746 : :
5765 magnus@hagander.net 747 [ + - + + : 1604 : foreach(lopt, options)
+ + ]
748 : : {
749 : 1393 : DefElem *defel = (DefElem *) lfirst(lopt);
750 : :
751 [ + + ]: 1393 : if (strcmp(defel->defname, "label") == 0)
752 : : {
753 [ - + ]: 214 : if (o_label)
5765 magnus@hagander.net 754 [ # # ]:UBC 0 : ereport(ERROR,
755 : : (errcode(ERRCODE_SYNTAX_ERROR),
756 : : errmsg("duplicate option \"%s\"", defel->defname)));
1857 rhaas@postgresql.org 757 :CBC 214 : opt->label = defGetString(defel);
5765 magnus@hagander.net 758 : 214 : o_label = true;
759 : : }
760 [ + + ]: 1179 : else if (strcmp(defel->defname, "progress") == 0)
761 : : {
762 [ - + ]: 214 : if (o_progress)
5765 magnus@hagander.net 763 [ # # ]:UBC 0 : ereport(ERROR,
764 : : (errcode(ERRCODE_SYNTAX_ERROR),
765 : : errmsg("duplicate option \"%s\"", defel->defname)));
1857 rhaas@postgresql.org 766 :CBC 214 : opt->progress = defGetBoolean(defel);
5765 magnus@hagander.net 767 : 214 : o_progress = true;
768 : : }
1857 rhaas@postgresql.org 769 [ + + ]: 965 : else if (strcmp(defel->defname, "checkpoint") == 0)
770 : : {
771 : 204 : char *optval = defGetString(defel);
772 : :
773 [ - + ]: 204 : if (o_checkpoint)
5765 magnus@hagander.net 774 [ # # ]:UBC 0 : ereport(ERROR,
775 : : (errcode(ERRCODE_SYNTAX_ERROR),
776 : : errmsg("duplicate option \"%s\"", defel->defname)));
1857 rhaas@postgresql.org 777 [ + - ]:CBC 204 : if (pg_strcasecmp(optval, "fast") == 0)
778 : 204 : opt->fastcheckpoint = true;
1857 rhaas@postgresql.org 779 [ # # ]:UBC 0 : else if (pg_strcasecmp(optval, "spread") == 0)
780 : 0 : opt->fastcheckpoint = false;
781 : : else
782 [ # # ]: 0 : ereport(ERROR,
783 : : (errcode(ERRCODE_SYNTAX_ERROR),
784 : : errmsg("unrecognized checkpoint type: \"%s\"",
785 : : optval)));
1857 rhaas@postgresql.org 786 :CBC 204 : o_checkpoint = true;
787 : : }
788 [ + + ]: 761 : else if (strcmp(defel->defname, "wait") == 0)
789 : : {
5748 magnus@hagander.net 790 [ - + ]: 204 : if (o_nowait)
5748 magnus@hagander.net 791 [ # # ]:UBC 0 : ereport(ERROR,
792 : : (errcode(ERRCODE_SYNTAX_ERROR),
793 : : errmsg("duplicate option \"%s\"", defel->defname)));
1857 rhaas@postgresql.org 794 :CBC 204 : opt->nowait = !defGetBoolean(defel);
5748 magnus@hagander.net 795 : 204 : o_nowait = true;
796 : : }
5758 797 [ + + ]: 557 : else if (strcmp(defel->defname, "wal") == 0)
798 : : {
799 [ - + ]: 19 : if (o_wal)
5758 magnus@hagander.net 800 [ # # ]:UBC 0 : ereport(ERROR,
801 : : (errcode(ERRCODE_SYNTAX_ERROR),
802 : : errmsg("duplicate option \"%s\"", defel->defname)));
1857 rhaas@postgresql.org 803 :CBC 19 : opt->includewal = defGetBoolean(defel);
5758 magnus@hagander.net 804 : 19 : o_wal = true;
805 : : }
1051 rhaas@postgresql.org 806 [ + + ]: 538 : else if (strcmp(defel->defname, "incremental") == 0)
807 : : {
808 [ - + ]: 15 : if (o_incremental)
1051 rhaas@postgresql.org 809 [ # # ]:UBC 0 : ereport(ERROR,
810 : : (errcode(ERRCODE_SYNTAX_ERROR),
811 : : errmsg("duplicate option \"%s\"", defel->defname)));
1051 rhaas@postgresql.org 812 :CBC 15 : opt->incremental = defGetBoolean(defel);
813 [ + - - + ]: 15 : if (opt->incremental && !summarize_wal)
1051 rhaas@postgresql.org 814 [ # # ]:UBC 0 : ereport(ERROR,
815 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
816 : : errmsg("incremental backups cannot be taken unless WAL summarization is enabled")));
1051 rhaas@postgresql.org 817 :CBC 15 : o_incremental = true;
818 : : }
4634 alvherre@alvh.no-ip. 819 [ + + ]: 523 : else if (strcmp(defel->defname, "max_rate") == 0)
820 : : {
821 : : int64 maxrate;
822 : :
823 [ - + ]: 1 : if (o_maxrate)
4634 alvherre@alvh.no-ip. 824 [ # # ]:UBC 0 : ereport(ERROR,
825 : : (errcode(ERRCODE_SYNTAX_ERROR),
826 : : errmsg("duplicate option \"%s\"", defel->defname)));
827 : :
1857 rhaas@postgresql.org 828 :CBC 1 : maxrate = defGetInt64(defel);
4634 alvherre@alvh.no-ip. 829 [ + - - + ]: 1 : if (maxrate < MAX_RATE_LOWER || maxrate > MAX_RATE_UPPER)
4634 alvherre@alvh.no-ip. 830 [ # # ]:UBC 0 : ereport(ERROR,
831 : : (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
832 : : errmsg("%" PRId64 " is outside the valid range for parameter \"%s\" (%d .. %d)",
833 : : maxrate, "MAX_RATE", MAX_RATE_LOWER, MAX_RATE_UPPER)));
834 : :
4634 alvherre@alvh.no-ip. 835 :CBC 1 : opt->maxrate = (uint32) maxrate;
836 : 1 : o_maxrate = true;
837 : : }
4195 andrew@dunslane.net 838 [ + + ]: 522 : else if (strcmp(defel->defname, "tablespace_map") == 0)
839 : : {
840 [ - + ]: 36 : if (o_tablespace_map)
4195 andrew@dunslane.net 841 [ # # ]:UBC 0 : ereport(ERROR,
842 : : (errcode(ERRCODE_SYNTAX_ERROR),
843 : : errmsg("duplicate option \"%s\"", defel->defname)));
1857 rhaas@postgresql.org 844 :CBC 36 : opt->sendtblspcmapfile = defGetBoolean(defel);
4195 andrew@dunslane.net 845 : 36 : o_tablespace_map = true;
846 : : }
1857 rhaas@postgresql.org 847 [ + + ]: 486 : else if (strcmp(defel->defname, "verify_checksums") == 0)
848 : : {
3138 magnus@hagander.net 849 [ - + ]: 1 : if (o_noverify_checksums)
3138 magnus@hagander.net 850 [ # # ]:UBC 0 : ereport(ERROR,
851 : : (errcode(ERRCODE_SYNTAX_ERROR),
852 : : errmsg("duplicate option \"%s\"", defel->defname)));
1857 rhaas@postgresql.org 853 :CBC 1 : noverify_checksums = !defGetBoolean(defel);
3138 magnus@hagander.net 854 : 1 : o_noverify_checksums = true;
855 : : }
2407 rhaas@postgresql.org 856 [ + + ]: 485 : else if (strcmp(defel->defname, "manifest") == 0)
857 : : {
1857 858 : 213 : char *optval = defGetString(defel);
859 : : bool manifest_bool;
860 : :
2407 861 [ - + ]: 213 : if (o_manifest)
2407 rhaas@postgresql.org 862 [ # # ]:UBC 0 : ereport(ERROR,
863 : : (errcode(ERRCODE_SYNTAX_ERROR),
864 : : errmsg("duplicate option \"%s\"", defel->defname)));
2407 rhaas@postgresql.org 865 [ + - ]:CBC 213 : if (parse_bool(optval, &manifest_bool))
866 : : {
867 [ + - ]: 213 : if (manifest_bool)
868 : 213 : opt->manifest = MANIFEST_OPTION_YES;
869 : : else
2407 rhaas@postgresql.org 870 :UBC 0 : opt->manifest = MANIFEST_OPTION_NO;
871 : : }
2407 rhaas@postgresql.org 872 [ # # ]:LBC (1) : else if (pg_strcasecmp(optval, "force-encode") == 0)
873 : (1) : opt->manifest = MANIFEST_OPTION_FORCE_ENCODE;
874 : : else
2407 rhaas@postgresql.org 875 [ # # ]:UBC 0 : ereport(ERROR,
876 : : (errcode(ERRCODE_SYNTAX_ERROR),
877 : : errmsg("unrecognized manifest option: \"%s\"",
878 : : optval)));
2407 rhaas@postgresql.org 879 :CBC 213 : o_manifest = true;
880 : : }
881 [ + + ]: 272 : else if (strcmp(defel->defname, "manifest_checksums") == 0)
882 : : {
1857 883 : 14 : char *optval = defGetString(defel);
884 : :
2407 885 [ - + ]: 14 : if (o_manifest_checksums)
2407 rhaas@postgresql.org 886 [ # # ]:UBC 0 : ereport(ERROR,
887 : : (errcode(ERRCODE_SYNTAX_ERROR),
888 : : errmsg("duplicate option \"%s\"", defel->defname)));
2407 rhaas@postgresql.org 889 [ + + ]:CBC 14 : if (!pg_checksum_parse_type(optval,
890 : : &opt->manifest_checksum_type))
891 [ + - ]: 2 : ereport(ERROR,
892 : : (errcode(ERRCODE_SYNTAX_ERROR),
893 : : errmsg("unrecognized checksum algorithm: \"%s\"",
894 : : optval)));
895 : 12 : o_manifest_checksums = true;
896 : : }
1752 897 [ + + ]: 258 : else if (strcmp(defel->defname, "target") == 0)
898 : : {
899 [ - + ]: 212 : if (o_target)
1752 rhaas@postgresql.org 900 [ # # ]:UBC 0 : ereport(ERROR,
901 : : (errcode(ERRCODE_SYNTAX_ERROR),
902 : : errmsg("duplicate option \"%s\"", defel->defname)));
1696 rhaas@postgresql.org 903 :CBC 212 : target_str = defGetString(defel);
1752 904 : 212 : o_target = true;
905 : : }
1815 906 [ + + ]: 46 : else if (strcmp(defel->defname, "target_detail") == 0)
907 : : {
908 : 8 : char *optval = defGetString(defel);
909 : :
910 [ - + ]: 8 : if (o_target_detail)
1815 rhaas@postgresql.org 911 [ # # ]:UBC 0 : ereport(ERROR,
912 : : (errcode(ERRCODE_SYNTAX_ERROR),
913 : : errmsg("duplicate option \"%s\"", defel->defname)));
1696 rhaas@postgresql.org 914 :CBC 8 : target_detail_str = optval;
1815 915 : 8 : o_target_detail = true;
916 : : }
1746 917 [ + + ]: 38 : else if (strcmp(defel->defname, "compression") == 0)
918 : : {
919 : 26 : char *optval = defGetString(defel);
920 : :
921 [ - + ]: 26 : if (o_compression)
1746 rhaas@postgresql.org 922 [ # # ]:UBC 0 : ereport(ERROR,
923 : : (errcode(ERRCODE_SYNTAX_ERROR),
924 : : errmsg("duplicate option \"%s\"", defel->defname)));
1668 michael@paquier.xyz 925 [ + + ]:CBC 26 : if (!parse_compress_algorithm(optval, &opt->compression))
1746 rhaas@postgresql.org 926 [ + - ]: 1 : ereport(ERROR,
927 : : (errcode(ERRCODE_SYNTAX_ERROR),
928 : : errmsg("unrecognized compression algorithm: \"%s\"",
929 : : optval)));
930 : 25 : o_compression = true;
931 : : }
1688 932 [ + - ]: 12 : else if (strcmp(defel->defname, "compression_detail") == 0)
933 : : {
934 [ - + ]: 12 : if (o_compression_detail)
1746 rhaas@postgresql.org 935 [ # # ]:UBC 0 : ereport(ERROR,
936 : : (errcode(ERRCODE_SYNTAX_ERROR),
937 : : errmsg("duplicate option \"%s\"", defel->defname)));
1688 rhaas@postgresql.org 938 :CBC 12 : compression_detail_str = defGetString(defel);
939 : 12 : o_compression_detail = true;
940 : : }
941 : : else
1746 rhaas@postgresql.org 942 [ # # ]:UBC 0 : ereport(ERROR,
943 : : (errcode(ERRCODE_SYNTAX_ERROR),
944 : : errmsg("unrecognized base backup option: \"%s\"",
945 : : defel->defname)));
946 : : }
947 : :
5765 magnus@hagander.net 948 [ - + ]:CBC 211 : if (opt->label == NULL)
5765 magnus@hagander.net 949 :UBC 0 : opt->label = "base backup";
2407 rhaas@postgresql.org 950 [ + + ]:CBC 211 : if (opt->manifest == MANIFEST_OPTION_NO)
951 : : {
952 [ - + ]: 1 : if (o_manifest_checksums)
2407 rhaas@postgresql.org 953 [ # # ]:UBC 0 : ereport(ERROR,
954 : : (errcode(ERRCODE_SYNTAX_ERROR),
955 : : errmsg("manifest checksums require a backup manifest")));
2407 rhaas@postgresql.org 956 :CBC 1 : opt->manifest_checksum_type = CHECKSUM_TYPE_NONE;
957 : : }
958 : :
1696 959 [ - + ]: 211 : if (target_str == NULL)
960 : : {
1696 rhaas@postgresql.org 961 [ # # ]:UBC 0 : if (target_detail_str != NULL)
1815 962 [ # # ]: 0 : ereport(ERROR,
963 : : (errcode(ERRCODE_SYNTAX_ERROR),
964 : : errmsg("target detail cannot be used without target")));
1696 965 : 0 : opt->use_copytblspc = true;
966 : 0 : opt->send_to_client = true;
967 : : }
1696 rhaas@postgresql.org 968 [ + + ]:CBC 211 : else if (strcmp(target_str, "client") == 0)
969 : : {
970 [ - + ]: 197 : if (target_detail_str != NULL)
1815 rhaas@postgresql.org 971 [ # # ]:UBC 0 : ereport(ERROR,
972 : : (errcode(ERRCODE_SYNTAX_ERROR),
973 : : errmsg("target \"%s\" does not accept a target detail",
974 : : target_str)));
1696 rhaas@postgresql.org 975 :CBC 197 : opt->send_to_client = true;
976 : : }
977 : : else
978 : 12 : opt->target_handle =
979 : 14 : BaseBackupGetTargetHandle(target_str, target_detail_str);
980 : :
1688 981 [ + + - + ]: 209 : if (o_compression_detail && !o_compression)
1746 rhaas@postgresql.org 982 [ # # ]:UBC 0 : ereport(ERROR,
983 : : (errcode(ERRCODE_SYNTAX_ERROR),
984 : : errmsg("compression detail cannot be specified unless compression is enabled")));
985 : :
1688 rhaas@postgresql.org 986 [ + + ]:CBC 209 : if (o_compression)
987 : : {
988 : : char *error_detail;
989 : :
1668 michael@paquier.xyz 990 : 23 : parse_compress_specification(opt->compression, compression_detail_str,
991 : : &opt->compression_specification);
992 : : error_detail =
993 : 23 : validate_compress_specification(&opt->compression_specification);
1688 rhaas@postgresql.org 994 [ + + ]: 23 : if (error_detail != NULL)
995 [ + - ]: 9 : ereport(ERROR,
996 : : errcode(ERRCODE_SYNTAX_ERROR),
997 : : errmsg("invalid compression specification: %s",
998 : : error_detail));
999 : : }
5765 magnus@hagander.net 1000 : 200 : }
1001 : :
1002 : :
1003 : : /*
1004 : : * SendBaseBackup() - send a complete base backup.
1005 : : *
1006 : : * The function will put the system into backup mode like pg_backup_start()
1007 : : * does, so that the backup is consistent even though we read directly from
1008 : : * the filesystem, bypassing the buffer cache.
1009 : : */
1010 : : void
1051 rhaas@postgresql.org 1011 : 215 : SendBaseBackup(BaseBackupCmd *cmd, IncrementalBackupInfo *ib)
1012 : : {
1013 : : basebackup_options opt;
1014 : : bbsink *sink;
1577 fujii@postgresql.org 1015 : 215 : SessionBackupState status = get_backup_status();
1016 : :
1017 [ + + ]: 215 : if (status == SESSION_BACKUP_RUNNING)
1018 [ + - ]: 1 : ereport(ERROR,
1019 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1020 : : errmsg("a backup is already in progress in this session")));
1021 : :
5765 magnus@hagander.net 1022 : 214 : parse_basebackup_options(cmd->options, &opt);
1023 : :
5777 1024 : 200 : WalSndSetState(WALSNDSTATE_BACKUP);
1025 : :
5778 1026 [ + - ]: 200 : if (update_process_title)
1027 : : {
1028 : : char activitymsg[50];
1029 : :
1030 : 200 : snprintf(activitymsg, sizeof(activitymsg), "sending backup \"%s\"",
1031 : : opt.label);
2430 peter@eisentraut.org 1032 : 200 : set_ps_display(activitymsg);
1033 : : }
1034 : :
1035 : : /*
1036 : : * If we're asked to perform an incremental backup and the user has not
1037 : : * supplied a manifest, that's an ERROR.
1038 : : *
1039 : : * If we're asked to perform a full backup and the user did supply a
1040 : : * manifest, just ignore it.
1041 : : */
1051 rhaas@postgresql.org 1042 [ + + ]: 200 : if (!opt.incremental)
1043 : 185 : ib = NULL;
1044 [ - + ]: 15 : else if (ib == NULL)
1051 rhaas@postgresql.org 1045 [ # # ]:UBC 0 : ereport(ERROR,
1046 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1047 : : errmsg("must UPLOAD_MANIFEST before performing an incremental BASE_BACKUP")));
1048 : :
1049 : : /*
1050 : : * If the target is specifically 'client' then set up to stream the backup
1051 : : * to the client; otherwise, it's being sent someplace else and should not
1052 : : * be sent to the client. BaseBackupGetSink has the job of setting up a
1053 : : * sink to send the backup data wherever it needs to go.
1054 : : */
1696 rhaas@postgresql.org 1055 :CBC 200 : sink = bbsink_copystream_new(opt.send_to_client);
1056 [ + + ]: 200 : if (opt.target_handle != NULL)
1057 : 12 : sink = BaseBackupGetSink(opt.target_handle, sink);
1058 : :
1059 : : /* Set up network throttling, if client requested it */
1826 1060 [ + + ]: 197 : if (opt.maxrate > 0)
1061 : 1 : sink = bbsink_throttle_new(sink, opt.maxrate);
1062 : :
1063 : : /* Set up server-side compression, if client requested it */
1668 michael@paquier.xyz 1064 [ + + ]: 197 : if (opt.compression == PG_COMPRESSION_GZIP)
1688 rhaas@postgresql.org 1065 : 2 : sink = bbsink_gzip_new(sink, &opt.compression_specification);
1668 michael@paquier.xyz 1066 [ + + ]: 195 : else if (opt.compression == PG_COMPRESSION_LZ4)
1688 rhaas@postgresql.org 1067 : 3 : sink = bbsink_lz4_new(sink, &opt.compression_specification);
1668 michael@paquier.xyz 1068 [ - + ]: 192 : else if (opt.compression == PG_COMPRESSION_ZSTD)
1688 rhaas@postgresql.org 1069 :UBC 0 : sink = bbsink_zstd_new(sink, &opt.compression_specification);
1070 : :
1071 : : /* Set up progress reporting. */
457 msawada@postgresql.o 1072 :CBC 197 : sink = bbsink_progress_new(sink, opt.progress, opt.incremental);
1073 : :
1074 : : /*
1075 : : * Perform the base backup, but make sure we clean up the bbsink even if
1076 : : * an error occurs.
1077 : : */
1826 rhaas@postgresql.org 1078 [ + + ]: 197 : PG_TRY();
1079 : : {
1051 1080 : 197 : perform_base_backup(&opt, sink, ib);
1081 : : }
1826 1082 : 5 : PG_FINALLY();
1083 : : {
1084 : 193 : bbsink_cleanup(sink);
1085 : : }
1086 [ + + ]: 193 : PG_END_TRY();
5754 magnus@hagander.net 1087 : 188 : }
1088 : :
1089 : : /*
1090 : : * Inject a file with given name and content in the output tar stream.
1091 : : *
1092 : : * "len" can optionally be set to an arbitrary length of data sent. If set
1093 : : * to -1, the content sent is treated as a string with strlen() as length.
1094 : : */
1095 : : static void
1826 rhaas@postgresql.org 1096 : 432 : sendFileWithContent(bbsink *sink, const char *filename, const char *content,
1097 : : int len, backup_manifest_info *manifest)
1098 : : {
1099 : : struct stat statbuf;
1088 michael@paquier.xyz 1100 : 432 : int bytes_done = 0;
1101 : : pg_checksum_context checksum_ctx;
1102 : :
2164 1103 [ - + ]: 432 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
2164 michael@paquier.xyz 1104 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1105 : : filename);
1106 : :
1088 michael@paquier.xyz 1107 [ + + ]:CBC 432 : if (len < 0)
1108 : 241 : len = strlen(content);
1109 : :
1110 : : /*
1111 : : * Construct a stat struct for the file we're injecting in the tar.
1112 : : */
1113 : :
1114 : : /* Windows doesn't have the concept of uid and gid */
1115 : : #ifdef WIN32
1116 : : statbuf.st_uid = 0;
1117 : : statbuf.st_gid = 0;
1118 : : #else
5757 heikki.linnakangas@i 1119 : 432 : statbuf.st_uid = geteuid();
1120 : 432 : statbuf.st_gid = getegid();
1121 : : #endif
1122 : 432 : statbuf.st_mtime = time(NULL);
3134 sfrost@snowman.net 1123 : 432 : statbuf.st_mode = pg_file_create_mode;
5757 heikki.linnakangas@i 1124 : 432 : statbuf.st_size = len;
1125 : :
1826 rhaas@postgresql.org 1126 : 432 : _tarWriteHeader(sink, filename, NULL, &statbuf, false);
1127 : :
283 peter@eisentraut.org 1128 [ - + ]: 432 : if (pg_checksum_update(&checksum_ctx, (const uint8 *) content, len) < 0)
1826 rhaas@postgresql.org 1129 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of file \"%s\"",
1130 : : filename);
1131 : :
1826 rhaas@postgresql.org 1132 [ + + ]:CBC 825 : while (bytes_done < len)
1133 : : {
1134 : 393 : size_t remaining = len - bytes_done;
1135 : 393 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1136 : :
1137 : 393 : memcpy(sink->bbs_buffer, content, nbytes);
1138 : 393 : bbsink_archive_contents(sink, nbytes);
1139 : 393 : bytes_done += nbytes;
1424 1140 : 393 : content += nbytes;
1141 : : }
1142 : :
1826 1143 : 432 : _tarWritePadding(sink, len);
1144 : :
1109 1145 : 432 : AddFileToBackupManifest(manifest, InvalidOid, filename, len,
2387 1146 : 432 : (pg_time_t) statbuf.st_mtime, &checksum_ctx);
5757 heikki.linnakangas@i 1147 : 432 : }
1148 : :
1149 : : /*
1150 : : * Include the tablespace directory pointed to by 'path' in the output tar
1151 : : * stream. If 'sizeonly' is true, we just calculate a total length and return
1152 : : * it, without actually sending anything.
1153 : : *
1154 : : * Only used to send auxiliary tablespaces, not PGDATA.
1155 : : */
1156 : : static int64
1109 rhaas@postgresql.org 1157 : 76 : sendTablespace(bbsink *sink, char *path, Oid spcoid, bool sizeonly,
1158 : : backup_manifest_info *manifest, IncrementalBackupInfo *ib)
1159 : : {
1160 : : int64 size;
1161 : : char pathbuf[MAXPGPATH];
1162 : : struct stat statbuf;
1163 : :
1164 : : /*
1165 : : * 'path' points to the tablespace location, but we only want to include
1166 : : * the version directory in it that belongs to us.
1167 : : */
4973 heikki.linnakangas@i 1168 : 76 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path,
1169 : : TABLESPACE_VERSION_DIRECTORY);
1170 : :
1171 : : /*
1172 : : * Store a directory entry in the tar file so we get the permissions
1173 : : * right.
1174 : : */
1175 [ - + ]: 76 : if (lstat(pathbuf, &statbuf) != 0)
1176 : : {
4973 heikki.linnakangas@i 1177 [ # # ]:UBC 0 : if (errno != ENOENT)
1178 [ # # ]: 0 : ereport(ERROR,
1179 : : (errcode_for_file_access(),
1180 : : errmsg("could not stat file or directory \"%s\": %m",
1181 : : pathbuf)));
1182 : :
1183 : : /* If the tablespace went away while scanning, it's no error. */
1184 : 0 : return 0;
1185 : : }
1186 : :
1826 rhaas@postgresql.org 1187 :CBC 76 : size = _tarWriteHeader(sink, TABLESPACE_VERSION_DIRECTORY, NULL, &statbuf,
1188 : : sizeonly);
1189 : :
1190 : : /* Send all the files in the tablespace version directory */
1191 : 76 : size += sendDir(sink, pathbuf, strlen(path), sizeonly, NIL, true, manifest,
1192 : : spcoid, ib);
1193 : :
4973 heikki.linnakangas@i 1194 : 76 : return size;
1195 : : }
1196 : :
1197 : : /*
1198 : : * Include all files from the given directory in the output tar stream. If
1199 : : * 'sizeonly' is true, we just calculate a total length and return it, without
1200 : : * actually sending anything. A file that will be sent incrementally is
1201 : : * counted at the size of the incremental file.
1202 : : *
1203 : : * Omit any directory in the tablespaces list, to avoid backing up
1204 : : * tablespaces twice when they were created inside PGDATA.
1205 : : *
1206 : : * If sendtblspclinks is true, we need to include symlink
1207 : : * information in the tar file. If not, we can skip that
1208 : : * as it will be sent separately in the tablespace_map file.
1209 : : */
1210 : : static int64
1826 rhaas@postgresql.org 1211 : 6769 : sendDir(bbsink *sink, const char *path, int basepathlen, bool sizeonly,
1212 : : List *tablespaces, bool sendtblspclinks, backup_manifest_info *manifest,
1213 : : Oid spcoid, IncrementalBackupInfo *ib)
1214 : : {
1215 : : DIR *dir;
1216 : : struct dirent *de;
1217 : : char pathbuf[MAXPGPATH * 2];
1218 : : struct stat statbuf;
5778 magnus@hagander.net 1219 : 6769 : int64 size = 0;
1220 : : const char *lastDir; /* Split last dir from parent path. */
1087 rhaas@postgresql.org 1221 : 6769 : bool isRelationDir = false; /* Does directory contain relations? */
1051 1222 : 6769 : bool isGlobalDir = false;
1087 1223 : 6769 : Oid dboid = InvalidOid;
1051 1224 : 6769 : BlockNumber *relative_block_numbers = NULL;
1225 : :
1226 : : /*
1227 : : * Since this array is relatively large, avoid putting it on the stack.
1228 : : * But we don't need it at all if this is not an incremental backup.
1229 : : */
1230 [ + + ]: 6769 : if (ib != NULL)
330 michael@paquier.xyz 1231 : 484 : relative_block_numbers = palloc_array(BlockNumber, RELSEG_SIZE);
1232 : :
1233 : : /*
1234 : : * Determine if the current path is a database directory that can contain
1235 : : * relations.
1236 : : *
1237 : : * Start by finding the location of the delimiter between the parent path
1238 : : * and the current path.
1239 : : */
3149 teodor@sigaev.ru 1240 : 6769 : lastDir = last_dir_separator(path);
1241 : :
1242 : : /* Does this path look like a database path (i.e. all digits)? */
1243 [ + + ]: 6769 : if (lastDir != NULL &&
1244 [ + + ]: 6377 : strspn(lastDir + 1, "0123456789") == strlen(lastDir + 1))
1245 : 1250 : {
1246 : : /* Part of path that contains the parent directory. */
3115 tgl@sss.pgh.pa.us 1247 : 1250 : int parentPathLen = lastDir - path;
1248 : :
1249 : : /*
1250 : : * Mark path as a database directory if the parent path is either
1251 : : * $PGDATA/base or a tablespace version path.
1252 : : */
3149 teodor@sigaev.ru 1253 [ + + ]: 1250 : if (strncmp(path, "./base", parentPathLen) == 0 ||
1254 [ + - ]: 54 : (parentPathLen >= (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) &&
1255 [ + - ]: 54 : strncmp(lastDir - (sizeof(TABLESPACE_VERSION_DIRECTORY) - 1),
1256 : : TABLESPACE_VERSION_DIRECTORY,
1257 : : sizeof(TABLESPACE_VERSION_DIRECTORY) - 1) == 0))
1258 : : {
1087 rhaas@postgresql.org 1259 : 1250 : isRelationDir = true;
1260 : 1250 : dboid = atooid(lastDir + 1);
1261 : : }
1262 : : }
1263 [ + + ]: 5519 : else if (strcmp(path, "./global") == 0)
1264 : : {
1265 : 389 : isRelationDir = true;
1051 1266 : 389 : isGlobalDir = true;
1267 : : }
1268 : :
5778 magnus@hagander.net 1269 : 6769 : dir = AllocateDir(path);
1270 [ + + ]: 418331 : while ((de = ReadDir(dir, path)) != NULL)
1271 : : {
1272 : : int excludeIdx;
1273 : : bool excludeFound;
1087 rhaas@postgresql.org 1274 : 411571 : RelFileNumber relfilenumber = InvalidRelFileNumber;
1275 : 411571 : ForkNumber relForkNum = InvalidForkNumber;
1276 : 411571 : unsigned segno = 0;
1277 : 411571 : bool isRelationFile = false;
1278 : :
1279 : : /* Skip special stuff */
5778 magnus@hagander.net 1280 [ + + + + ]: 411571 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1281 : 19130 : continue;
1282 : :
1283 : : /* Skip temporary files */
heikki.linnakangas@i 1284 [ + + ]: 398047 : if (strncmp(de->d_name,
1285 : : PG_TEMP_FILE_PREFIX,
1286 : : strlen(PG_TEMP_FILE_PREFIX)) == 0)
1287 : 387 : continue;
1288 : :
1289 : : /* Skip macOS system files */
996 dgustafsson@postgres 1290 [ + + ]: 397660 : if (strcmp(de->d_name, ".DS_Store") == 0)
1291 : 68 : continue;
1292 : :
1293 : : /*
1294 : : * Check if the postmaster has signaled us to exit, and abort with an
1295 : : * error in that case. The error handler further up will call
1296 : : * do_pg_abort_backup() for us. Also check that if the backup was
1297 : : * started while still in recovery, the server wasn't promoted.
1298 : : * do_pg_backup_stop() will check that too, but it's better to stop
1299 : : * the backup early than continue to the end and fail there.
1300 : : */
5075 heikki.linnakangas@i 1301 [ + + ]: 397592 : CHECK_FOR_INTERRUPTS();
1302 [ - + ]: 397588 : if (RecoveryInProgress() != backup_started_in_recovery)
5774 magnus@hagander.net 1303 [ # # ]:UBC 0 : ereport(ERROR,
1304 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
1305 : : errmsg("the standby was promoted during online backup"),
1306 : : errhint("This means that the backup being taken is corrupt "
1307 : : "and should not be used. "
1308 : : "Try taking another online backup.")));
1309 : :
1310 : : /* Scan for files that should be excluded */
3690 peter_e@gmx.net 1311 :CBC 397588 : excludeFound = false;
2446 michael@paquier.xyz 1312 [ + + ]: 3572482 : for (excludeIdx = 0; excludeFiles[excludeIdx].name != NULL; excludeIdx++)
1313 : : {
1314 : 3176429 : int cmplen = strlen(excludeFiles[excludeIdx].name);
1315 : :
1316 [ + + ]: 3176429 : if (!excludeFiles[excludeIdx].match_prefix)
1317 : 2778973 : cmplen++;
1318 [ + + ]: 3176429 : if (strncmp(de->d_name, excludeFiles[excludeIdx].name, cmplen) == 0)
1319 : : {
3690 peter_e@gmx.net 1320 [ + + ]: 1535 : elog(DEBUG1, "file \"%s\" excluded from backup", de->d_name);
1321 : 1535 : excludeFound = true;
1322 : 1535 : break;
1323 : : }
1324 : : }
1325 : :
1326 [ + + ]: 397588 : if (excludeFound)
5778 magnus@hagander.net 1327 : 1535 : continue;
1328 : :
1329 : : /*
1330 : : * If there could be non-temporary relation files in this directory,
1331 : : * try to parse the filename.
1332 : : */
1087 rhaas@postgresql.org 1333 [ + + ]: 396053 : if (isRelationDir)
1334 : : isRelationFile =
1335 : 382473 : parse_filename_for_nontemp_relation(de->d_name,
1336 : : &relfilenumber,
1337 : : &relForkNum, &segno);
1338 : :
1339 : : /* Exclude all forks for unlogged tables except the init fork */
1340 [ + + + + ]: 396053 : if (isRelationFile && relForkNum != INIT_FORKNUM)
1341 : : {
1342 : : char initForkFile[MAXPGPATH];
1343 : :
1344 : : /*
1345 : : * If any other type of fork, check if there is an init fork with
1346 : : * the same RelFileNumber. If so, the file can be excluded.
1347 : : */
1348 : 379188 : snprintf(initForkFile, sizeof(initForkFile), "%s/%u_init",
1349 : : path, relfilenumber);
1350 : :
1351 [ + + ]: 379188 : if (lstat(initForkFile, &statbuf) == 0)
1352 : : {
1353 [ - + ]: 81 : elog(DEBUG2,
1354 : : "unlogged relation file \"%s\" excluded from backup",
1355 : : de->d_name);
1356 : :
1357 : 81 : continue;
1358 : : }
1359 : : }
1360 : :
1361 : : /* Exclude temporary relations */
1362 [ + + + + ]: 395972 : if (OidIsValid(dboid) && looks_like_temp_rel_name(de->d_name))
1363 : : {
3145 teodor@sigaev.ru 1364 [ - + ]: 36 : elog(DEBUG2,
1365 : : "temporary relation file \"%s\" excluded from backup",
1366 : : de->d_name);
1367 : :
1368 : 36 : continue;
1369 : : }
1370 : :
3495 peter_e@gmx.net 1371 : 395936 : snprintf(pathbuf, sizeof(pathbuf), "%s/%s", path, de->d_name);
1372 : :
1373 : : /* Skip pg_control here to back up it last */
577 fujii@postgresql.org 1374 [ + + ]: 395936 : if (strcmp(pathbuf, "./" XLOG_CONTROL_FILE) == 0)
5398 simon@2ndQuadrant.co 1375 : 388 : continue;
1376 : :
5778 magnus@hagander.net 1377 [ - + ]: 395548 : if (lstat(pathbuf, &statbuf) != 0)
1378 : : {
5778 magnus@hagander.net 1379 [ # # ]:UBC 0 : if (errno != ENOENT)
1380 [ # # ]: 0 : ereport(ERROR,
1381 : : (errcode_for_file_access(),
1382 : : errmsg("could not stat file or directory \"%s\": %m",
1383 : : pathbuf)));
1384 : :
1385 : : /* If the file went away while scanning, it's not an error. */
1386 : 0 : continue;
1387 : : }
1388 : :
1389 : : /* Scan for directories whose contents should be excluded */
3690 peter_e@gmx.net 1390 :CBC 395548 : excludeFound = false;
1391 [ + + ]: 3153481 : for (excludeIdx = 0; excludeDirContents[excludeIdx] != NULL; excludeIdx++)
1392 : : {
1393 [ + + ]: 2760656 : if (strcmp(de->d_name, excludeDirContents[excludeIdx]) == 0)
1394 : : {
1395 [ + + ]: 2723 : elog(DEBUG1, "contents of directory \"%s\" excluded from backup", de->d_name);
1850 rhaas@postgresql.org 1396 : 2723 : convert_link_to_directory(pathbuf, &statbuf);
1826 1397 : 2723 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1398 : : &statbuf, sizeonly);
3690 peter_e@gmx.net 1399 : 2723 : excludeFound = true;
1400 : 2723 : break;
1401 : : }
1402 : : }
1403 : :
1404 [ + + ]: 395548 : if (excludeFound)
1405 : 2723 : continue;
1406 : :
1407 : : /*
1408 : : * We can skip pg_wal, the WAL segments need to be fetched from the
1409 : : * WAL archive anyway. But include it as an empty directory anyway, so
1410 : : * we get permissions right.
1411 : : */
3668 rhaas@postgresql.org 1412 [ + + ]: 392825 : if (strcmp(pathbuf, "./pg_wal") == 0)
1413 : : {
1414 : : /* If pg_wal is a symlink, write it as a directory anyway */
1850 1415 : 388 : convert_link_to_directory(pathbuf, &statbuf);
1826 1416 : 388 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL,
1417 : : &statbuf, sizeonly);
1418 : :
1419 : : /*
1420 : : * Also send archive_status and summaries directories (by
1421 : : * hackishly reusing statbuf from above ...).
1422 : : */
1423 : 388 : size += _tarWriteHeader(sink, "./pg_wal/archive_status", NULL,
1424 : : &statbuf, sizeonly);
1051 1425 : 388 : size += _tarWriteHeader(sink, "./pg_wal/summaries", NULL,
1426 : : &statbuf, sizeonly);
1427 : :
3668 1428 : 388 : continue; /* don't recurse into pg_wal */
1429 : : }
1430 : :
1431 : : /* Allow symbolic links in pg_tblspc only */
1552 tmunro@postgresql.or 1432 [ + + + + ]: 392437 : if (strcmp(path, "./pg_tblspc") == 0 && S_ISLNK(statbuf.st_mode))
5778 magnus@hagander.net 1433 : 39 : {
1434 : : char linkpath[MAXPGPATH];
1435 : : ssize_t rllen;
1436 : :
5447 tgl@sss.pgh.pa.us 1437 : 39 : rllen = readlink(pathbuf, linkpath, sizeof(linkpath));
1438 [ - + ]: 39 : if (rllen < 0)
5778 magnus@hagander.net 1439 [ # # ]:UBC 0 : ereport(ERROR,
1440 : : (errcode_for_file_access(),
1441 : : errmsg("could not read symbolic link \"%s\": %m",
1442 : : pathbuf)));
5447 tgl@sss.pgh.pa.us 1443 [ - + ]:CBC 39 : if (rllen >= sizeof(linkpath))
5447 tgl@sss.pgh.pa.us 1444 [ # # ]:UBC 0 : ereport(ERROR,
1445 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
1446 : : errmsg("symbolic link \"%s\" target is too long",
1447 : : pathbuf)));
5447 tgl@sss.pgh.pa.us 1448 :CBC 39 : linkpath[rllen] = '\0';
1449 : :
1826 rhaas@postgresql.org 1450 : 39 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, linkpath,
1451 : : &statbuf, sizeonly);
1452 : : }
5778 magnus@hagander.net 1453 [ + + ]: 392398 : else if (S_ISDIR(statbuf.st_mode))
1454 : : {
4685 1455 : 6359 : bool skip_this_dir = false;
1456 : : ListCell *lc;
1457 : :
1458 : : /*
1459 : : * Store a directory entry in the tar file so we can get the
1460 : : * permissions right.
1461 : : */
1826 rhaas@postgresql.org 1462 : 6359 : size += _tarWriteHeader(sink, pathbuf + basepathlen + 1, NULL, &statbuf,
1463 : : sizeonly);
1464 : :
1465 : : /*
1466 : : * Call ourselves recursively for a directory, unless it happens
1467 : : * to be a separate tablespace located within PGDATA.
1468 : : */
4685 magnus@hagander.net 1469 [ + + + + : 13884 : foreach(lc, tablespaces)
+ + ]
1470 : : {
1471 : 7554 : tablespaceinfo *ti = (tablespaceinfo *) lfirst(lc);
1472 : :
1473 : : /*
1474 : : * ti->rpath is the tablespace relative path within PGDATA, or
1475 : : * NULL if the tablespace has been properly located somewhere
1476 : : * else.
1477 : : *
1478 : : * Skip past the leading "./" in pathbuf when comparing.
1479 : : */
1480 [ + + + + ]: 7554 : if (ti->rpath && strcmp(ti->rpath, pathbuf + 2) == 0)
1481 : : {
1482 : 29 : skip_this_dir = true;
1483 : 29 : break;
1484 : : }
1485 : : }
1486 : :
1487 : : /*
1488 : : * skip sending directories inside pg_tblspc, if not required.
1489 : : */
4195 andrew@dunslane.net 1490 [ + + + + ]: 6359 : if (strcmp(pathbuf, "./pg_tblspc") == 0 && !sendtblspclinks)
1491 : 29 : skip_this_dir = true;
1492 : :
4685 magnus@hagander.net 1493 [ + + ]: 6359 : if (!skip_this_dir)
1826 rhaas@postgresql.org 1494 : 6301 : size += sendDir(sink, pathbuf, basepathlen, sizeonly, tablespaces,
1495 : : sendtblspclinks, manifest, spcoid, ib);
1496 : : }
5778 magnus@hagander.net 1497 [ + - ]: 386039 : else if (S_ISREG(statbuf.st_mode))
1498 : : {
4908 bruce@momjian.us 1499 : 386039 : bool sent = false;
1051 rhaas@postgresql.org 1500 : 386039 : unsigned num_blocks_required = 0;
1501 : 386039 : unsigned truncation_block_length = 0;
1502 : : char tarfilenamebuf[MAXPGPATH * 2];
1503 : 386039 : char *tarfilename = pathbuf + basepathlen + 1;
1504 : 386039 : FileBackupMethod method = BACK_UP_FILE_FULLY;
1505 : :
1506 [ + + + + ]: 386039 : if (ib != NULL && isRelationFile)
1507 : : {
1508 : : Oid relspcoid;
1509 : : char *lookup_path;
1510 : :
1511 [ + + ]: 27936 : if (OidIsValid(spcoid))
1512 : : {
1513 : 18 : relspcoid = spcoid;
793 michael@paquier.xyz 1514 : 18 : lookup_path = psprintf("%s/%u/%s", PG_TBLSPC_DIR, spcoid,
1515 : : tarfilename);
1516 : : }
1517 : : else
1518 : : {
1051 rhaas@postgresql.org 1519 [ + + ]: 27918 : if (isGlobalDir)
1520 : 1568 : relspcoid = GLOBALTABLESPACE_OID;
1521 : : else
1522 : 26350 : relspcoid = DEFAULTTABLESPACE_OID;
1523 : 27918 : lookup_path = pstrdup(tarfilename);
1524 : : }
1525 : :
1526 : 27936 : method = GetFileBackupMethod(ib, lookup_path, dboid, relspcoid,
1527 : : relfilenumber, relForkNum,
1528 : 27936 : segno, statbuf.st_size,
1529 : : &num_blocks_required,
1530 : : relative_block_numbers,
1531 : : &truncation_block_length);
1532 [ + + ]: 27936 : if (method == BACK_UP_FILE_INCREMENTALLY)
1533 : : {
1534 : 18650 : statbuf.st_size =
1535 : 18650 : GetIncrementalFileSize(num_blocks_required);
1536 : 18650 : snprintf(tarfilenamebuf, sizeof(tarfilenamebuf),
1537 : : "%s/INCREMENTAL.%s",
1538 : 18650 : path + basepathlen + 1,
1539 : 18650 : de->d_name);
1540 : 18650 : tarfilename = tarfilenamebuf;
1541 : : }
1542 : :
1543 : 27936 : pfree(lookup_path);
1544 : : }
1545 : :
5778 magnus@hagander.net 1546 [ + + ]: 386039 : if (!sizeonly)
1051 rhaas@postgresql.org 1547 [ + + ]: 190664 : sent = sendFile(sink, pathbuf, tarfilename, &statbuf,
1548 : : true, dboid, spcoid,
1549 : : relfilenumber, segno, manifest,
1550 : : num_blocks_required,
1551 : : method == BACK_UP_FILE_INCREMENTALLY ? relative_block_numbers : NULL,
1552 : : truncation_block_length);
1553 : :
5067 heikki.linnakangas@i 1554 [ + + + - ]: 386038 : if (sent || sizeonly)
1555 : : {
1556 : : /* Add size. */
2386 rhaas@postgresql.org 1557 : 386038 : size += statbuf.st_size;
1558 : :
1559 : : /* Pad to a multiple of the tar block size. */
1560 : 386038 : size += tarPaddingBytesRequired(statbuf.st_size);
1561 : :
1562 : : /* Size of the header for the file. */
1563 : 386038 : size += TAR_BLOCK_SIZE;
1564 : : }
1565 : : }
1566 : : else
5778 magnus@hagander.net 1567 [ # # ]:UBC 0 : ereport(WARNING,
1568 : : (errmsg("skipping special file \"%s\"", pathbuf)));
1569 : : }
1570 : :
1051 rhaas@postgresql.org 1571 [ + + ]:CBC 6760 : if (relative_block_numbers != NULL)
1572 : 484 : pfree(relative_block_numbers);
1573 : :
5778 magnus@hagander.net 1574 : 6760 : FreeDir(dir);
1575 : 6760 : return size;
1576 : : }
1577 : :
1578 : : /*
1579 : : * Given the member, write the TAR header & send the file.
1580 : : *
1581 : : * If 'missing_ok' is true, will not throw an error if the file is not found.
1582 : : *
1583 : : * If dboid is anything other than InvalidOid then any checksum failures
1584 : : * detected will get reported to the cumulative stats system.
1585 : : *
1586 : : * If the file is to be sent incrementally, then num_incremental_blocks
1587 : : * should be the number of blocks to be sent, and incremental_blocks
1588 : : * an array of block numbers relative to the start of the current segment.
1589 : : * If the whole file is to be sent, then incremental_blocks should be NULL,
1590 : : * and num_incremental_blocks can have any value, as it will be ignored.
1591 : : *
1592 : : * For non-relation files, the stat data of the opened file is used, not
1593 : : * 'statbuf'.
1594 : : *
1595 : : * Returns true if the file was successfully sent, false if 'missing_ok',
1596 : : * and the file did not exist.
1597 : : */
1598 : : static bool
1826 rhaas@postgresql.org 1599 : 190664 : sendFile(bbsink *sink, const char *readfilename, const char *tarfilename,
1600 : : const struct stat *statbuf, bool missing_ok, Oid dboid, Oid spcoid,
1601 : : RelFileNumber relfilenumber, unsigned segno,
1602 : : backup_manifest_info *manifest, unsigned num_incremental_blocks,
1603 : : BlockNumber *incremental_blocks, unsigned truncation_block_length)
1604 : : {
1605 : : int fd;
3138 magnus@hagander.net 1606 : 190664 : BlockNumber blkno = 0;
1607 : 190664 : int checksum_failures = 0;
1608 : : off_t cnt;
1129 rhaas@postgresql.org 1609 : 190664 : pgoff_t bytes_done = 0;
3138 magnus@hagander.net 1610 : 190664 : bool verify_checksum = false;
1611 : : pg_checksum_context checksum_ctx;
1051 rhaas@postgresql.org 1612 : 190664 : int ibindex = 0;
1613 : : struct stat filestat;
1614 : :
2164 michael@paquier.xyz 1615 [ - + ]: 190664 : if (pg_checksum_init(&checksum_ctx, manifest->checksum_type) < 0)
2164 michael@paquier.xyz 1616 [ # # ]:UBC 0 : elog(ERROR, "could not initialize checksum of file \"%s\"",
1617 : : readfilename);
1618 : :
2332 rhaas@postgresql.org 1619 :CBC 190664 : fd = OpenTransientFile(readfilename, O_RDONLY | PG_BINARY);
1620 [ - + ]: 190664 : if (fd < 0)
1621 : : {
5067 heikki.linnakangas@i 1622 [ # # # # ]:UBC 0 : if (errno == ENOENT && missing_ok)
1623 : 0 : return false;
5778 magnus@hagander.net 1624 [ # # ]: 0 : ereport(ERROR,
1625 : : (errcode_for_file_access(),
1626 : : errmsg("could not open file \"%s\": %m", readfilename)));
1627 : : }
1628 : :
1629 : : /*
1630 : : * Non-relation files may be replaced by a rename(), making irrelevant the
1631 : : * caller's stat data. Reuse fstat() on the opened file.
1632 : : */
9 michael@paquier.xyz 1633 :CBC 190664 : filestat = *statbuf;
1634 : :
1635 [ + + - + ]: 190664 : if (!RelFileNumberIsValid(relfilenumber) && fstat(fd, &filestat) != 0)
9 michael@paquier.xyz 1636 [ # # ]:UBC 0 : ereport(ERROR,
1637 : : (errcode_for_file_access(),
1638 : : errmsg("could not stat file \"%s\": %m", readfilename)));
1639 : :
9 michael@paquier.xyz 1640 :CBC 190664 : _tarWriteHeader(sink, tarfilename, NULL, &filestat, false);
1641 : :
1642 : : /*
1643 : : * Checksums are verified in multiples of BLCKSZ, so the buffer length
1644 : : * should be a multiple of the block size as well.
1645 : : */
1129 rhaas@postgresql.org 1646 [ - + ]: 190663 : Assert((sink->bbs_buffer_length % BLCKSZ) == 0);
1647 : :
1648 : : /*
1649 : : * Verify checksums unless the client requested otherwise, but only for
1650 : : * relation files, and only while checksums have been continuously enabled
1651 : : * since the checkpoint this backup started from. Checksums can still be
1652 : : * disabled while the backup runs, thus we check at each point where we
1653 : : * could be validating a checksum.
1654 : : */
79 dgustafsson@postgres 1655 [ + + + + :GNC 376975 : if (!noverify_checksums && RelFileNumberIsValid(relfilenumber) &&
+ + ]
1656 : 186312 : backup_checksums_verifiable(sink->bbs_state->startptr))
1087 rhaas@postgresql.org 1657 :CBC 180590 : verify_checksum = true;
1658 : :
1659 : : /*
1660 : : * If we're sending an incremental file, write the file header.
1661 : : */
1051 1662 [ + + ]: 190663 : if (incremental_blocks != NULL)
1663 : : {
1664 : 9325 : unsigned magic = INCREMENTAL_MAGIC;
1665 : 9325 : size_t header_bytes_done = 0;
1666 : : char padding[BLCKSZ];
1667 : : size_t paddinglen;
1668 : :
1669 : : /* Emit header data. */
1670 : 9325 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1671 : : &magic, sizeof(magic));
1672 : 9325 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1673 : : &num_incremental_blocks, sizeof(num_incremental_blocks));
1674 : 9325 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1675 : : &truncation_block_length, sizeof(truncation_block_length));
1676 : 9325 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1677 : : incremental_blocks,
1678 : : sizeof(BlockNumber) * num_incremental_blocks);
1679 : :
1680 : : /*
1681 : : * Add padding to align header to a multiple of BLCKSZ, but only if
1682 : : * the incremental file has some blocks, and the alignment is actually
1683 : : * needed (i.e. header is not already a multiple of BLCKSZ). If there
1684 : : * are no blocks we don't want to make the file unnecessarily large,
1685 : : * as that might make some filesystem optimizations impossible.
1686 : : */
935 tomas.vondra@postgre 1687 [ + + + - ]: 9325 : if ((num_incremental_blocks > 0) && (header_bytes_done % BLCKSZ != 0))
1688 : : {
944 1689 : 29 : paddinglen = (BLCKSZ - (header_bytes_done % BLCKSZ));
1690 : :
1691 : 29 : memset(padding, 0, paddinglen);
1692 : 29 : bytes_done += paddinglen;
1693 : :
1694 : 29 : push_to_sink(sink, &checksum_ctx, &header_bytes_done,
1695 : : padding, paddinglen);
1696 : : }
1697 : :
1698 : : /* Flush out any data still in the buffer so it's again empty. */
1051 rhaas@postgresql.org 1699 [ + - ]: 9325 : if (header_bytes_done > 0)
1700 : : {
1701 : 9325 : bbsink_archive_contents(sink, header_bytes_done);
1702 [ - + ]: 9325 : if (pg_checksum_update(&checksum_ctx,
1703 : 9325 : (uint8 *) sink->bbs_buffer,
1704 : : header_bytes_done) < 0)
1051 rhaas@postgresql.org 1705 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1706 : : }
1707 : :
1708 : : /* Update our notion of file position. */
1051 rhaas@postgresql.org 1709 :CBC 9325 : bytes_done += sizeof(magic);
1710 : 9325 : bytes_done += sizeof(num_incremental_blocks);
1711 : 9325 : bytes_done += sizeof(truncation_block_length);
1712 : 9325 : bytes_done += sizeof(BlockNumber) * num_incremental_blocks;
1713 : : }
1714 : :
1715 : : /*
1716 : : * Loop until we read the amount of data the caller told us to expect. The
1717 : : * file could be longer, if it was extended while we were sending it, but
1718 : : * for a base backup we can ignore such extended data. It will be restored
1719 : : * from WAL.
1720 : : */
1721 : : while (1)
1722 : : {
1723 : : /*
1724 : : * Determine whether we've read all the data that we need, and if not,
1725 : : * read some more.
1726 : : */
1727 [ + + ]: 409319 : if (incremental_blocks == NULL)
1728 : : {
9 michael@paquier.xyz 1729 : 399953 : size_t remaining = filestat.st_size - bytes_done;
1730 : :
1731 : : /*
1732 : : * If we've read the required number of bytes, then it's time to
1733 : : * stop.
1734 : : */
1735 [ + + ]: 399953 : if (bytes_done >= filestat.st_size)
1051 rhaas@postgresql.org 1736 : 181338 : break;
1737 : :
1738 : : /*
1739 : : * Read as many bytes as will fit in the buffer, or however many
1740 : : * are left to read, whichever is less.
1741 : : */
1742 : 218615 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1743 : : bytes_done, remaining,
1744 : 218615 : blkno + segno * RELSEG_SIZE,
1745 : : verify_checksum,
1746 : : &checksum_failures);
1747 : : }
1748 : : else
1749 : : {
1750 : : BlockNumber relative_blkno;
1751 : :
1752 : : /*
1753 : : * If we've read all the blocks, then it's time to stop.
1754 : : */
1755 [ + + ]: 9366 : if (ibindex >= num_incremental_blocks)
1756 : 9325 : break;
1757 : :
1758 : : /*
1759 : : * Read just one block, whichever one is the next that we're
1760 : : * supposed to include.
1761 : : */
1762 : 41 : relative_blkno = incremental_blocks[ibindex++];
1763 : 41 : cnt = read_file_data_into_buffer(sink, readfilename, fd,
1764 : 41 : relative_blkno * BLCKSZ,
1765 : : BLCKSZ,
1766 : 41 : relative_blkno + segno * RELSEG_SIZE,
1767 : : verify_checksum,
1768 : : &checksum_failures);
1769 : :
1770 : : /*
1771 : : * If we get a partial read, that must mean that the relation is
1772 : : * being truncated. Ultimately, it should be truncated to a
1773 : : * multiple of BLCKSZ, since this path should only be reached for
1774 : : * relation files, but we might transiently observe an
1775 : : * intermediate value.
1776 : : *
1777 : : * It should be fine to treat this just as if the entire block had
1778 : : * been truncated away - i.e. fill this and all later blocks with
1779 : : * zeroes. WAL replay will fix things up.
1780 : : */
1781 [ - + ]: 41 : if (cnt < BLCKSZ)
1051 rhaas@postgresql.org 1782 :UBC 0 : break;
1783 : : }
1784 : :
1785 : : /*
1786 : : * If the amount of data we were able to read was not a multiple of
1787 : : * BLCKSZ, we cannot verify checksums, which are block-level.
1788 : : */
79 dgustafsson@postgres 1789 [ + + + - ]:GNC 426401 : if (verify_checksum &&
1790 : 207745 : backup_checksums_verifiable(sink->bbs_state->startptr) &&
1791 [ - + ]: 207745 : (cnt % BLCKSZ != 0))
1792 : : {
3138 magnus@hagander.net 1793 [ # # ]:UBC 0 : ereport(WARNING,
1794 : : (errmsg("could not verify checksum in file \"%s\", block "
1795 : : "%u: read buffer size %d and page size %d "
1796 : : "differ",
1797 : : readfilename, blkno, (int) cnt, BLCKSZ)));
1798 : 0 : verify_checksum = false;
1799 : : }
1800 : :
1801 : : /*
1802 : : * If we hit end-of-file, a concurrent truncation must have occurred.
1803 : : * That's not an error condition, because WAL replay will fix things
1804 : : * up.
1805 : : */
1372 rhaas@postgresql.org 1806 [ - + ]:CBC 218656 : if (cnt == 0)
1372 rhaas@postgresql.org 1807 :UBC 0 : break;
1808 : :
1809 : : /* Update block number and # of bytes done for next loop iteration. */
1129 rhaas@postgresql.org 1810 :CBC 218656 : blkno += cnt / BLCKSZ;
1811 : 218656 : bytes_done += cnt;
1812 : :
1813 : : /*
1814 : : * Make sure incremental files with block data are properly aligned
1815 : : * (header is a multiple of BLCKSZ, blocks are BLCKSZ too).
1816 : : */
944 tomas.vondra@postgre 1817 [ + + + - : 218656 : Assert(!((incremental_blocks != NULL && num_incremental_blocks > 0) &&
- + ]
1818 : : (bytes_done % BLCKSZ != 0)));
1819 : :
1820 : : /* Archive the data we just read. */
1826 rhaas@postgresql.org 1821 : 218656 : bbsink_archive_contents(sink, cnt);
1822 : :
1823 : : /* Also feed it to the checksum machinery. */
1824 [ - + ]: 218656 : if (pg_checksum_update(&checksum_ctx,
1825 : 218656 : (uint8 *) sink->bbs_buffer, cnt) < 0)
2164 michael@paquier.xyz 1826 [ # # ]:UBC 0 : elog(ERROR, "could not update checksum of base backup");
1827 : : }
1828 : :
1829 : : /* If the file was truncated while we were sending it, pad it with zeros */
9 michael@paquier.xyz 1830 [ - + ]:CBC 190663 : while (bytes_done < filestat.st_size)
1831 : : {
9 michael@paquier.xyz 1832 :UBC 0 : size_t remaining = filestat.st_size - bytes_done;
1826 rhaas@postgresql.org 1833 : 0 : size_t nbytes = Min(sink->bbs_buffer_length, remaining);
1834 : :
1835 [ # # # # : 0 : MemSet(sink->bbs_buffer, 0, nbytes);
# # # # #
# ]
1836 [ # # ]: 0 : if (pg_checksum_update(&checksum_ctx,
1837 : 0 : (uint8 *) sink->bbs_buffer,
1838 : : nbytes) < 0)
1839 [ # # ]: 0 : elog(ERROR, "could not update checksum of base backup");
1840 : 0 : bbsink_archive_contents(sink, nbytes);
1129 1841 : 0 : bytes_done += nbytes;
1842 : : }
1843 : :
1844 : : /*
1845 : : * Pad to a block boundary, per tar format requirements. (This small piece
1846 : : * of data is probably not worth throttling, and is not checksummed
1847 : : * because it's not actually part of the file.)
1848 : : */
1129 rhaas@postgresql.org 1849 :CBC 190663 : _tarWritePadding(sink, bytes_done);
1850 : :
2332 1851 : 190663 : CloseTransientFile(fd);
1852 : :
78 michael@paquier.xyz 1853 [ + + ]: 190663 : if (checksum_failures > 0)
1854 : : {
3138 magnus@hagander.net 1855 [ + - ]: 4 : ereport(WARNING,
1856 : : (errmsg_plural("file \"%s\" has a total of %d checksum verification failure",
1857 : : "file \"%s\" has a total of %d checksum verification failures",
1858 : : checksum_failures,
1859 : : readfilename, checksum_failures)));
1860 : :
585 andres@anarazel.de 1861 : 4 : pgstat_prepare_report_checksum_failure(dboid);
2764 magnus@hagander.net 1862 : 4 : pgstat_report_checksum_failures_in_db(dboid, checksum_failures);
1863 : : }
1864 : :
3138 1865 : 190663 : total_checksum_failures += checksum_failures;
1866 : :
9 michael@paquier.xyz 1867 : 190663 : AddFileToBackupManifest(manifest, spcoid, tarfilename, filestat.st_size,
1868 : 190663 : (pg_time_t) filestat.st_mtime, &checksum_ctx);
1869 : :
5067 heikki.linnakangas@i 1870 : 190663 : return true;
1871 : : }
1872 : :
1873 : : /*
1874 : : * Read some more data from the file into the bbsink's buffer, verifying
1875 : : * checksums as required.
1876 : : *
1877 : : * 'offset' is the file offset from which we should begin to read, and
1878 : : * 'length' is the amount of data that should be read. The actual amount
1879 : : * of data read will be less than the requested amount if the bbsink's
1880 : : * buffer isn't big enough to hold it all, or if the underlying file has
1881 : : * been truncated. The return value is the number of bytes actually read.
1882 : : *
1883 : : * 'blkno' is the block number of the first page in the bbsink's buffer
1884 : : * relative to the start of the relation.
1885 : : *
1886 : : * 'verify_checksum' determines if the user has asked to verify checksums, but
1887 : : * since data checksums can be disabled, or become disabled, we need to check
1888 : : * state before verifying individual pages. If we do this, we'll update
1889 : : * *checksum_failures and issue warnings as appropriate.
1890 : : */
1891 : : static off_t
1129 rhaas@postgresql.org 1892 : 218656 : read_file_data_into_buffer(bbsink *sink, const char *readfilename, int fd,
1893 : : off_t offset, size_t length, BlockNumber blkno,
1894 : : bool verify_checksum, int *checksum_failures)
1895 : : {
1896 : : off_t cnt;
1897 : : int i;
1898 : : char *page;
1899 : :
1900 : : /* Try to read some more data. */
1901 : 218656 : cnt = basebackup_read_file(fd, sink->bbs_buffer,
1902 : 218656 : Min(sink->bbs_buffer_length, length),
1903 : : offset, readfilename, true);
1904 : :
1905 : : /* Can't verify checksums if read length is not a multiple of BLCKSZ. */
1906 [ + + - + ]: 218656 : if (!verify_checksum || (cnt % BLCKSZ) != 0)
1907 : 10911 : return cnt;
1908 : :
1909 : : /* Verify checksum for each block. */
1910 [ + + ]: 724825 : for (i = 0; i < cnt / BLCKSZ; i++)
1911 : : {
1912 : : int reread_cnt;
1913 : : uint16 expected_checksum;
1914 : :
1915 : : /*
1916 : : * The data checksum state can change at any point, so we need to
1917 : : * re-check before each page.
1918 : : */
79 dgustafsson@postgres 1919 [ - + ]:GNC 517080 : if (!backup_checksums_verifiable(sink->bbs_state->startptr))
216 dgustafsson@postgres 1920 :UNC 0 : return cnt;
1921 : :
1129 rhaas@postgresql.org 1922 :CBC 517080 : page = sink->bbs_buffer + BLCKSZ * i;
1923 : :
1924 : : /* If the page is OK, go on to the next one. */
1925 [ + + ]: 517080 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1926 : : &expected_checksum))
1927 : 517066 : continue;
1928 : :
1929 : : /*
1930 : : * Retry the block on the first failure. It's possible that we read
1931 : : * the first 4K page of the block just before postgres updated the
1932 : : * entire block so it ends up looking torn to us. If, before we retry
1933 : : * the read, the concurrent write of the block finishes, the page LSN
1934 : : * will be updated and we'll realize that we should ignore this block.
1935 : : *
1936 : : * There's no guarantee that this will actually happen, though: the
1937 : : * torn write could take an arbitrarily long time to complete.
1938 : : * Retrying multiple times wouldn't fix this problem, either, though
1939 : : * it would reduce the chances of it happening in practice. The only
1940 : : * real fix here seems to be to have some kind of interlock that
1941 : : * allows us to wait until we can be certain that no write to the
1942 : : * block is in progress. Since we don't have any such thing right now,
1943 : : * we just do this and hope for the best.
1944 : : *
1945 : : * The data checksum state may also have changed concurrently so check
1946 : : * again.
1947 : : */
79 dgustafsson@postgres 1948 [ - + ]:GNC 14 : if (!backup_checksums_verifiable(sink->bbs_state->startptr))
216 dgustafsson@postgres 1949 :UNC 0 : return cnt;
1129 rhaas@postgresql.org 1950 :CBC 14 : reread_cnt =
1951 : 14 : basebackup_read_file(fd, sink->bbs_buffer + BLCKSZ * i,
1952 : 14 : BLCKSZ, offset + BLCKSZ * i,
1953 : : readfilename, false);
1954 [ - + ]: 14 : if (reread_cnt == 0)
1955 : : {
1956 : : /*
1957 : : * If we hit end-of-file, a concurrent truncation must have
1958 : : * occurred, so reduce cnt to reflect only the blocks already
1959 : : * processed and break out of this loop.
1960 : : */
1129 rhaas@postgresql.org 1961 :UBC 0 : cnt = BLCKSZ * i;
1962 : 0 : break;
1963 : : }
1964 : :
1965 : : /* If the page now looks OK, go on to the next one. */
1129 rhaas@postgresql.org 1966 [ - + ]:CBC 14 : if (verify_page_checksum(page, sink->bbs_state->startptr, blkno + i,
1967 : : &expected_checksum))
1129 rhaas@postgresql.org 1968 :UBC 0 : continue;
1969 : :
1970 : : /* Handle checksum failure. */
1129 rhaas@postgresql.org 1971 :CBC 14 : (*checksum_failures)++;
1972 [ + + ]: 14 : if (*checksum_failures <= 5)
1973 [ + - ]: 12 : ereport(WARNING,
1974 : : (errmsg("checksum verification failed in "
1975 : : "file \"%s\", block %u: calculated "
1976 : : "%X but expected %X",
1977 : : readfilename, blkno + i, expected_checksum,
1978 : : ((PageHeader) page)->pd_checksum)));
1979 [ + + ]: 14 : if (*checksum_failures == 5)
1980 [ + - ]: 2 : ereport(WARNING,
1981 : : (errmsg("further checksum verification "
1982 : : "failures in file \"%s\" will not "
1983 : : "be reported", readfilename)));
1984 : : }
1985 : :
1986 : 207745 : return cnt;
1987 : : }
1988 : :
1989 : : /*
1990 : : * Push data into a bbsink.
1991 : : *
1992 : : * It's better, when possible, to read data directly into the bbsink's buffer,
1993 : : * rather than using this function to copy it into the buffer; this function is
1994 : : * for cases where that approach is not practical.
1995 : : *
1996 : : * bytes_done should point to a count of the number of bytes that are
1997 : : * currently used in the bbsink's buffer. Upon return, the bytes identified by
1998 : : * data and length will have been copied into the bbsink's buffer, flushing
1999 : : * as required, and *bytes_done will have been updated accordingly. If the
2000 : : * buffer was flushed, the previous contents will also have been fed to
2001 : : * checksum_ctx.
2002 : : *
2003 : : * Note that after one or more calls to this function it is the caller's
2004 : : * responsibility to perform any required final flush.
2005 : : */
2006 : : static void
1051 2007 : 37329 : push_to_sink(bbsink *sink, pg_checksum_context *checksum_ctx,
2008 : : size_t *bytes_done, void *data, size_t length)
2009 : : {
2010 [ + + ]: 37329 : while (length > 0)
2011 : : {
2012 : : size_t bytes_to_copy;
2013 : :
2014 : : /*
2015 : : * We use < here rather than <= so that if the data exactly fills the
2016 : : * remaining buffer space, we trigger a flush now.
2017 : : */
2018 [ + - ]: 28033 : if (length < sink->bbs_buffer_length - *bytes_done)
2019 : : {
2020 : : /* Append remaining data to buffer. */
2021 : 28033 : memcpy(sink->bbs_buffer + *bytes_done, data, length);
2022 : 28033 : *bytes_done += length;
2023 : 28033 : return;
2024 : : }
2025 : :
2026 : : /* Copy until buffer is full and flush it. */
1051 rhaas@postgresql.org 2027 :UBC 0 : bytes_to_copy = sink->bbs_buffer_length - *bytes_done;
2028 : 0 : memcpy(sink->bbs_buffer + *bytes_done, data, bytes_to_copy);
2029 : 0 : data = ((char *) data) + bytes_to_copy;
2030 : 0 : length -= bytes_to_copy;
2031 : 0 : bbsink_archive_contents(sink, sink->bbs_buffer_length);
2032 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, (uint8 *) sink->bbs_buffer,
2033 : : sink->bbs_buffer_length) < 0)
2034 [ # # ]: 0 : elog(ERROR, "could not update checksum");
2035 : 0 : *bytes_done = 0;
2036 : : }
2037 : : }
2038 : :
2039 : : /*
2040 : : * Check whether data checksums can be verified for a backup started at
2041 : : * start_lsn.
2042 : : *
2043 : : * Checksums are verified only while they have been continuously enabled
2044 : : * since the checkpoint the backup started from: the state must be "on" and
2045 : : * the last state change must predate the backup start. Such a checkpoint
2046 : : * guarantees that every page flushed before it has a checksum written. Any
2047 : : * later state change ends verification for the rest of the backup: pages
2048 : : * written while checksums were off can lack checksums yet keep LSNs older
2049 : : * than the backup start, and re-enabling completes before the rewritten
2050 : : * pages are flushed, so observing the "on" state again is not enough to
2051 : : * resume.
2052 : : */
2053 : : static bool
79 dgustafsson@postgres 2054 :GNC 1427217 : backup_checksums_verifiable(XLogRecPtr start_lsn)
2055 : : {
2056 [ + + + - ]: 2848712 : return DataChecksumsNeedVerify() &&
2057 : 1421495 : GetLastChecksumChangeRecPtr() <= start_lsn;
2058 : : }
2059 : :
2060 : : /*
2061 : : * Try to verify the checksum for the provided page, if it seems appropriate
2062 : : * to do so.
2063 : : *
2064 : : * Returns true if verification succeeds or if we decide not to check it,
2065 : : * and false if verification fails. When return false, it also sets
2066 : : * *expected_checksum to the computed value.
2067 : : */
2068 : : static bool
1129 rhaas@postgresql.org 2069 :CBC 517094 : verify_page_checksum(Page page, XLogRecPtr start_lsn, BlockNumber blkno,
2070 : : uint16 *expected_checksum)
2071 : : {
2072 : : PageHeader phdr;
2073 : : uint16 checksum;
2074 : :
2075 : : /*
2076 : : * Only check pages which have not been modified since the start of the
2077 : : * base backup. Otherwise, they might have been written only halfway and
2078 : : * the checksum would not be valid. However, replaying WAL would
2079 : : * reinstate the correct page in this case. We also skip completely new
2080 : : * pages, since they don't have a checksum yet.
2081 : : */
2082 [ + + + + ]: 517094 : if (PageIsNew(page) || PageGetLSN(page) >= start_lsn)
2083 : 1028 : return true;
2084 : :
79 dgustafsson@postgres 2085 [ - + ]:GNC 516066 : if (!backup_checksums_verifiable(start_lsn))
216 dgustafsson@postgres 2086 :UNC 0 : return true;
2087 : :
2088 : : /* Perform the actual checksum calculation. */
1129 rhaas@postgresql.org 2089 :CBC 516066 : checksum = pg_checksum_page(page, blkno);
2090 : :
2091 : : #ifdef USE_INJECTION_POINTS
2092 : : {
2093 : : /*
2094 : : * Make it possible to test checksum verification failure without
2095 : : * having to destroy data on disk. There is cap on how many times we
2096 : : * want to cause verification failure to make tests more interesting
2097 : : * and less log intensive. This makes it easy to test pg_basebackup
2098 : : * with the command_checks_all test function.
2099 : : */
2100 : : static int hit = 0;
2101 : :
79 dgustafsson@postgres 2102 [ - + ]:GNC 516066 : if (IS_INJECTION_POINT_ATTACHED("basebackup-fail-checksum-verification"))
2103 : : {
79 dgustafsson@postgres 2104 [ # # ]:UNC 0 : if (hit++ < 5)
2105 : : {
2106 : 0 : checksum = 0;
2107 : 0 : INJECTION_POINT_CACHED("basebackup-fail-checksum-verification",
2108 : : NULL);
2109 : : }
2110 : : }
2111 : : }
2112 : : #endif
2113 : :
2114 : : /* See whether it matches the value from the page. */
1129 rhaas@postgresql.org 2115 :CBC 516066 : phdr = (PageHeader) page;
2116 [ + + ]: 516066 : if (phdr->pd_checksum == checksum)
2117 : 516038 : return true;
2118 : 28 : *expected_checksum = checksum;
2119 : 28 : return false;
2120 : : }
2121 : :
2122 : : static int64
1826 2123 : 201472 : _tarWriteHeader(bbsink *sink, const char *filename, const char *linktarget,
2124 : : struct stat *statbuf, bool sizeonly)
2125 : : {
2126 : : enum tarError rc;
2127 : :
3690 peter_e@gmx.net 2128 [ + + ]: 201472 : if (!sizeonly)
2129 : : {
2130 : : /*
2131 : : * As of this writing, the smallest supported block size is 1kB, which
2132 : : * is twice TAR_BLOCK_SIZE. Since the buffer size is required to be a
2133 : : * multiple of BLCKSZ, it should be safe to assume that the buffer is
2134 : : * large enough to fit an entire tar block. We double-check by means
2135 : : * of these assertions.
2136 : : */
2137 : : StaticAssertDecl(TAR_BLOCK_SIZE <= BLCKSZ,
2138 : : "BLCKSZ too small for tar block");
1826 rhaas@postgresql.org 2139 [ - + ]: 196252 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2140 : :
2141 : 196252 : rc = tarCreateHeader(sink->bbs_buffer, filename, linktarget,
2142 : : statbuf->st_size, statbuf->st_mode,
2143 : : statbuf->st_uid, statbuf->st_gid,
2144 : : statbuf->st_mtime);
2145 : :
3690 peter_e@gmx.net 2146 [ + + - - ]: 196252 : switch (rc)
2147 : : {
2148 : 196251 : case TAR_OK:
2149 : 196251 : break;
2150 : 1 : case TAR_NAME_TOO_LONG:
2151 [ + - ]: 1 : ereport(ERROR,
2152 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2153 : : errmsg("file name too long for tar format: \"%s\"",
2154 : : filename)));
2155 : : break;
3690 peter_e@gmx.net 2156 :UBC 0 : case TAR_SYMLINK_TOO_LONG:
2157 [ # # ]: 0 : ereport(ERROR,
2158 : : (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
2159 : : errmsg("symbolic link target too long for tar format: "
2160 : : "file name \"%s\", target \"%s\"",
2161 : : filename, linktarget)));
2162 : : break;
2163 : 0 : default:
2164 [ # # ]: 0 : elog(ERROR, "unrecognized tar error: %d", rc);
2165 : : }
2166 : :
1826 rhaas@postgresql.org 2167 :CBC 196251 : bbsink_archive_contents(sink, TAR_BLOCK_SIZE);
2168 : : }
2169 : :
2170 : 201471 : return TAR_BLOCK_SIZE;
2171 : : }
2172 : :
2173 : : /*
2174 : : * Pad with zero bytes out to a multiple of TAR_BLOCK_SIZE.
2175 : : */
2176 : : static void
2177 : 191095 : _tarWritePadding(bbsink *sink, int len)
2178 : : {
2179 : 191095 : int pad = tarPaddingBytesRequired(len);
2180 : :
2181 : : /*
2182 : : * As in _tarWriteHeader, it should be safe to assume that the buffer is
2183 : : * large enough that we don't need to do this in multiple chunks.
2184 : : */
2185 [ - + ]: 191095 : Assert(sink->bbs_buffer_length >= TAR_BLOCK_SIZE);
2186 [ - + ]: 191095 : Assert(pad <= TAR_BLOCK_SIZE);
2187 : :
2188 [ + + ]: 191095 : if (pad > 0)
2189 : : {
2190 [ + - + + : 37277 : MemSet(sink->bbs_buffer, 0, pad);
+ - + - +
+ ]
2191 : 12297 : bbsink_archive_contents(sink, pad);
2192 : : }
4634 alvherre@alvh.no-ip. 2193 : 191095 : }
2194 : :
2195 : : /*
2196 : : * If the entry in statbuf is a link, then adjust statbuf to make it look like a
2197 : : * directory, so that it will be written that way.
2198 : : */
2199 : : static void
1826 rhaas@postgresql.org 2200 : 3111 : convert_link_to_directory(const char *pathbuf, struct stat *statbuf)
2201 : : {
2202 : : /* If symlink, write it as a directory anyway */
2203 [ + + ]: 3111 : if (S_ISLNK(statbuf->st_mode))
2204 : 66 : statbuf->st_mode = S_IFDIR | pg_dir_create_mode;
2438 fujii@postgresql.org 2205 : 3111 : }
2206 : :
2207 : : /*
2208 : : * Read some data from a file, setting a wait event and reporting any error
2209 : : * encountered.
2210 : : *
2211 : : * If partial_read_ok is false, also report an error if the number of bytes
2212 : : * read is not equal to the number of bytes requested.
2213 : : *
2214 : : * Returns the number of bytes read.
2215 : : */
2216 : : static ssize_t
2332 rhaas@postgresql.org 2217 : 226350 : basebackup_read_file(int fd, char *buf, size_t nbytes, off_t offset,
2218 : : const char *filename, bool partial_read_ok)
2219 : : {
2220 : : ssize_t rc;
2221 : :
2222 : 226350 : pgstat_report_wait_start(WAIT_EVENT_BASEBACKUP_READ);
1498 tmunro@postgresql.or 2223 : 226350 : rc = pg_pread(fd, buf, nbytes, offset);
2332 rhaas@postgresql.org 2224 : 226350 : pgstat_report_wait_end();
2225 : :
2226 [ - + ]: 226350 : if (rc < 0)
2332 rhaas@postgresql.org 2227 [ # # ]:UBC 0 : ereport(ERROR,
2228 : : (errcode_for_file_access(),
2229 : : errmsg("could not read file \"%s\": %m", filename)));
2332 rhaas@postgresql.org 2230 [ + + + - :CBC 226350 : if (!partial_read_ok && rc > 0 && rc != nbytes)
- + ]
2332 rhaas@postgresql.org 2231 [ # # ]:UBC 0 : ereport(ERROR,
2232 : : (errcode_for_file_access(),
2233 : : errmsg("could not read file \"%s\": read %zd of %zu",
2234 : : filename, rc, nbytes)));
2235 : :
2332 rhaas@postgresql.org 2236 :CBC 226350 : return rc;
2237 : : }
|
