Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * pg_combinebackup.c
4 : : * Combine incremental backups with prior backups.
5 : : *
6 : : * Copyright (c) 2017-2026, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/bin/pg_combinebackup/pg_combinebackup.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres_fe.h"
14 : :
15 : : #include <dirent.h>
16 : : #include <fcntl.h>
17 : : #include <limits.h>
18 : :
19 : : #ifdef HAVE_COPYFILE_H
20 : : #include <copyfile.h>
21 : : #endif
22 : : #ifdef __linux__
23 : : #include <sys/ioctl.h>
24 : : #include <linux/fs.h>
25 : : #endif
26 : :
27 : : #include "access/xlog_internal.h"
28 : : #include "backup_label.h"
29 : : #include "common/checksum_helper.h"
30 : : #include "common/controldata_utils.h"
31 : : #include "common/file_perm.h"
32 : : #include "common/file_utils.h"
33 : : #include "common/logging.h"
34 : : #include "common/relpath.h"
35 : : #include "copy_file.h"
36 : : #include "fe_utils/option_utils.h"
37 : : #include "fe_utils/version.h"
38 : : #include "getopt_long.h"
39 : : #include "lib/stringinfo.h"
40 : : #include "load_manifest.h"
41 : : #include "reconstruct.h"
42 : : #include "write_manifest.h"
43 : :
44 : : /* Incremental file naming convention. */
45 : : #define INCREMENTAL_PREFIX "INCREMENTAL."
46 : : #define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
47 : :
48 : : /*
49 : : * Tracking for directories that need to be removed, or have their contents
50 : : * removed, if the operation fails.
51 : : */
52 : : typedef struct cb_cleanup_dir
53 : : {
54 : : char *target_path;
55 : : bool rmtopdir;
56 : : struct cb_cleanup_dir *next;
57 : : } cb_cleanup_dir;
58 : :
59 : : /*
60 : : * Stores a tablespace mapping provided using -T, --tablespace-mapping.
61 : : */
62 : : typedef struct cb_tablespace_mapping
63 : : {
64 : : char old_dir[MAXPGPATH];
65 : : char new_dir[MAXPGPATH];
66 : : struct cb_tablespace_mapping *next;
67 : : } cb_tablespace_mapping;
68 : :
69 : : /*
70 : : * Stores data parsed from all command-line options.
71 : : */
72 : : typedef struct cb_options
73 : : {
74 : : bool debug;
75 : : char *output;
76 : : bool dry_run;
77 : : bool no_sync;
78 : : cb_tablespace_mapping *tsmappings;
79 : : pg_checksum_type manifest_checksums;
80 : : bool no_manifest;
81 : : DataDirSyncMethod sync_method;
82 : : CopyMethod copy_method;
83 : : } cb_options;
84 : :
85 : : /*
86 : : * Data about a tablespace.
87 : : *
88 : : * Every normal tablespace needs a tablespace mapping, but in-place tablespaces
89 : : * don't, so the list of tablespaces can contain more entries than the list of
90 : : * tablespace mappings.
91 : : */
92 : : typedef struct cb_tablespace
93 : : {
94 : : Oid oid;
95 : : bool in_place;
96 : : char old_dir[MAXPGPATH];
97 : : char new_dir[MAXPGPATH];
98 : : struct cb_tablespace *next;
99 : : } cb_tablespace;
100 : :
101 : : /* Directories to be removed if we exit uncleanly. */
102 : : static cb_cleanup_dir *cleanup_dir_list = NULL;
103 : :
104 : : static void add_tablespace_mapping(cb_options *opt, char *arg);
105 : : static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
106 : : static uint64 check_control_files(int n_backups, char **backup_dirs);
107 : : static void check_input_dir_permissions(char *dir);
108 : : static void cleanup_directories_atexit(void);
109 : : static void create_output_directory(char *dirname, cb_options *opt);
110 : : static void help(const char *progname);
111 : : static bool parse_oid(char *s, Oid *result);
112 : : static void process_directory_recursively(Oid tsoid,
113 : : char *input_directory,
114 : : char *output_directory,
115 : : char *relative_path,
116 : : int n_prior_backups,
117 : : char **prior_backup_dirs,
118 : : manifest_data **manifests,
119 : : manifest_writer *mwriter,
120 : : cb_options *opt);
121 : : static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
122 : : static void reset_directory_cleanup_list(void);
123 : : static cb_tablespace *scan_for_existing_tablespaces(char *pathname,
124 : : cb_options *opt);
125 : : static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen);
126 : :
127 : : /*
128 : : * Main program.
129 : : */
130 : : int
131 : 27 : main(int argc, char *argv[])
132 : : {
133 : : static struct option long_options[] = {
134 : : {"debug", no_argument, NULL, 'd'},
135 : : {"dry-run", no_argument, NULL, 'n'},
136 : : {"no-sync", no_argument, NULL, 'N'},
137 : : {"output", required_argument, NULL, 'o'},
138 : : {"tablespace-mapping", required_argument, NULL, 'T'},
139 : : {"link", no_argument, NULL, 'k'},
140 : : {"manifest-checksums", required_argument, NULL, 1},
141 : : {"no-manifest", no_argument, NULL, 2},
142 : : {"sync-method", required_argument, NULL, 3},
143 : : {"clone", no_argument, NULL, 4},
144 : : {"copy", no_argument, NULL, 5},
145 : : {"copy-file-range", no_argument, NULL, 6},
146 : : {NULL, 0, NULL, 0}
147 : : };
148 : :
149 : : const char *progname;
150 : : char *last_input_dir;
151 : : int i;
152 : : int optindex;
153 : : int c;
154 : : int n_backups;
155 : : int n_prior_backups;
156 : : uint32 version;
157 : : uint64 system_identifier;
158 : : char **prior_backup_dirs;
159 : : cb_options opt;
160 : : cb_tablespace *tablespaces;
161 : : cb_tablespace *ts;
162 : : StringInfo last_backup_label;
163 : : manifest_data **manifests;
164 : : manifest_writer *mwriter;
165 : : char *pgdata;
166 : :
167 : 27 : pg_logging_init(argv[0]);
168 : 27 : progname = get_progname(argv[0]);
169 : 27 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
170 : 27 : handle_help_version_opts(argc, argv, progname, help);
171 : :
172 : 25 : memset(&opt, 0, sizeof(opt));
173 : 25 : opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
174 : 25 : opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
175 : 25 : opt.copy_method = COPY_METHOD_COPY;
176 : :
177 : : /* process command-line options */
178 : 79 : while ((c = getopt_long(argc, argv, "dknNo:T:",
179 [ + + ]: 79 : long_options, &optindex)) != -1)
180 : : {
181 [ + + - + : 55 : switch (c)
+ + + + -
- + - + ]
182 : : {
183 : 8 : case 'd':
184 : 8 : opt.debug = true;
185 : 8 : pg_logging_increase_verbosity();
186 : 8 : break;
187 : 1 : case 'k':
188 : 1 : opt.copy_method = COPY_METHOD_LINK;
189 : 1 : break;
190 : 0 : case 'n':
191 : 0 : opt.dry_run = true;
192 : 0 : break;
193 : 3 : case 'N':
194 : 3 : opt.no_sync = true;
195 : 3 : break;
196 : 22 : case 'o':
197 : 22 : opt.output = optarg;
198 : 22 : break;
199 : 1 : case 'T':
200 : 1 : add_tablespace_mapping(&opt, optarg);
201 : 1 : break;
202 : 2 : case 1:
203 [ - + ]: 2 : if (!pg_checksum_parse_type(optarg,
204 : : &opt.manifest_checksums))
205 : 0 : pg_fatal("unrecognized checksum algorithm: \"%s\"",
206 : : optarg);
207 : 2 : break;
208 : 1 : case 2:
209 : 1 : opt.no_manifest = true;
210 : 1 : break;
211 : 0 : case 3:
212 [ # # ]: 0 : if (!parse_sync_method(optarg, &opt.sync_method))
213 : 0 : exit(1);
214 : 0 : break;
215 : 0 : case 4:
216 : 0 : opt.copy_method = COPY_METHOD_CLONE;
217 : 0 : break;
218 : 16 : case 5:
219 : 16 : opt.copy_method = COPY_METHOD_COPY;
220 : 16 : break;
221 : 0 : case 6:
222 : 0 : opt.copy_method = COPY_METHOD_COPY_FILE_RANGE;
223 : 0 : break;
224 : 1 : default:
225 : : /* getopt_long already emitted a complaint */
226 : 1 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
227 : 1 : exit(1);
228 : : }
229 : : }
230 : :
231 [ + + ]: 24 : if (optind >= argc)
232 : : {
233 : 1 : pg_log_error("no input directories specified");
234 : 1 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
235 : 1 : exit(1);
236 : : }
237 : :
238 [ + + ]: 23 : if (opt.output == NULL)
239 : 1 : pg_fatal("no output directory specified");
240 : :
241 : : /* If no manifest is needed, no checksums are needed, either. */
242 [ + + ]: 22 : if (opt.no_manifest)
243 : 1 : opt.manifest_checksums = CHECKSUM_TYPE_NONE;
244 : :
245 [ - + ]: 22 : if (opt.dry_run)
246 : : {
247 : 0 : pg_log_info("executing in dry-run mode");
248 : 0 : pg_log_info_detail("The target directory will not be modified.");
249 : : }
250 : :
251 : : /* Check that the platform supports the requested copy method. */
252 [ - + ]: 22 : if (opt.copy_method == COPY_METHOD_CLONE)
253 : : {
254 : : #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
255 : : (defined(__linux__) && defined(FICLONE))
256 : :
257 [ # # ]: 0 : if (opt.dry_run)
258 [ # # ]: 0 : pg_log_debug("would use cloning to copy files");
259 : : else
260 [ # # ]: 0 : pg_log_debug("will use cloning to copy files");
261 : :
262 : : #else
263 : : pg_fatal("file cloning not supported on this platform");
264 : : #endif
265 : : }
266 [ - + ]: 22 : else if (opt.copy_method == COPY_METHOD_COPY_FILE_RANGE)
267 : : {
268 : : #if defined(HAVE_COPY_FILE_RANGE)
269 : :
270 [ # # ]: 0 : if (opt.dry_run)
271 [ # # ]: 0 : pg_log_debug("would use copy_file_range to copy blocks");
272 : : else
273 [ # # ]: 0 : pg_log_debug("will use copy_file_range to copy blocks");
274 : :
275 : : #else
276 : : pg_fatal("copy_file_range not supported on this platform");
277 : : #endif
278 : : }
279 : :
280 : : /* Read the server version from the final backup. */
281 : 22 : pgdata = argv[argc - 1];
282 : 22 : version = get_pg_version(pgdata, NULL);
283 [ - + ]: 22 : if (GET_PG_MAJORVERSION_NUM(version) < 10)
284 : 0 : pg_fatal("server version too old");
285 [ + + ]: 22 : pg_log_debug("read server version %u from file \"%s/%s\"",
286 : : GET_PG_MAJORVERSION_NUM(version), pgdata, "PG_VERSION");
287 : :
288 : : /* Sanity-check control files. */
289 : 22 : n_backups = argc - optind;
290 : 22 : system_identifier = check_control_files(n_backups, argv + optind);
291 : :
292 : : /* Sanity-check backup_label files, and get the contents of the last one. */
293 : 21 : last_backup_label = check_backup_label_files(n_backups, argv + optind);
294 : :
295 : : /*
296 : : * We'll need the pathnames to the prior backups. By "prior" we mean all
297 : : * but the last one listed on the command line.
298 : : */
299 : 16 : n_prior_backups = argc - optind - 1;
300 : 16 : prior_backup_dirs = argv + optind;
301 : :
302 : : /* Load backup manifests. */
303 : 16 : manifests = load_backup_manifests(n_backups, prior_backup_dirs);
304 : :
305 : : /*
306 : : * Validate the manifest system identifier against the backup system
307 : : * identifier.
308 : : */
309 [ + + ]: 46 : for (i = 0; i < n_backups; i++)
310 : : {
311 [ + - ]: 31 : if (manifests[i] &&
312 [ + + ]: 31 : manifests[i]->system_identifier != system_identifier)
313 : : {
314 : : char *controlpath;
315 : :
316 : 1 : controlpath = psprintf("%s/%s", prior_backup_dirs[i], XLOG_CONTROL_FILE);
317 : :
318 : 1 : pg_fatal("%s: manifest system identifier is %" PRIu64 ", but control file has %" PRIu64,
319 : : controlpath,
320 : : manifests[i]->system_identifier,
321 : : system_identifier);
322 : : }
323 : : }
324 : :
325 : : /* Figure out which tablespaces are going to be included in the output. */
326 : 15 : last_input_dir = argv[argc - 1];
327 : 15 : check_input_dir_permissions(last_input_dir);
328 : 15 : tablespaces = scan_for_existing_tablespaces(last_input_dir, &opt);
329 : :
330 : : /*
331 : : * Create output directories.
332 : : *
333 : : * We create one output directory for the main data directory plus one for
334 : : * each non-in-place tablespace. create_output_directory() will arrange
335 : : * for those directories to be cleaned up on failure. In-place tablespaces
336 : : * aren't handled at this stage because they're located beneath the main
337 : : * output directory, and thus the cleanup of that directory will get rid
338 : : * of them. Plus, the pg_tblspc directory that needs to contain them
339 : : * doesn't exist yet.
340 : : */
341 : 15 : atexit(cleanup_directories_atexit);
342 : 15 : create_output_directory(opt.output, &opt);
343 [ + + ]: 16 : for (ts = tablespaces; ts != NULL; ts = ts->next)
344 [ + - ]: 1 : if (!ts->in_place)
345 : 1 : create_output_directory(ts->new_dir, &opt);
346 : :
347 : : /* If we need to write a backup_manifest, prepare to do so. */
348 [ + - + + ]: 15 : if (!opt.dry_run && !opt.no_manifest)
349 : : {
350 : 14 : mwriter = create_manifest_writer(opt.output, system_identifier);
351 : :
352 : : /*
353 : : * Verify that we have a backup manifest for the final backup; else we
354 : : * won't have the WAL ranges for the resulting manifest.
355 : : */
356 [ - + ]: 14 : if (manifests[n_prior_backups] == NULL)
357 : 0 : pg_fatal("cannot generate a manifest because no manifest is available for the final input backup");
358 : : }
359 : : else
360 : 1 : mwriter = NULL;
361 : :
362 : : /* Write backup label into output directory. */
363 [ - + ]: 15 : if (opt.dry_run)
364 [ # # ]: 0 : pg_log_debug("would generate \"%s/backup_label\"", opt.output);
365 : : else
366 : : {
367 [ + + ]: 15 : pg_log_debug("generating \"%s/backup_label\"", opt.output);
368 : 15 : last_backup_label->cursor = 0;
369 : 15 : write_backup_label(opt.output, last_backup_label,
370 : : opt.manifest_checksums, mwriter);
371 : : }
372 : :
373 : : /* Process everything that's not part of a user-defined tablespace. */
374 [ + + ]: 15 : pg_log_debug("processing backup directory \"%s\"", last_input_dir);
375 : 15 : process_directory_recursively(InvalidOid, last_input_dir, opt.output,
376 : : NULL, n_prior_backups, prior_backup_dirs,
377 : : manifests, mwriter, &opt);
378 : :
379 : : /* Process user-defined tablespaces. */
380 [ + + ]: 15 : for (ts = tablespaces; ts != NULL; ts = ts->next)
381 : : {
382 [ + - ]: 1 : pg_log_debug("processing tablespace directory \"%s\"", ts->old_dir);
383 : :
384 : : /*
385 : : * If it's a normal tablespace, we need to set up a symbolic link from
386 : : * pg_tblspc/${OID} to the target directory; if it's an in-place
387 : : * tablespace, we need to create a directory at pg_tblspc/${OID}.
388 : : */
389 [ + - ]: 1 : if (!ts->in_place)
390 : : {
391 : : char linkpath[MAXPGPATH];
392 : :
393 : 1 : snprintf(linkpath, MAXPGPATH, "%s/%s/%u", opt.output, PG_TBLSPC_DIR,
394 : : ts->oid);
395 : :
396 [ - + ]: 1 : if (opt.dry_run)
397 [ # # ]: 0 : pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
398 : : linkpath, ts->new_dir);
399 : : else
400 : : {
401 [ + - ]: 1 : pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
402 : : linkpath, ts->new_dir);
403 [ - + ]: 1 : if (symlink(ts->new_dir, linkpath) != 0)
404 : 0 : pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
405 : : linkpath, ts->new_dir);
406 : : }
407 : : }
408 : : else
409 : : {
410 [ # # ]: 0 : if (opt.dry_run)
411 [ # # ]: 0 : pg_log_debug("would create directory \"%s\"", ts->new_dir);
412 : : else
413 : : {
414 [ # # ]: 0 : pg_log_debug("creating directory \"%s\"", ts->new_dir);
415 [ # # ]: 0 : if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
416 : 0 : pg_fatal("could not create directory \"%s\": %m",
417 : : ts->new_dir);
418 : : }
419 : : }
420 : :
421 : : /* OK, now handle the directory contents. */
422 : 1 : process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
423 : : NULL, n_prior_backups, prior_backup_dirs,
424 : : manifests, mwriter, &opt);
425 : : }
426 : :
427 : : /* Finalize the backup_manifest, if we're generating one. */
428 [ + + ]: 14 : if (mwriter != NULL)
429 : 13 : finalize_manifest(mwriter,
430 : 13 : manifests[n_prior_backups]->first_wal_range);
431 : :
432 : : /* fsync that output directory unless we've been told not to do so */
433 [ + + ]: 14 : if (!opt.no_sync)
434 : : {
435 [ - + ]: 11 : if (opt.dry_run)
436 [ # # ]: 0 : pg_log_debug("would recursively fsync \"%s\"", opt.output);
437 : : else
438 : : {
439 [ + + ]: 11 : pg_log_debug("recursively fsyncing \"%s\"", opt.output);
440 : 11 : sync_pgdata(opt.output, version, opt.sync_method, true);
441 : : }
442 : : }
443 : :
444 : : /* Warn about the possibility of compromising the backups, when link mode */
445 [ + + ]: 14 : if (opt.copy_method == COPY_METHOD_LINK)
446 : 1 : pg_log_warning("--link mode was used; any modifications to the output "
447 : : "directory might destructively modify input directories");
448 : :
449 : : /* It's a success, so don't remove the output directories. */
450 : 14 : reset_directory_cleanup_list();
451 : 14 : exit(0);
452 : : }
453 : :
454 : : /*
455 : : * Process the option argument for the -T, --tablespace-mapping switch.
456 : : */
457 : : static void
458 : 1 : add_tablespace_mapping(cb_options *opt, char *arg)
459 : : {
460 : 1 : cb_tablespace_mapping *tsmap = pg_malloc0_object(cb_tablespace_mapping);
461 : : char *dst;
462 : : char *dst_ptr;
463 : : char *arg_ptr;
464 : :
465 : : /*
466 : : * Basically, we just want to copy everything before the equals sign to
467 : : * tsmap->old_dir and everything afterwards to tsmap->new_dir, but if
468 : : * there's more or less than one equals sign, that's an error, and if
469 : : * there's an equals sign preceded by a backslash, don't treat it as a
470 : : * field separator but instead copy a literal equals sign.
471 : : */
472 : 1 : dst_ptr = dst = tsmap->old_dir;
473 [ + + ]: 50 : for (arg_ptr = arg; *arg_ptr != '\0'; arg_ptr++)
474 : : {
475 [ - + ]: 49 : if (dst_ptr - dst >= MAXPGPATH)
476 : 0 : pg_fatal("directory name too long");
477 : :
478 [ - + - - ]: 49 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
479 : : ; /* skip backslash escaping = */
480 [ + + + - : 49 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
+ - ]
481 : : {
482 [ - + ]: 1 : if (tsmap->new_dir[0] != '\0')
483 : 0 : pg_fatal("multiple \"=\" signs in tablespace mapping");
484 : : else
485 : 1 : dst = dst_ptr = tsmap->new_dir;
486 : : }
487 : : else
488 : 48 : *dst_ptr++ = *arg_ptr;
489 : : }
490 [ + - - + ]: 1 : if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
491 : 0 : pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
492 : :
493 : : /*
494 : : * All tablespaces are created with absolute directories, so specifying a
495 : : * non-absolute path here would never match, possibly confusing users.
496 : : *
497 : : * In contrast to pg_basebackup, both the old and new directories are on
498 : : * the local machine, so the local machine's definition of an absolute
499 : : * path is the only relevant one.
500 : : */
501 [ - + ]: 1 : if (!is_absolute_path(tsmap->old_dir))
502 : 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
503 : : tsmap->old_dir);
504 : :
505 [ - + ]: 1 : if (!is_absolute_path(tsmap->new_dir))
506 : 0 : pg_fatal("new directory is not an absolute path in tablespace mapping: %s",
507 : : tsmap->new_dir);
508 : :
509 : : /* Canonicalize paths to avoid spurious failures when comparing. */
510 : 1 : canonicalize_path(tsmap->old_dir);
511 : 1 : canonicalize_path(tsmap->new_dir);
512 : :
513 : : /* Add it to the list. */
514 : 1 : tsmap->next = opt->tsmappings;
515 : 1 : opt->tsmappings = tsmap;
516 : 1 : }
517 : :
518 : : /*
519 : : * Check that the backup_label files form a coherent backup chain, and return
520 : : * the contents of the backup_label file from the latest backup.
521 : : */
522 : : static StringInfo
523 : 21 : check_backup_label_files(int n_backups, char **backup_dirs)
524 : : {
525 : 21 : StringInfo buf = makeStringInfo();
526 : 21 : StringInfo lastbuf = buf;
527 : : int i;
528 : 21 : TimeLineID check_tli = 0;
529 : 21 : XLogRecPtr check_lsn = InvalidXLogRecPtr;
530 : :
531 : : /* Try to read each backup_label file in turn, last to first. */
532 [ + + ]: 57 : for (i = n_backups - 1; i >= 0; --i)
533 : : {
534 : : char pathbuf[MAXPGPATH];
535 : : int fd;
536 : : TimeLineID start_tli;
537 : : TimeLineID previous_tli;
538 : : XLogRecPtr start_lsn;
539 : : XLogRecPtr previous_lsn;
540 : :
541 : : /* Open the backup_label file. */
542 : 41 : snprintf(pathbuf, MAXPGPATH, "%s/backup_label", backup_dirs[i]);
543 [ + + ]: 41 : pg_log_debug("reading \"%s\"", pathbuf);
544 [ - + ]: 41 : if ((fd = open(pathbuf, O_RDONLY, 0)) < 0)
545 : 0 : pg_fatal("could not open file \"%s\": %m", pathbuf);
546 : :
547 : : /*
548 : : * Slurp the whole file into memory.
549 : : *
550 : : * The exact size limit that we impose here doesn't really matter --
551 : : * most of what's supposed to be in the file is fixed size and quite
552 : : * short. However, the length of the backup_label is limited (at least
553 : : * by some parts of the code) to MAXPGPATH, so include that value in
554 : : * the maximum length that we tolerate.
555 : : */
556 : 41 : slurp_file(fd, pathbuf, buf, 10000 + MAXPGPATH);
557 : :
558 : : /* Close the file. */
559 [ - + ]: 41 : if (close(fd) != 0)
560 : 0 : pg_fatal("could not close file \"%s\": %m", pathbuf);
561 : :
562 : : /* Parse the file contents. */
563 : 41 : parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
564 : : &previous_tli, &previous_lsn);
565 : :
566 : : /*
567 : : * Sanity checks.
568 : : *
569 : : * XXX. It's actually not required that start_lsn == check_lsn. It
570 : : * would be OK if start_lsn > check_lsn provided that start_lsn is
571 : : * less than or equal to the relevant switchpoint. But at the moment
572 : : * we don't have that information.
573 : : */
574 [ + + + + ]: 41 : if (i > 0 && previous_tli == 0)
575 : 1 : pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
576 : : backup_dirs[i]);
577 [ + + + + ]: 40 : if (i == 0 && previous_tli != 0)
578 : 1 : pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
579 : : backup_dirs[i]);
580 [ + + - + ]: 39 : if (i < n_backups - 1 && start_tli != check_tli)
581 : 0 : pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
582 : : backup_dirs[i], start_tli, check_tli);
583 [ + + + + ]: 39 : if (i < n_backups - 1 && start_lsn != check_lsn)
584 : 3 : pg_fatal("backup at \"%s\" starts at LSN %X/%08X, but expected %X/%08X",
585 : : backup_dirs[i],
586 : : LSN_FORMAT_ARGS(start_lsn),
587 : : LSN_FORMAT_ARGS(check_lsn));
588 : 36 : check_tli = previous_tli;
589 : 36 : check_lsn = previous_lsn;
590 : :
591 : : /*
592 : : * The last backup label in the chain needs to be saved for later use,
593 : : * while the others are only needed within this loop.
594 : : */
595 [ + + ]: 36 : if (lastbuf == buf)
596 : 20 : buf = makeStringInfo();
597 : : else
598 : 16 : resetStringInfo(buf);
599 : : }
600 : :
601 : : /* Free memory that we don't need any more. */
602 [ + - ]: 16 : if (lastbuf != buf)
603 : 16 : destroyStringInfo(buf);
604 : :
605 : : /*
606 : : * Return the data from the first backup_info that we read (which is the
607 : : * backup_label from the last directory specified on the command line).
608 : : */
609 : 16 : return lastbuf;
610 : : }
611 : :
612 : : /*
613 : : * Sanity check control files and return system_identifier.
614 : : */
615 : : static uint64
616 : 22 : check_control_files(int n_backups, char **backup_dirs)
617 : : {
618 : : int i;
619 : 22 : uint64 system_identifier = 0; /* placate compiler */
620 : 22 : uint32 data_checksum_version = PG_DATA_CHECKSUM_OFF; /* placate compiler */
621 : 22 : bool data_checksum_mismatch = false;
622 : :
623 : : /* Try to read each control file in turn, last to first. */
624 [ + + ]: 66 : for (i = n_backups - 1; i >= 0; --i)
625 : : {
626 : : ControlFileData *control_file;
627 : : bool crc_ok;
628 : : char *controlpath;
629 : :
630 : 45 : controlpath = psprintf("%s/%s", backup_dirs[i], XLOG_CONTROL_FILE);
631 [ + + ]: 45 : pg_log_debug("reading \"%s\"", controlpath);
632 : 45 : control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
633 : :
634 : : /* Control file contents not meaningful if CRC is bad. */
635 [ - + ]: 45 : if (!crc_ok)
636 : 0 : pg_fatal("%s: CRC is incorrect", controlpath);
637 : :
638 : : /* Can't interpret control file if not current version. */
639 [ - + ]: 45 : if (control_file->pg_control_version != PG_CONTROL_VERSION)
640 : 0 : pg_fatal("%s: unexpected control file version",
641 : : controlpath);
642 : :
643 : : /* System identifiers should all match. */
644 [ + + ]: 45 : if (i == n_backups - 1)
645 : 22 : system_identifier = control_file->system_identifier;
646 [ + + ]: 23 : else if (system_identifier != control_file->system_identifier)
647 : 1 : pg_fatal("%s: expected system identifier %" PRIu64 ", but found %" PRIu64,
648 : : controlpath, system_identifier,
649 : : control_file->system_identifier);
650 : :
651 : : /*
652 : : * Detect checksum mismatches, but only if the last backup in the
653 : : * chain has checksums enabled.
654 : : */
655 [ + + ]: 44 : if (i == n_backups - 1)
656 : 22 : data_checksum_version = control_file->data_checksum_version;
657 [ + - ]: 22 : else if (data_checksum_version != PG_DATA_CHECKSUM_OFF &&
658 [ - + ]: 22 : data_checksum_version != control_file->data_checksum_version)
659 : 0 : data_checksum_mismatch = true;
660 : :
661 : : /* Release memory. */
662 : 44 : pfree(control_file);
663 : 44 : pfree(controlpath);
664 : : }
665 : :
666 : : /*
667 : : * If debug output is enabled, make a note of the system identifier that
668 : : * we found in all of the relevant control files.
669 : : */
670 [ + + ]: 21 : pg_log_debug("system identifier is %" PRIu64, system_identifier);
671 : :
672 : : /*
673 : : * Warn the user if not all backups are in the same state with regards to
674 : : * checksums.
675 : : */
676 [ - + ]: 21 : if (data_checksum_mismatch)
677 : : {
678 : 0 : pg_log_warning("only some backups have checksums enabled");
679 : 0 : pg_log_warning_hint("Disable, and optionally reenable, checksums on the output directory to avoid failures.");
680 : : }
681 : :
682 : 21 : return system_identifier;
683 : : }
684 : :
685 : : /*
686 : : * Set default permissions for new files and directories based on the
687 : : * permissions of the given directory. The intent here is that the output
688 : : * directory should use the same permissions scheme as the final input
689 : : * directory.
690 : : */
691 : : static void
692 : 15 : check_input_dir_permissions(char *dir)
693 : : {
694 : : struct stat st;
695 : :
696 [ - + ]: 15 : if (stat(dir, &st) != 0)
697 : 0 : pg_fatal("could not stat file \"%s\": %m", dir);
698 : :
699 : 15 : SetDataDirectoryCreatePerm(st.st_mode);
700 : 15 : }
701 : :
702 : : /*
703 : : * Clean up output directories before exiting.
704 : : */
705 : : static void
706 : 15 : cleanup_directories_atexit(void)
707 : : {
708 [ + + ]: 16 : while (cleanup_dir_list != NULL)
709 : : {
710 : 1 : cb_cleanup_dir *dir = cleanup_dir_list;
711 : :
712 [ + - ]: 1 : if (dir->rmtopdir)
713 : : {
714 : 1 : pg_log_info("removing output directory \"%s\"", dir->target_path);
715 [ - + ]: 1 : if (!rmtree(dir->target_path, dir->rmtopdir))
716 : 0 : pg_log_error("failed to remove output directory");
717 : : }
718 : : else
719 : : {
720 : 0 : pg_log_info("removing contents of output directory \"%s\"",
721 : : dir->target_path);
722 [ # # ]: 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
723 : 0 : pg_log_error("failed to remove contents of output directory");
724 : : }
725 : :
726 : 1 : cleanup_dir_list = cleanup_dir_list->next;
727 : 1 : pfree(dir);
728 : : }
729 : 15 : }
730 : :
731 : : /*
732 : : * Create the named output directory, unless it already exists or we're in
733 : : * dry-run mode. If it already exists but is not empty, that's a fatal error.
734 : : *
735 : : * Adds the created directory to the list of directories to be cleaned up
736 : : * at process exit.
737 : : */
738 : : static void
739 : 16 : create_output_directory(char *dirname, cb_options *opt)
740 : : {
741 [ + - - - : 16 : switch (pg_check_dir(dirname))
- ]
742 : : {
743 : 16 : case 0:
744 [ - + ]: 16 : if (opt->dry_run)
745 : : {
746 [ # # ]: 0 : pg_log_debug("would create directory \"%s\"", dirname);
747 : 0 : return;
748 : : }
749 [ + + ]: 16 : pg_log_debug("creating directory \"%s\"", dirname);
750 [ - + ]: 16 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
751 : 0 : pg_fatal("could not create directory \"%s\": %m", dirname);
752 : 16 : remember_to_cleanup_directory(dirname, true);
753 : 16 : break;
754 : :
755 : 0 : case 1:
756 [ # # ]: 0 : pg_log_debug("using existing directory \"%s\"", dirname);
757 : 0 : remember_to_cleanup_directory(dirname, false);
758 : 0 : break;
759 : :
760 : 0 : case 2:
761 : : case 3:
762 : : case 4:
763 : 0 : pg_fatal("directory \"%s\" exists but is not empty", dirname);
764 : :
765 : 0 : case -1:
766 : 0 : pg_fatal("could not access directory \"%s\": %m", dirname);
767 : : }
768 : : }
769 : :
770 : : /*
771 : : * help
772 : : *
773 : : * Prints help page for the program
774 : : *
775 : : * progname: the name of the executed program, such as "pg_combinebackup"
776 : : */
777 : : static void
778 : 1 : help(const char *progname)
779 : : {
780 : 1 : printf(_("%s reconstructs full backups from incrementals.\n\n"), progname);
781 : 1 : printf(_("Usage:\n"));
782 : 1 : printf(_(" %s [OPTION]... DIRECTORY...\n"), progname);
783 : 1 : printf(_("\nOptions:\n"));
784 : 1 : printf(_(" -d, --debug generate lots of debugging output\n"));
785 : 1 : printf(_(" -k, --link link files instead of copying\n"));
786 : 1 : printf(_(" -n, --dry-run do not actually do anything\n"));
787 : 1 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
788 : 1 : printf(_(" -o, --output=DIRECTORY output directory\n"));
789 : 1 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
790 : : " relocate tablespace in OLDDIR to NEWDIR\n"));
791 : 1 : printf(_(" --clone clone (reflink) files instead of copying\n"));
792 : 1 : printf(_(" --copy copy files (default)\n"));
793 : 1 : printf(_(" --copy-file-range copy using copy_file_range() system call\n"));
794 : 1 : printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
795 : : " use algorithm for manifest checksums\n"));
796 : 1 : printf(_(" --no-manifest suppress generation of backup manifest\n"));
797 : 1 : printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
798 : 1 : printf(_(" -V, --version output version information, then exit\n"));
799 : 1 : printf(_(" -?, --help show this help, then exit\n"));
800 : :
801 : 1 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
802 : 1 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
803 : 1 : }
804 : :
805 : : /*
806 : : * Try to parse a string as a non-zero OID without leading zeroes.
807 : : *
808 : : * If it works, return true and set *result to the answer, else return false.
809 : : */
810 : : static bool
811 : 2 : parse_oid(char *s, Oid *result)
812 : : {
813 : : Oid oid;
814 : : char *ep;
815 : :
816 : 2 : errno = 0;
817 : 2 : oid = strtoul(s, &ep, 10);
818 [ + - + - : 2 : if (errno != 0 || *ep != '\0' || oid < 1 || oid > PG_UINT32_MAX)
- + ]
819 : 0 : return false;
820 : :
821 : 2 : *result = oid;
822 : 2 : return true;
823 : : }
824 : :
825 : : /*
826 : : * Copy files from the input directory to the output directory, reconstructing
827 : : * full files from incremental files as required.
828 : : *
829 : : * If processing a user-defined tablespace, the tsoid should be the OID
830 : : * of that tablespace and input_directory and output_directory should be the
831 : : * toplevel input and output directories for that tablespace. Otherwise,
832 : : * tsoid should be InvalidOid and input_directory and output_directory should
833 : : * be the main input and output directories.
834 : : *
835 : : * relative_path is the path beneath the given input and output directories
836 : : * that we are currently processing. If NULL, it indicates that we're
837 : : * processing the input and output directories themselves.
838 : : *
839 : : * n_prior_backups is the number of prior backups that we have available.
840 : : * This doesn't count the very last backup, which is referenced by
841 : : * input_directory, just the older ones. prior_backup_dirs is an array of
842 : : * the locations of those previous backups.
843 : : */
844 : : static void
845 : 408 : process_directory_recursively(Oid tsoid,
846 : : char *input_directory,
847 : : char *output_directory,
848 : : char *relative_path,
849 : : int n_prior_backups,
850 : : char **prior_backup_dirs,
851 : : manifest_data **manifests,
852 : : manifest_writer *mwriter,
853 : : cb_options *opt)
854 : : {
855 : : char ifulldir[MAXPGPATH];
856 : : char ofulldir[MAXPGPATH];
857 : : char manifest_prefix[MAXPGPATH];
858 : : DIR *dir;
859 : : struct dirent *de;
860 : 408 : bool is_pg_tblspc = false;
861 : 408 : bool is_pg_wal = false;
862 : 408 : bool is_incremental_dir = false;
863 : 408 : manifest_data *latest_manifest = manifests[n_prior_backups];
864 : : pg_checksum_type checksum_type;
865 : :
866 : : /*
867 : : * Classify this directory.
868 : : *
869 : : * We set is_pg_tblspc only for the toplevel pg_tblspc directory, because
870 : : * the symlinks in that specific directory require special handling.
871 : : *
872 : : * We set is_pg_wal for the toplevel WAL directory and all of its
873 : : * subdirectories, because those files are not included in the backup
874 : : * manifest and hence need special treatment. (Since incremental backup
875 : : * does not exist in pre-v10 versions, we don't have to worry about the
876 : : * old pg_xlog naming.)
877 : : *
878 : : * We set is_incremental_dir for directories that can contain incremental
879 : : * files requiring reconstruction. If such files occur outside these
880 : : * directories, we want to just copy them straight to the output
881 : : * directory. This is to protect against a user creating a file with a
882 : : * strange name like INCREMENTAL.config and then complaining that
883 : : * incremental backups don't work properly. The test here is a bit tricky:
884 : : * incremental files occur in subdirectories of base, in pg_global itself,
885 : : * and in subdirectories of pg_tblspc only if in-place tablespaces are
886 : : * used.
887 : : */
888 [ + + ]: 408 : if (OidIsValid(tsoid))
889 : 3 : is_incremental_dir = true;
890 [ + + ]: 405 : else if (relative_path != NULL)
891 : : {
892 : 390 : is_pg_tblspc = strcmp(relative_path, PG_TBLSPC_DIR) == 0;
893 [ + + ]: 765 : is_pg_wal = (strcmp(relative_path, "pg_wal") == 0 ||
894 [ + + ]: 375 : strncmp(relative_path, "pg_wal/", 7) == 0);
895 : 390 : is_incremental_dir = strncmp(relative_path, "base/", 5) == 0 ||
896 [ + + + + ]: 718 : strcmp(relative_path, "global") == 0 ||
897 [ - + ]: 328 : strncmp(relative_path, PG_TBLSPC_DIR_SLASH, 10) == 0;
898 : : }
899 : :
900 : : /*
901 : : * If we're under pg_wal, then we don't need checksums, because these
902 : : * files aren't included in the backup manifest. Otherwise use whatever
903 : : * type of checksum is configured.
904 : : */
905 [ + + ]: 408 : if (!is_pg_wal)
906 : 363 : checksum_type = opt->manifest_checksums;
907 : : else
908 : 45 : checksum_type = CHECKSUM_TYPE_NONE;
909 : :
910 : : /*
911 : : * Append the relative path to the input and output directories, and
912 : : * figure out the appropriate prefix to add to files in this directory
913 : : * when looking them up in a backup manifest.
914 : : */
915 [ + + ]: 408 : if (relative_path == NULL)
916 : : {
917 : 16 : strlcpy(ifulldir, input_directory, MAXPGPATH);
918 : 16 : strlcpy(ofulldir, output_directory, MAXPGPATH);
919 [ + + ]: 16 : if (OidIsValid(tsoid))
920 : 1 : snprintf(manifest_prefix, MAXPGPATH, "%s/%u/", PG_TBLSPC_DIR, tsoid);
921 : : else
922 : 15 : manifest_prefix[0] = '\0';
923 : : }
924 : : else
925 : : {
926 : 392 : snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
927 : : relative_path);
928 : 392 : snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
929 : : relative_path);
930 [ + + ]: 392 : if (OidIsValid(tsoid))
931 : 2 : snprintf(manifest_prefix, MAXPGPATH, "%s/%u/%s/",
932 : : PG_TBLSPC_DIR, tsoid, relative_path);
933 : : else
934 : 390 : snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
935 : : }
936 : :
937 : : /*
938 : : * Toplevel output directories have already been created by the time this
939 : : * function is called, but any subdirectories are our responsibility.
940 : : */
941 [ + + ]: 408 : if (relative_path != NULL)
942 : : {
943 [ - + ]: 392 : if (opt->dry_run)
944 [ # # ]: 0 : pg_log_debug("would create directory \"%s\"", ofulldir);
945 : : else
946 : : {
947 [ + + ]: 392 : pg_log_debug("creating directory \"%s\"", ofulldir);
948 [ - + ]: 392 : if (mkdir(ofulldir, pg_dir_create_mode) == -1)
949 : 0 : pg_fatal("could not create directory \"%s\": %m", ofulldir);
950 : : }
951 : : }
952 : :
953 : : /* It's time to scan the directory. */
954 [ - + ]: 408 : if ((dir = opendir(ifulldir)) == NULL)
955 : 0 : pg_fatal("could not open directory \"%s\": %m", ifulldir);
956 [ + + ]: 17471 : while (errno = 0, (de = readdir(dir)) != NULL)
957 : : {
958 : : PGFileType type;
959 : : char ifullpath[MAXPGPATH];
960 : : char ofullpath[MAXPGPATH];
961 : : char manifest_path[MAXPGPATH];
962 : 17066 : Oid oid = InvalidOid;
963 : 17066 : int checksum_length = 0;
964 : 17066 : uint8 *checksum_payload = NULL;
965 : : pg_checksum_context checksum_ctx;
966 : :
967 : : /* Ignore "." and ".." entries. */
968 [ + + ]: 17066 : if (strcmp(de->d_name, ".") == 0 ||
969 [ + + ]: 16658 : strcmp(de->d_name, "..") == 0)
970 : 1237 : continue;
971 : :
972 : : /* Construct input path. */
973 : 16250 : snprintf(ifullpath, MAXPGPATH, "%s/%s", ifulldir, de->d_name);
974 : :
975 : : /* Figure out what kind of directory entry this is. */
976 : 16250 : type = get_dirent_type(ifullpath, de, false, PG_LOG_ERROR);
977 [ - + ]: 16250 : if (type == PGFILETYPE_ERROR)
978 : 0 : exit(1);
979 : :
980 : : /*
981 : : * If we're processing pg_tblspc, then check whether the filename
982 : : * looks like it could be a tablespace OID. If so, and if the
983 : : * directory entry is a symbolic link or a directory, skip it.
984 : : *
985 : : * Our goal here is to ignore anything that would have been considered
986 : : * by scan_for_existing_tablespaces to be a tablespace.
987 : : */
988 [ + + + - : 16250 : if (is_pg_tblspc && parse_oid(de->d_name, &oid) &&
- + ]
989 [ # # ]: 0 : (type == PGFILETYPE_LNK || type == PGFILETYPE_DIR))
990 : 1 : continue;
991 : :
992 : : /* If it's a directory, recurse. */
993 [ + + ]: 16249 : if (type == PGFILETYPE_DIR)
994 : 390 : {
995 : : char new_relative_path[MAXPGPATH];
996 : :
997 : : /* Append new pathname component to relative path. */
998 [ + + ]: 392 : if (relative_path == NULL)
999 : 254 : strlcpy(new_relative_path, de->d_name, MAXPGPATH);
1000 : : else
1001 : 138 : snprintf(new_relative_path, MAXPGPATH, "%s/%s", relative_path,
1002 : 138 : de->d_name);
1003 : :
1004 : : /* And recurse. */
1005 : 392 : process_directory_recursively(tsoid,
1006 : : input_directory, output_directory,
1007 : : new_relative_path,
1008 : : n_prior_backups, prior_backup_dirs,
1009 : : manifests, mwriter, opt);
1010 : 390 : continue;
1011 : : }
1012 : :
1013 : : /* Skip anything that's not a regular file. */
1014 [ - + ]: 15857 : if (type != PGFILETYPE_REG)
1015 : : {
1016 [ # # ]: 0 : if (type == PGFILETYPE_LNK)
1017 : 0 : pg_log_warning("skipping symbolic link \"%s\"", ifullpath);
1018 : : else
1019 : 0 : pg_log_warning("skipping special file \"%s\"", ifullpath);
1020 : 0 : continue;
1021 : : }
1022 : :
1023 : : /*
1024 : : * Skip the backup_label and backup_manifest files; they require
1025 : : * special handling and are handled elsewhere.
1026 : : */
1027 [ + + ]: 15857 : if (relative_path == NULL &&
1028 [ + + ]: 124 : (strcmp(de->d_name, "backup_label") == 0 ||
1029 [ + + ]: 109 : strcmp(de->d_name, "backup_manifest") == 0))
1030 : 30 : continue;
1031 : :
1032 : : /*
1033 : : * If it's an incremental file, hand it off to the reconstruction
1034 : : * code, which will figure out what to do.
1035 : : */
1036 [ + + ]: 15827 : if (is_incremental_dir &&
1037 [ + + ]: 15657 : strncmp(de->d_name, INCREMENTAL_PREFIX,
1038 : : INCREMENTAL_PREFIX_LENGTH) == 0)
1039 : : {
1040 : : /* Output path should not include "INCREMENTAL." prefix. */
1041 : 8241 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
1042 : 8241 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1043 : :
1044 : :
1045 : : /* Manifest path likewise omits incremental prefix. */
1046 : 8241 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1047 : 8241 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1048 : :
1049 : : /* Reconstruction logic will do the rest. */
1050 : 8241 : reconstruct_from_incremental_file(ifullpath, ofullpath,
1051 : : manifest_prefix,
1052 : 8241 : de->d_name + INCREMENTAL_PREFIX_LENGTH,
1053 : : n_prior_backups,
1054 : : prior_backup_dirs,
1055 : : manifests,
1056 : : manifest_path,
1057 : : checksum_type,
1058 : : &checksum_length,
1059 : : &checksum_payload,
1060 : : opt->copy_method,
1061 : 8241 : opt->debug,
1062 : 8241 : opt->dry_run);
1063 : : }
1064 : : else
1065 : : {
1066 : : /* Construct the path that the backup_manifest will use. */
1067 : 7586 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1068 : 7586 : de->d_name);
1069 : :
1070 : : /*
1071 : : * It's not an incremental file, so we need to copy the entire
1072 : : * file to the output directory.
1073 : : *
1074 : : * If a checksum of the required type already exists in the
1075 : : * backup_manifest for the final input directory, we can save some
1076 : : * work by reusing that checksum instead of computing a new one.
1077 : : */
1078 [ + + + - ]: 7586 : if (checksum_type != CHECKSUM_TYPE_NONE &&
1079 : : latest_manifest != NULL)
1080 : : {
1081 : : manifest_file *mfile;
1082 : :
1083 : 5514 : mfile = manifest_files_lookup(latest_manifest->files,
1084 : : manifest_path);
1085 [ - + ]: 5514 : if (mfile == NULL)
1086 : : {
1087 : : char *bmpath;
1088 : :
1089 : : /*
1090 : : * The directory is out of sync with the backup_manifest,
1091 : : * so emit a warning.
1092 : : */
1093 : 0 : bmpath = psprintf("%s/%s", input_directory,
1094 : : "backup_manifest");
1095 : 0 : pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
1096 : : bmpath, manifest_path);
1097 : 0 : pfree(bmpath);
1098 : : }
1099 [ + + ]: 5514 : else if (mfile->checksum_type == checksum_type)
1100 : : {
1101 : 4486 : checksum_length = mfile->checksum_length;
1102 : 4486 : checksum_payload = mfile->checksum_payload;
1103 : : }
1104 : : }
1105 : :
1106 : : /*
1107 : : * If we're reusing a checksum, then we don't need copy_file() to
1108 : : * compute one for us, but otherwise, it needs to compute whatever
1109 : : * type of checksum we need.
1110 : : */
1111 [ + + ]: 7586 : if (checksum_length != 0)
1112 : 4486 : pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_NONE);
1113 : : else
1114 : 3100 : pg_checksum_init(&checksum_ctx, checksum_type);
1115 : :
1116 : : /* Actually copy the file. */
1117 : 7586 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
1118 : 7586 : copy_file(ifullpath, ofullpath, &checksum_ctx,
1119 : 7586 : opt->copy_method, opt->dry_run);
1120 : :
1121 : : /*
1122 : : * If copy_file() performed a checksum calculation for us, then
1123 : : * save the results (except in dry-run mode, when there's no
1124 : : * point).
1125 : : */
1126 [ + + + - ]: 7586 : if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
1127 : : {
1128 : 1028 : checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
1129 : 1028 : checksum_length = pg_checksum_final(&checksum_ctx,
1130 : : checksum_payload);
1131 : : }
1132 : : }
1133 : :
1134 : : /* Generate manifest entry, if needed. */
1135 [ + + ]: 15826 : if (mwriter != NULL)
1136 : : {
1137 : : struct stat sb;
1138 : :
1139 : : /*
1140 : : * In order to generate a manifest entry, we need the file size
1141 : : * and mtime. We have no way to know the correct mtime except to
1142 : : * stat() the file, so just do that and get the size as well.
1143 : : *
1144 : : * If we didn't need the mtime here, we could try to obtain the
1145 : : * file size from the reconstruction or file copy process above,
1146 : : * although that is actually not convenient in all cases. If we
1147 : : * write the file ourselves then clearly we can keep a count of
1148 : : * bytes, but if we use something like CopyFile() then it's
1149 : : * trickier. Since we have to stat() anyway to get the mtime,
1150 : : * there's no point in worrying about it.
1151 : : */
1152 [ - + ]: 14797 : if (stat(ofullpath, &sb) < 0)
1153 : 0 : pg_fatal("could not stat file \"%s\": %m", ofullpath);
1154 : :
1155 : : /* OK, now do the work. */
1156 : 14797 : add_file_to_manifest(mwriter, manifest_path,
1157 : 14797 : sb.st_size, sb.st_mtime,
1158 : : checksum_type, checksum_length,
1159 : : checksum_payload);
1160 : : }
1161 : :
1162 : : /* Avoid leaking memory. */
1163 [ + + ]: 15826 : if (checksum_payload != NULL)
1164 : 13754 : pg_free(checksum_payload);
1165 : : }
1166 : :
1167 : 405 : closedir(dir);
1168 : 405 : }
1169 : :
1170 : : /*
1171 : : * Add a directory to the list of output directories to clean up.
1172 : : */
1173 : : static void
1174 : 16 : remember_to_cleanup_directory(char *target_path, bool rmtopdir)
1175 : : {
1176 : 16 : cb_cleanup_dir *dir = pg_malloc_object(cb_cleanup_dir);
1177 : :
1178 : 16 : dir->target_path = target_path;
1179 : 16 : dir->rmtopdir = rmtopdir;
1180 : 16 : dir->next = cleanup_dir_list;
1181 : 16 : cleanup_dir_list = dir;
1182 : 16 : }
1183 : :
1184 : : /*
1185 : : * Empty out the list of directories scheduled for cleanup at exit.
1186 : : *
1187 : : * We want to remove the output directories only on a failure, so call this
1188 : : * function when we know that the operation has succeeded.
1189 : : *
1190 : : * Since we only expect this to be called when we're about to exit, we could
1191 : : * just set cleanup_dir_list to NULL and be done with it, but we free the
1192 : : * memory to be tidy.
1193 : : */
1194 : : static void
1195 : 14 : reset_directory_cleanup_list(void)
1196 : : {
1197 [ + + ]: 29 : while (cleanup_dir_list != NULL)
1198 : : {
1199 : 15 : cb_cleanup_dir *dir = cleanup_dir_list;
1200 : :
1201 : 15 : cleanup_dir_list = cleanup_dir_list->next;
1202 : 15 : pfree(dir);
1203 : : }
1204 : 14 : }
1205 : :
1206 : : /*
1207 : : * Scan the pg_tblspc directory of the final input backup to get a canonical
1208 : : * list of what tablespaces are part of the backup.
1209 : : *
1210 : : * 'pathname' should be the path to the toplevel backup directory for the
1211 : : * final backup in the backup chain.
1212 : : */
1213 : : static cb_tablespace *
1214 : 15 : scan_for_existing_tablespaces(char *pathname, cb_options *opt)
1215 : : {
1216 : : char pg_tblspc[MAXPGPATH];
1217 : : DIR *dir;
1218 : : struct dirent *de;
1219 : 15 : cb_tablespace *tslist = NULL;
1220 : :
1221 : 15 : snprintf(pg_tblspc, MAXPGPATH, "%s/%s", pathname, PG_TBLSPC_DIR);
1222 [ + + ]: 15 : pg_log_debug("scanning \"%s\"", pg_tblspc);
1223 : :
1224 [ - + ]: 15 : if ((dir = opendir(pg_tblspc)) == NULL)
1225 : 0 : pg_fatal("could not open directory \"%s\": %m", pg_tblspc);
1226 : :
1227 [ + + ]: 46 : while (errno = 0, (de = readdir(dir)) != NULL)
1228 : : {
1229 : : Oid oid;
1230 : : char tblspcdir[MAXPGPATH];
1231 : : char link_target[MAXPGPATH];
1232 : : cb_tablespace *ts;
1233 : : cb_tablespace *otherts;
1234 : : PGFileType type;
1235 : :
1236 : : /* Silently ignore "." and ".." entries. */
1237 [ + + + + ]: 31 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1238 : 30 : continue;
1239 : :
1240 : : /* Construct full pathname. */
1241 : 1 : snprintf(tblspcdir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1242 : :
1243 : : /* Ignore any file name that doesn't look like a proper OID. */
1244 [ - + ]: 1 : if (!parse_oid(de->d_name, &oid))
1245 : : {
1246 [ # # ]: 0 : pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
1247 : : tblspcdir);
1248 : 0 : continue;
1249 : : }
1250 : :
1251 : : /* Only symbolic links and directories are tablespaces. */
1252 : 1 : type = get_dirent_type(tblspcdir, de, false, PG_LOG_ERROR);
1253 [ - + ]: 1 : if (type == PGFILETYPE_ERROR)
1254 : 0 : exit(1);
1255 [ - + - - ]: 1 : if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
1256 : : {
1257 [ # # ]: 0 : pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
1258 : : tblspcdir);
1259 : 0 : continue;
1260 : : }
1261 : :
1262 : : /* Create a new tablespace object. */
1263 : 1 : ts = pg_malloc0_object(cb_tablespace);
1264 : 1 : ts->oid = oid;
1265 : :
1266 : : /*
1267 : : * If it's a link, it's not an in-place tablespace. Otherwise, it must
1268 : : * be a directory, and thus an in-place tablespace.
1269 : : */
1270 [ + - ]: 1 : if (type == PGFILETYPE_LNK)
1271 : : {
1272 : : ssize_t link_length;
1273 : : cb_tablespace_mapping *tsmap;
1274 : :
1275 : : /* Read the link target. */
1276 : 1 : link_length = readlink(tblspcdir, link_target, sizeof(link_target));
1277 [ - + ]: 1 : if (link_length < 0)
1278 : 0 : pg_fatal("could not read symbolic link \"%s\": %m",
1279 : : tblspcdir);
1280 [ - + ]: 1 : if (link_length >= sizeof(link_target))
1281 : 0 : pg_fatal("target of symbolic link \"%s\" is too long", tblspcdir);
1282 : 1 : link_target[link_length] = '\0';
1283 [ - + ]: 1 : if (!is_absolute_path(link_target))
1284 : 0 : pg_fatal("target of symbolic link \"%s\" is relative", tblspcdir);
1285 : :
1286 : : /* Canonicalize the link target. */
1287 : 1 : canonicalize_path(link_target);
1288 : :
1289 : : /*
1290 : : * Find the corresponding tablespace mapping and copy the relevant
1291 : : * details into the new tablespace entry.
1292 : : */
1293 [ + - ]: 1 : for (tsmap = opt->tsmappings; tsmap != NULL; tsmap = tsmap->next)
1294 : : {
1295 [ + - ]: 1 : if (strcmp(tsmap->old_dir, link_target) == 0)
1296 : : {
1297 : 1 : strlcpy(ts->old_dir, tsmap->old_dir, MAXPGPATH);
1298 : 1 : strlcpy(ts->new_dir, tsmap->new_dir, MAXPGPATH);
1299 : 1 : ts->in_place = false;
1300 : 1 : break;
1301 : : }
1302 : : }
1303 : :
1304 : : /* Every non-in-place tablespace must be mapped. */
1305 [ - + ]: 1 : if (tsmap == NULL)
1306 : 0 : pg_fatal("tablespace at \"%s\" has no tablespace mapping",
1307 : : link_target);
1308 : : }
1309 : : else
1310 : : {
1311 : : /*
1312 : : * For an in-place tablespace, there's no separate directory, so
1313 : : * we just record the paths within the data directories.
1314 : : */
1315 : 0 : snprintf(ts->old_dir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1316 : 0 : snprintf(ts->new_dir, MAXPGPATH, "%s/%s/%s", opt->output,
1317 : 0 : PG_TBLSPC_DIR, de->d_name);
1318 : 0 : ts->in_place = true;
1319 : : }
1320 : :
1321 : : /* Tablespaces should not share a directory. */
1322 [ - + ]: 1 : for (otherts = tslist; otherts != NULL; otherts = otherts->next)
1323 [ # # ]: 0 : if (strcmp(ts->new_dir, otherts->new_dir) == 0)
1324 : 0 : pg_fatal("tablespaces with OIDs %u and %u both point at directory \"%s\"",
1325 : : otherts->oid, oid, ts->new_dir);
1326 : :
1327 : : /* Add this tablespace to the list. */
1328 : 1 : ts->next = tslist;
1329 : 1 : tslist = ts;
1330 : : }
1331 : :
1332 [ - + ]: 15 : if (closedir(dir) != 0)
1333 : 0 : pg_fatal("could not close directory \"%s\": %m", pg_tblspc);
1334 : :
1335 : 15 : return tslist;
1336 : : }
1337 : :
1338 : : /*
1339 : : * Read a file into a StringInfo.
1340 : : *
1341 : : * fd is used for the actual file I/O, filename for error reporting purposes.
1342 : : * A file longer than maxlen is a fatal error.
1343 : : */
1344 : : static void
1345 : 41 : slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
1346 : : {
1347 : : struct stat st;
1348 : : ssize_t rb;
1349 : : size_t len;
1350 : :
1351 : : /* Check file size, and complain if it's too large. */
1352 [ - + ]: 41 : if (fstat(fd, &st) != 0)
1353 : 0 : pg_fatal("could not stat file \"%s\": %m", filename);
1354 [ - + ]: 41 : if (st.st_size > maxlen)
1355 : 0 : pg_fatal("file \"%s\" is too large", filename);
1356 : :
1357 : 41 : len = st.st_size;
1358 : :
1359 : : /* Make sure we have enough space. */
1360 : 41 : enlargeStringInfo(buf, len);
1361 : :
1362 : : /* Read the data. */
1363 : 41 : rb = read(fd, &buf->data[buf->len], len);
1364 : :
1365 : : /*
1366 : : * We don't expect any concurrent changes, so we should read exactly the
1367 : : * expected number of bytes.
1368 : : */
1369 [ - + ]: 41 : if (rb != len)
1370 : : {
1371 [ # # ]: 0 : if (rb < 0)
1372 : 0 : pg_fatal("could not read file \"%s\": %m", filename);
1373 : : else
1374 : 0 : pg_fatal("could not read file \"%s\": read %zd of %zu",
1375 : : filename, rb, len);
1376 : : }
1377 : :
1378 : : /* Adjust buffer length for new data and restore trailing-\0 invariant */
1379 : 41 : buf->len += rb;
1380 : 41 : buf->data[buf->len] = '\0';
1381 : 41 : }
|
