Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_combinebackup.c
4 : * Combine incremental backups with prior backups.
5 : *
6 : * Copyright (c) 2017-2025, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/bin/pg_combinebackup/pg_combinebackup.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres_fe.h"
14 :
15 : #include <dirent.h>
16 : #include <fcntl.h>
17 : #include <limits.h>
18 :
19 : #ifdef HAVE_COPYFILE_H
20 : #include <copyfile.h>
21 : #endif
22 : #ifdef __linux__
23 : #include <sys/ioctl.h>
24 : #include <linux/fs.h>
25 : #endif
26 :
27 : #include "backup_label.h"
28 : #include "common/checksum_helper.h"
29 : #include "common/controldata_utils.h"
30 : #include "common/file_perm.h"
31 : #include "common/file_utils.h"
32 : #include "common/logging.h"
33 : #include "common/relpath.h"
34 : #include "copy_file.h"
35 : #include "fe_utils/option_utils.h"
36 : #include "getopt_long.h"
37 : #include "lib/stringinfo.h"
38 : #include "load_manifest.h"
39 : #include "reconstruct.h"
40 : #include "write_manifest.h"
41 :
42 : /* Incremental file naming convention. */
43 : #define INCREMENTAL_PREFIX "INCREMENTAL."
44 : #define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
45 :
46 : /*
47 : * Tracking for directories that need to be removed, or have their contents
48 : * removed, if the operation fails.
49 : */
50 : typedef struct cb_cleanup_dir
51 : {
52 : char *target_path;
53 : bool rmtopdir;
54 : struct cb_cleanup_dir *next;
55 : } cb_cleanup_dir;
56 :
57 : /*
58 : * Stores a tablespace mapping provided using -T, --tablespace-mapping.
59 : */
60 : typedef struct cb_tablespace_mapping
61 : {
62 : char old_dir[MAXPGPATH];
63 : char new_dir[MAXPGPATH];
64 : struct cb_tablespace_mapping *next;
65 : } cb_tablespace_mapping;
66 :
67 : /*
68 : * Stores data parsed from all command-line options.
69 : */
70 : typedef struct cb_options
71 : {
72 : bool debug;
73 : char *output;
74 : bool dry_run;
75 : bool no_sync;
76 : cb_tablespace_mapping *tsmappings;
77 : pg_checksum_type manifest_checksums;
78 : bool no_manifest;
79 : DataDirSyncMethod sync_method;
80 : CopyMethod copy_method;
81 : } cb_options;
82 :
83 : /*
84 : * Data about a tablespace.
85 : *
86 : * Every normal tablespace needs a tablespace mapping, but in-place tablespaces
87 : * don't, so the list of tablespaces can contain more entries than the list of
88 : * tablespace mappings.
89 : */
90 : typedef struct cb_tablespace
91 : {
92 : Oid oid;
93 : bool in_place;
94 : char old_dir[MAXPGPATH];
95 : char new_dir[MAXPGPATH];
96 : struct cb_tablespace *next;
97 : } cb_tablespace;
98 :
99 : /* Directories to be removed if we exit uncleanly. */
100 : static cb_cleanup_dir *cleanup_dir_list = NULL;
101 :
102 : static void add_tablespace_mapping(cb_options *opt, char *arg);
103 : static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
104 : static uint64 check_control_files(int n_backups, char **backup_dirs);
105 : static void check_input_dir_permissions(char *dir);
106 : static void cleanup_directories_atexit(void);
107 : static void create_output_directory(char *dirname, cb_options *opt);
108 : static void help(const char *progname);
109 : static bool parse_oid(char *s, Oid *result);
110 : static void process_directory_recursively(Oid tsoid,
111 : char *input_directory,
112 : char *output_directory,
113 : char *relative_path,
114 : int n_prior_backups,
115 : char **prior_backup_dirs,
116 : manifest_data **manifests,
117 : manifest_writer *mwriter,
118 : cb_options *opt);
119 : static int read_pg_version_file(char *directory);
120 : static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
121 : static void reset_directory_cleanup_list(void);
122 : static cb_tablespace *scan_for_existing_tablespaces(char *pathname,
123 : cb_options *opt);
124 : static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen);
125 :
126 : /*
127 : * Main program.
128 : */
129 : int
130 50 : main(int argc, char *argv[])
131 : {
132 : static struct option long_options[] = {
133 : {"debug", no_argument, NULL, 'd'},
134 : {"dry-run", no_argument, NULL, 'n'},
135 : {"no-sync", no_argument, NULL, 'N'},
136 : {"output", required_argument, NULL, 'o'},
137 : {"tablespace-mapping", required_argument, NULL, 'T'},
138 : {"link", no_argument, NULL, 'k'},
139 : {"manifest-checksums", required_argument, NULL, 1},
140 : {"no-manifest", no_argument, NULL, 2},
141 : {"sync-method", required_argument, NULL, 3},
142 : {"clone", no_argument, NULL, 4},
143 : {"copy", no_argument, NULL, 5},
144 : {"copy-file-range", no_argument, NULL, 6},
145 : {NULL, 0, NULL, 0}
146 : };
147 :
148 : const char *progname;
149 : char *last_input_dir;
150 : int i;
151 : int optindex;
152 : int c;
153 : int n_backups;
154 : int n_prior_backups;
155 : int version;
156 : uint64 system_identifier;
157 : char **prior_backup_dirs;
158 : cb_options opt;
159 : cb_tablespace *tablespaces;
160 : cb_tablespace *ts;
161 : StringInfo last_backup_label;
162 : manifest_data **manifests;
163 : manifest_writer *mwriter;
164 :
165 50 : pg_logging_init(argv[0]);
166 50 : progname = get_progname(argv[0]);
167 50 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
168 50 : handle_help_version_opts(argc, argv, progname, help);
169 :
170 46 : memset(&opt, 0, sizeof(opt));
171 46 : opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
172 46 : opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
173 46 : opt.copy_method = COPY_METHOD_COPY;
174 :
175 : /* process command-line options */
176 144 : while ((c = getopt_long(argc, argv, "dknNo:T:",
177 : long_options, &optindex)) != -1)
178 : {
179 100 : switch (c)
180 : {
181 12 : case 'd':
182 12 : opt.debug = true;
183 12 : pg_logging_increase_verbosity();
184 12 : break;
185 2 : case 'k':
186 2 : opt.copy_method = COPY_METHOD_LINK;
187 2 : break;
188 0 : case 'n':
189 0 : opt.dry_run = true;
190 0 : break;
191 6 : case 'N':
192 6 : opt.no_sync = true;
193 6 : break;
194 40 : case 'o':
195 40 : opt.output = optarg;
196 40 : break;
197 2 : case 'T':
198 2 : add_tablespace_mapping(&opt, optarg);
199 2 : break;
200 4 : case 1:
201 4 : if (!pg_checksum_parse_type(optarg,
202 : &opt.manifest_checksums))
203 0 : pg_fatal("unrecognized checksum algorithm: \"%s\"",
204 : optarg);
205 4 : break;
206 2 : case 2:
207 2 : opt.no_manifest = true;
208 2 : break;
209 0 : case 3:
210 0 : if (!parse_sync_method(optarg, &opt.sync_method))
211 0 : exit(1);
212 0 : break;
213 0 : case 4:
214 0 : opt.copy_method = COPY_METHOD_CLONE;
215 0 : break;
216 30 : case 5:
217 30 : opt.copy_method = COPY_METHOD_COPY;
218 30 : break;
219 0 : case 6:
220 0 : opt.copy_method = COPY_METHOD_COPY_FILE_RANGE;
221 0 : break;
222 2 : default:
223 : /* getopt_long already emitted a complaint */
224 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
225 2 : exit(1);
226 : }
227 : }
228 :
229 44 : if (optind >= argc)
230 : {
231 2 : pg_log_error("no input directories specified");
232 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
233 2 : exit(1);
234 : }
235 :
236 42 : if (opt.output == NULL)
237 2 : pg_fatal("no output directory specified");
238 :
239 : /* If no manifest is needed, no checksums are needed, either. */
240 40 : if (opt.no_manifest)
241 2 : opt.manifest_checksums = CHECKSUM_TYPE_NONE;
242 :
243 : /* Check that the platform supports the requested copy method. */
244 40 : if (opt.copy_method == COPY_METHOD_CLONE)
245 : {
246 : #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
247 : (defined(__linux__) && defined(FICLONE))
248 :
249 0 : if (opt.dry_run)
250 0 : pg_log_debug("would use cloning to copy files");
251 : else
252 0 : pg_log_debug("will use cloning to copy files");
253 :
254 : #else
255 : pg_fatal("file cloning not supported on this platform");
256 : #endif
257 : }
258 40 : else if (opt.copy_method == COPY_METHOD_COPY_FILE_RANGE)
259 : {
260 : #if defined(HAVE_COPY_FILE_RANGE)
261 :
262 0 : if (opt.dry_run)
263 0 : pg_log_debug("would use copy_file_range to copy blocks");
264 : else
265 0 : pg_log_debug("will use copy_file_range to copy blocks");
266 :
267 : #else
268 : pg_fatal("copy_file_range not supported on this platform");
269 : #endif
270 : }
271 :
272 : /* Read the server version from the final backup. */
273 40 : version = read_pg_version_file(argv[argc - 1]);
274 :
275 : /* Sanity-check control files. */
276 40 : n_backups = argc - optind;
277 40 : system_identifier = check_control_files(n_backups, argv + optind);
278 :
279 : /* Sanity-check backup_label files, and get the contents of the last one. */
280 38 : last_backup_label = check_backup_label_files(n_backups, argv + optind);
281 :
282 : /*
283 : * We'll need the pathnames to the prior backups. By "prior" we mean all
284 : * but the last one listed on the command line.
285 : */
286 28 : n_prior_backups = argc - optind - 1;
287 28 : prior_backup_dirs = argv + optind;
288 :
289 : /* Load backup manifests. */
290 28 : manifests = load_backup_manifests(n_backups, prior_backup_dirs);
291 :
292 : /*
293 : * Validate the manifest system identifier against the backup system
294 : * identifier.
295 : */
296 80 : for (i = 0; i < n_backups; i++)
297 : {
298 54 : if (manifests[i] &&
299 54 : manifests[i]->system_identifier != system_identifier)
300 : {
301 : char *controlpath;
302 :
303 2 : controlpath = psprintf("%s/%s", prior_backup_dirs[i], "global/pg_control");
304 :
305 2 : pg_fatal("%s: manifest system identifier is %" PRIu64 ", but control file has %" PRIu64,
306 : controlpath,
307 : manifests[i]->system_identifier,
308 : system_identifier);
309 : }
310 : }
311 :
312 : /* Figure out which tablespaces are going to be included in the output. */
313 26 : last_input_dir = argv[argc - 1];
314 26 : check_input_dir_permissions(last_input_dir);
315 26 : tablespaces = scan_for_existing_tablespaces(last_input_dir, &opt);
316 :
317 : /*
318 : * Create output directories.
319 : *
320 : * We create one output directory for the main data directory plus one for
321 : * each non-in-place tablespace. create_output_directory() will arrange
322 : * for those directories to be cleaned up on failure. In-place tablespaces
323 : * aren't handled at this stage because they're located beneath the main
324 : * output directory, and thus the cleanup of that directory will get rid
325 : * of them. Plus, the pg_tblspc directory that needs to contain them
326 : * doesn't exist yet.
327 : */
328 26 : atexit(cleanup_directories_atexit);
329 26 : create_output_directory(opt.output, &opt);
330 28 : for (ts = tablespaces; ts != NULL; ts = ts->next)
331 2 : if (!ts->in_place)
332 2 : create_output_directory(ts->new_dir, &opt);
333 :
334 : /* If we need to write a backup_manifest, prepare to do so. */
335 26 : if (!opt.dry_run && !opt.no_manifest)
336 : {
337 24 : mwriter = create_manifest_writer(opt.output, system_identifier);
338 :
339 : /*
340 : * Verify that we have a backup manifest for the final backup; else we
341 : * won't have the WAL ranges for the resulting manifest.
342 : */
343 24 : if (manifests[n_prior_backups] == NULL)
344 0 : pg_fatal("cannot generate a manifest because no manifest is available for the final input backup");
345 : }
346 : else
347 2 : mwriter = NULL;
348 :
349 : /* Write backup label into output directory. */
350 26 : if (opt.dry_run)
351 0 : pg_log_debug("would generate \"%s/backup_label\"", opt.output);
352 : else
353 : {
354 26 : pg_log_debug("generating \"%s/backup_label\"", opt.output);
355 26 : last_backup_label->cursor = 0;
356 26 : write_backup_label(opt.output, last_backup_label,
357 : opt.manifest_checksums, mwriter);
358 : }
359 :
360 : /* Process everything that's not part of a user-defined tablespace. */
361 26 : pg_log_debug("processing backup directory \"%s\"", last_input_dir);
362 26 : process_directory_recursively(InvalidOid, last_input_dir, opt.output,
363 : NULL, n_prior_backups, prior_backup_dirs,
364 : manifests, mwriter, &opt);
365 :
366 : /* Process user-defined tablespaces. */
367 26 : for (ts = tablespaces; ts != NULL; ts = ts->next)
368 : {
369 2 : pg_log_debug("processing tablespace directory \"%s\"", ts->old_dir);
370 :
371 : /*
372 : * If it's a normal tablespace, we need to set up a symbolic link from
373 : * pg_tblspc/${OID} to the target directory; if it's an in-place
374 : * tablespace, we need to create a directory at pg_tblspc/${OID}.
375 : */
376 2 : if (!ts->in_place)
377 : {
378 : char linkpath[MAXPGPATH];
379 :
380 2 : snprintf(linkpath, MAXPGPATH, "%s/%s/%u", opt.output, PG_TBLSPC_DIR,
381 : ts->oid);
382 :
383 2 : if (opt.dry_run)
384 0 : pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
385 : linkpath, ts->new_dir);
386 : else
387 : {
388 2 : pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
389 : linkpath, ts->new_dir);
390 2 : if (symlink(ts->new_dir, linkpath) != 0)
391 0 : pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
392 : linkpath, ts->new_dir);
393 : }
394 : }
395 : else
396 : {
397 0 : if (opt.dry_run)
398 0 : pg_log_debug("would create directory \"%s\"", ts->new_dir);
399 : else
400 : {
401 0 : pg_log_debug("creating directory \"%s\"", ts->new_dir);
402 0 : if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
403 0 : pg_fatal("could not create directory \"%s\": %m",
404 : ts->new_dir);
405 : }
406 : }
407 :
408 : /* OK, now handle the directory contents. */
409 2 : process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
410 : NULL, n_prior_backups, prior_backup_dirs,
411 : manifests, mwriter, &opt);
412 : }
413 :
414 : /* Finalize the backup_manifest, if we're generating one. */
415 24 : if (mwriter != NULL)
416 22 : finalize_manifest(mwriter,
417 22 : manifests[n_prior_backups]->first_wal_range);
418 :
419 : /* fsync that output directory unless we've been told not to do so */
420 24 : if (!opt.no_sync)
421 : {
422 18 : if (opt.dry_run)
423 0 : pg_log_debug("would recursively fsync \"%s\"", opt.output);
424 : else
425 : {
426 18 : pg_log_debug("recursively fsyncing \"%s\"", opt.output);
427 18 : sync_pgdata(opt.output, version * 10000, opt.sync_method, true);
428 : }
429 : }
430 :
431 : /* Warn about the possibility of compromising the backups, when link mode */
432 24 : if (opt.copy_method == COPY_METHOD_LINK)
433 2 : pg_log_warning("--link mode was used; any modifications to the output "
434 : "directory may destructively modify input directories");
435 :
436 : /* It's a success, so don't remove the output directories. */
437 24 : reset_directory_cleanup_list();
438 24 : exit(0);
439 : }
440 :
441 : /*
442 : * Process the option argument for the -T, --tablespace-mapping switch.
443 : */
444 : static void
445 2 : add_tablespace_mapping(cb_options *opt, char *arg)
446 : {
447 2 : cb_tablespace_mapping *tsmap = pg_malloc0(sizeof(cb_tablespace_mapping));
448 : char *dst;
449 : char *dst_ptr;
450 : char *arg_ptr;
451 :
452 : /*
453 : * Basically, we just want to copy everything before the equals sign to
454 : * tsmap->old_dir and everything afterwards to tsmap->new_dir, but if
455 : * there's more or less than one equals sign, that's an error, and if
456 : * there's an equals sign preceded by a backslash, don't treat it as a
457 : * field separator but instead copy a literal equals sign.
458 : */
459 2 : dst_ptr = dst = tsmap->old_dir;
460 100 : for (arg_ptr = arg; *arg_ptr != '\0'; arg_ptr++)
461 : {
462 98 : if (dst_ptr - dst >= MAXPGPATH)
463 0 : pg_fatal("directory name too long");
464 :
465 98 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
466 : ; /* skip backslash escaping = */
467 98 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
468 : {
469 2 : if (tsmap->new_dir[0] != '\0')
470 0 : pg_fatal("multiple \"=\" signs in tablespace mapping");
471 : else
472 2 : dst = dst_ptr = tsmap->new_dir;
473 : }
474 : else
475 96 : *dst_ptr++ = *arg_ptr;
476 : }
477 2 : if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
478 0 : pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
479 :
480 : /*
481 : * All tablespaces are created with absolute directories, so specifying a
482 : * non-absolute path here would never match, possibly confusing users.
483 : *
484 : * In contrast to pg_basebackup, both the old and new directories are on
485 : * the local machine, so the local machine's definition of an absolute
486 : * path is the only relevant one.
487 : */
488 2 : if (!is_absolute_path(tsmap->old_dir))
489 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
490 : tsmap->old_dir);
491 :
492 2 : if (!is_absolute_path(tsmap->new_dir))
493 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
494 : tsmap->new_dir);
495 :
496 : /* Canonicalize paths to avoid spurious failures when comparing. */
497 2 : canonicalize_path(tsmap->old_dir);
498 2 : canonicalize_path(tsmap->new_dir);
499 :
500 : /* Add it to the list. */
501 2 : tsmap->next = opt->tsmappings;
502 2 : opt->tsmappings = tsmap;
503 2 : }
504 :
505 : /*
506 : * Check that the backup_label files form a coherent backup chain, and return
507 : * the contents of the backup_label file from the latest backup.
508 : */
509 : static StringInfo
510 38 : check_backup_label_files(int n_backups, char **backup_dirs)
511 : {
512 38 : StringInfo buf = makeStringInfo();
513 38 : StringInfo lastbuf = buf;
514 : int i;
515 38 : TimeLineID check_tli = 0;
516 38 : XLogRecPtr check_lsn = InvalidXLogRecPtr;
517 :
518 : /* Try to read each backup_label file in turn, last to first. */
519 102 : for (i = n_backups - 1; i >= 0; --i)
520 : {
521 : char pathbuf[MAXPGPATH];
522 : int fd;
523 : TimeLineID start_tli;
524 : TimeLineID previous_tli;
525 : XLogRecPtr start_lsn;
526 : XLogRecPtr previous_lsn;
527 :
528 : /* Open the backup_label file. */
529 74 : snprintf(pathbuf, MAXPGPATH, "%s/backup_label", backup_dirs[i]);
530 74 : pg_log_debug("reading \"%s\"", pathbuf);
531 74 : if ((fd = open(pathbuf, O_RDONLY, 0)) < 0)
532 0 : pg_fatal("could not open file \"%s\": %m", pathbuf);
533 :
534 : /*
535 : * Slurp the whole file into memory.
536 : *
537 : * The exact size limit that we impose here doesn't really matter --
538 : * most of what's supposed to be in the file is fixed size and quite
539 : * short. However, the length of the backup_label is limited (at least
540 : * by some parts of the code) to MAXPGPATH, so include that value in
541 : * the maximum length that we tolerate.
542 : */
543 74 : slurp_file(fd, pathbuf, buf, 10000 + MAXPGPATH);
544 :
545 : /* Close the file. */
546 74 : if (close(fd) != 0)
547 0 : pg_fatal("could not close file \"%s\": %m", pathbuf);
548 :
549 : /* Parse the file contents. */
550 74 : parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
551 : &previous_tli, &previous_lsn);
552 :
553 : /*
554 : * Sanity checks.
555 : *
556 : * XXX. It's actually not required that start_lsn == check_lsn. It
557 : * would be OK if start_lsn > check_lsn provided that start_lsn is
558 : * less than or equal to the relevant switchpoint. But at the moment
559 : * we don't have that information.
560 : */
561 74 : if (i > 0 && previous_tli == 0)
562 2 : pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
563 : backup_dirs[i]);
564 72 : if (i == 0 && previous_tli != 0)
565 2 : pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
566 : backup_dirs[i]);
567 70 : if (i < n_backups - 1 && start_tli != check_tli)
568 0 : pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
569 : backup_dirs[i], start_tli, check_tli);
570 70 : if (i < n_backups - 1 && start_lsn != check_lsn)
571 6 : pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
572 : backup_dirs[i],
573 : LSN_FORMAT_ARGS(start_lsn),
574 : LSN_FORMAT_ARGS(check_lsn));
575 64 : check_tli = previous_tli;
576 64 : check_lsn = previous_lsn;
577 :
578 : /*
579 : * The last backup label in the chain needs to be saved for later use,
580 : * while the others are only needed within this loop.
581 : */
582 64 : if (lastbuf == buf)
583 36 : buf = makeStringInfo();
584 : else
585 28 : resetStringInfo(buf);
586 : }
587 :
588 : /* Free memory that we don't need any more. */
589 28 : if (lastbuf != buf)
590 28 : destroyStringInfo(buf);
591 :
592 : /*
593 : * Return the data from the first backup_info that we read (which is the
594 : * backup_label from the last directory specified on the command line).
595 : */
596 28 : return lastbuf;
597 : }
598 :
599 : /*
600 : * Sanity check control files and return system_identifier.
601 : */
602 : static uint64
603 40 : check_control_files(int n_backups, char **backup_dirs)
604 : {
605 : int i;
606 40 : uint64 system_identifier = 0; /* placate compiler */
607 40 : uint32 data_checksum_version = 0; /* placate compiler */
608 40 : bool data_checksum_mismatch = false;
609 :
610 : /* Try to read each control file in turn, last to first. */
611 120 : for (i = n_backups - 1; i >= 0; --i)
612 : {
613 : ControlFileData *control_file;
614 : bool crc_ok;
615 : char *controlpath;
616 :
617 82 : controlpath = psprintf("%s/%s", backup_dirs[i], "global/pg_control");
618 82 : pg_log_debug("reading \"%s\"", controlpath);
619 82 : control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
620 :
621 : /* Control file contents not meaningful if CRC is bad. */
622 82 : if (!crc_ok)
623 0 : pg_fatal("%s: CRC is incorrect", controlpath);
624 :
625 : /* Can't interpret control file if not current version. */
626 82 : if (control_file->pg_control_version != PG_CONTROL_VERSION)
627 0 : pg_fatal("%s: unexpected control file version",
628 : controlpath);
629 :
630 : /* System identifiers should all match. */
631 82 : if (i == n_backups - 1)
632 40 : system_identifier = control_file->system_identifier;
633 42 : else if (system_identifier != control_file->system_identifier)
634 2 : pg_fatal("%s: expected system identifier %" PRIu64 ", but found %" PRIu64,
635 : controlpath, system_identifier,
636 : control_file->system_identifier);
637 :
638 : /*
639 : * Detect checksum mismatches, but only if the last backup in the
640 : * chain has checksums enabled.
641 : */
642 80 : if (i == n_backups - 1)
643 40 : data_checksum_version = control_file->data_checksum_version;
644 40 : else if (data_checksum_version != 0 &&
645 40 : data_checksum_version != control_file->data_checksum_version)
646 0 : data_checksum_mismatch = true;
647 :
648 : /* Release memory. */
649 80 : pfree(control_file);
650 80 : pfree(controlpath);
651 : }
652 :
653 : /*
654 : * If debug output is enabled, make a note of the system identifier that
655 : * we found in all of the relevant control files.
656 : */
657 38 : pg_log_debug("system identifier is %" PRIu64, system_identifier);
658 :
659 : /*
660 : * Warn the user if not all backups are in the same state with regards to
661 : * checksums.
662 : */
663 38 : if (data_checksum_mismatch)
664 : {
665 0 : pg_log_warning("only some backups have checksums enabled");
666 0 : pg_log_warning_hint("Disable, and optionally reenable, checksums on the output directory to avoid failures.");
667 : }
668 :
669 38 : return system_identifier;
670 : }
671 :
672 : /*
673 : * Set default permissions for new files and directories based on the
674 : * permissions of the given directory. The intent here is that the output
675 : * directory should use the same permissions scheme as the final input
676 : * directory.
677 : */
678 : static void
679 26 : check_input_dir_permissions(char *dir)
680 : {
681 : struct stat st;
682 :
683 26 : if (stat(dir, &st) != 0)
684 0 : pg_fatal("could not stat file \"%s\": %m", dir);
685 :
686 26 : SetDataDirectoryCreatePerm(st.st_mode);
687 26 : }
688 :
689 : /*
690 : * Clean up output directories before exiting.
691 : */
692 : static void
693 26 : cleanup_directories_atexit(void)
694 : {
695 28 : while (cleanup_dir_list != NULL)
696 : {
697 2 : cb_cleanup_dir *dir = cleanup_dir_list;
698 :
699 2 : if (dir->rmtopdir)
700 : {
701 2 : pg_log_info("removing output directory \"%s\"", dir->target_path);
702 2 : if (!rmtree(dir->target_path, dir->rmtopdir))
703 0 : pg_log_error("failed to remove output directory");
704 : }
705 : else
706 : {
707 0 : pg_log_info("removing contents of output directory \"%s\"",
708 : dir->target_path);
709 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
710 0 : pg_log_error("failed to remove contents of output directory");
711 : }
712 :
713 2 : cleanup_dir_list = cleanup_dir_list->next;
714 2 : pfree(dir);
715 : }
716 26 : }
717 :
718 : /*
719 : * Create the named output directory, unless it already exists or we're in
720 : * dry-run mode. If it already exists but is not empty, that's a fatal error.
721 : *
722 : * Adds the created directory to the list of directories to be cleaned up
723 : * at process exit.
724 : */
725 : static void
726 28 : create_output_directory(char *dirname, cb_options *opt)
727 : {
728 28 : switch (pg_check_dir(dirname))
729 : {
730 28 : case 0:
731 28 : if (opt->dry_run)
732 : {
733 0 : pg_log_debug("would create directory \"%s\"", dirname);
734 0 : return;
735 : }
736 28 : pg_log_debug("creating directory \"%s\"", dirname);
737 28 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
738 0 : pg_fatal("could not create directory \"%s\": %m", dirname);
739 28 : remember_to_cleanup_directory(dirname, true);
740 28 : break;
741 :
742 0 : case 1:
743 0 : pg_log_debug("using existing directory \"%s\"", dirname);
744 0 : remember_to_cleanup_directory(dirname, false);
745 0 : break;
746 :
747 0 : case 2:
748 : case 3:
749 : case 4:
750 0 : pg_fatal("directory \"%s\" exists but is not empty", dirname);
751 :
752 0 : case -1:
753 0 : pg_fatal("could not access directory \"%s\": %m", dirname);
754 : }
755 : }
756 :
757 : /*
758 : * help
759 : *
760 : * Prints help page for the program
761 : *
762 : * progname: the name of the executed program, such as "pg_combinebackup"
763 : */
764 : static void
765 2 : help(const char *progname)
766 : {
767 2 : printf(_("%s reconstructs full backups from incrementals.\n\n"), progname);
768 2 : printf(_("Usage:\n"));
769 2 : printf(_(" %s [OPTION]... DIRECTORY...\n"), progname);
770 2 : printf(_("\nOptions:\n"));
771 2 : printf(_(" -d, --debug generate lots of debugging output\n"));
772 2 : printf(_(" -k, --link link files instead of copying\n"));
773 2 : printf(_(" -n, --dry-run do not actually do anything\n"));
774 2 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
775 2 : printf(_(" -o, --output=DIRECTORY output directory\n"));
776 2 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
777 : " relocate tablespace in OLDDIR to NEWDIR\n"));
778 2 : printf(_(" --clone clone (reflink) files instead of copying\n"));
779 2 : printf(_(" --copy copy files (default)\n"));
780 2 : printf(_(" --copy-file-range copy using copy_file_range() system call\n"));
781 2 : printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
782 : " use algorithm for manifest checksums\n"));
783 2 : printf(_(" --no-manifest suppress generation of backup manifest\n"));
784 2 : printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
785 2 : printf(_(" -V, --version output version information, then exit\n"));
786 2 : printf(_(" -?, --help show this help, then exit\n"));
787 :
788 2 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
789 2 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
790 2 : }
791 :
792 : /*
793 : * Try to parse a string as a non-zero OID without leading zeroes.
794 : *
795 : * If it works, return true and set *result to the answer, else return false.
796 : */
797 : static bool
798 4 : parse_oid(char *s, Oid *result)
799 : {
800 : Oid oid;
801 : char *ep;
802 :
803 4 : errno = 0;
804 4 : oid = strtoul(s, &ep, 10);
805 4 : if (errno != 0 || *ep != '\0' || oid < 1 || oid > PG_UINT32_MAX)
806 0 : return false;
807 :
808 4 : *result = oid;
809 4 : return true;
810 : }
811 :
812 : /*
813 : * Copy files from the input directory to the output directory, reconstructing
814 : * full files from incremental files as required.
815 : *
816 : * If processing is a user-defined tablespace, the tsoid should be the OID
817 : * of that tablespace and input_directory and output_directory should be the
818 : * toplevel input and output directories for that tablespace. Otherwise,
819 : * tsoid should be InvalidOid and input_directory and output_directory should
820 : * be the main input and output directories.
821 : *
822 : * relative_path is the path beneath the given input and output directories
823 : * that we are currently processing. If NULL, it indicates that we're
824 : * processing the input and output directories themselves.
825 : *
826 : * n_prior_backups is the number of prior backups that we have available.
827 : * This doesn't count the very last backup, which is referenced by
828 : * output_directory, just the older ones. prior_backup_dirs is an array of
829 : * the locations of those previous backups.
830 : */
831 : static void
832 708 : process_directory_recursively(Oid tsoid,
833 : char *input_directory,
834 : char *output_directory,
835 : char *relative_path,
836 : int n_prior_backups,
837 : char **prior_backup_dirs,
838 : manifest_data **manifests,
839 : manifest_writer *mwriter,
840 : cb_options *opt)
841 : {
842 : char ifulldir[MAXPGPATH];
843 : char ofulldir[MAXPGPATH];
844 : char manifest_prefix[MAXPGPATH];
845 : DIR *dir;
846 : struct dirent *de;
847 708 : bool is_pg_tblspc = false;
848 708 : bool is_pg_wal = false;
849 708 : bool is_incremental_dir = false;
850 708 : manifest_data *latest_manifest = manifests[n_prior_backups];
851 : pg_checksum_type checksum_type;
852 :
853 : /*
854 : * Classify this directory.
855 : *
856 : * We set is_pg_tblspc only for the toplevel pg_tblspc directory, because
857 : * the symlinks in that specific directory require special handling.
858 : *
859 : * We set is_pg_wal for the toplevel WAL directory and all of its
860 : * subdirectories, because those files are not included in the backup
861 : * manifest and hence need special treatment. (Since incremental backup
862 : * does not exist in pre-v10 versions, we don't have to worry about the
863 : * old pg_xlog naming.)
864 : *
865 : * We set is_incremental_dir for directories that can contain incremental
866 : * files requiring reconstruction. If such files occur outside these
867 : * directories, we want to just copy them straight to the output
868 : * directory. This is to protect against a user creating a file with a
869 : * strange name like INCREMENTAL.config and then complaining that
870 : * incremental backups don't work properly. The test here is a bit tricky:
871 : * incremental files occur in subdirectories of base, in pg_global itself,
872 : * and in subdirectories of pg_tblspc only if in-place tablespaces are
873 : * used.
874 : */
875 708 : if (OidIsValid(tsoid))
876 6 : is_incremental_dir = true;
877 702 : else if (relative_path != NULL)
878 : {
879 676 : is_pg_tblspc = strcmp(relative_path, PG_TBLSPC_DIR) == 0;
880 1326 : is_pg_wal = (strcmp(relative_path, "pg_wal") == 0 ||
881 650 : strncmp(relative_path, "pg_wal/", 7) == 0);
882 676 : is_incremental_dir = strncmp(relative_path, "base/", 5) == 0 ||
883 1244 : strcmp(relative_path, "global") == 0 ||
884 568 : strncmp(relative_path, PG_TBLSPC_DIR_SLASH, 10) == 0;
885 : }
886 :
887 : /*
888 : * If we're under pg_wal, then we don't need checksums, because these
889 : * files aren't included in the backup manifest. Otherwise use whatever
890 : * type of checksum is configured.
891 : */
892 708 : if (!is_pg_wal)
893 630 : checksum_type = opt->manifest_checksums;
894 : else
895 78 : checksum_type = CHECKSUM_TYPE_NONE;
896 :
897 : /*
898 : * Append the relative path to the input and output directories, and
899 : * figure out the appropriate prefix to add to files in this directory
900 : * when looking them up in a backup manifest.
901 : */
902 708 : if (relative_path == NULL)
903 : {
904 28 : strlcpy(ifulldir, input_directory, MAXPGPATH);
905 28 : strlcpy(ofulldir, output_directory, MAXPGPATH);
906 28 : if (OidIsValid(tsoid))
907 2 : snprintf(manifest_prefix, MAXPGPATH, "%s/%u/", PG_TBLSPC_DIR, tsoid);
908 : else
909 26 : manifest_prefix[0] = '\0';
910 : }
911 : else
912 : {
913 680 : snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
914 : relative_path);
915 680 : snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
916 : relative_path);
917 680 : if (OidIsValid(tsoid))
918 4 : snprintf(manifest_prefix, MAXPGPATH, "%s/%u/%s/",
919 : PG_TBLSPC_DIR, tsoid, relative_path);
920 : else
921 676 : snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
922 : }
923 :
924 : /*
925 : * Toplevel output directories have already been created by the time this
926 : * function is called, but any subdirectories are our responsibility.
927 : */
928 708 : if (relative_path != NULL)
929 : {
930 680 : if (opt->dry_run)
931 0 : pg_log_debug("would create directory \"%s\"", ofulldir);
932 : else
933 : {
934 680 : pg_log_debug("creating directory \"%s\"", ofulldir);
935 680 : if (mkdir(ofulldir, pg_dir_create_mode) == -1)
936 0 : pg_fatal("could not create directory \"%s\": %m", ofulldir);
937 : }
938 : }
939 :
940 : /* It's time to scan the directory. */
941 708 : if ((dir = opendir(ifulldir)) == NULL)
942 0 : pg_fatal("could not open directory \"%s\": %m", ifulldir);
943 28784 : while (errno = 0, (de = readdir(dir)) != NULL)
944 : {
945 : PGFileType type;
946 : char ifullpath[MAXPGPATH];
947 : char ofullpath[MAXPGPATH];
948 : char manifest_path[MAXPGPATH];
949 28082 : Oid oid = InvalidOid;
950 28082 : int checksum_length = 0;
951 28082 : uint8 *checksum_payload = NULL;
952 : pg_checksum_context checksum_ctx;
953 :
954 : /* Ignore "." and ".." entries. */
955 28082 : if (strcmp(de->d_name, ".") == 0 ||
956 27374 : strcmp(de->d_name, "..") == 0)
957 2146 : continue;
958 :
959 : /* Construct input path. */
960 26666 : snprintf(ifullpath, MAXPGPATH, "%s/%s", ifulldir, de->d_name);
961 :
962 : /* Figure out what kind of directory entry this is. */
963 26666 : type = get_dirent_type(ifullpath, de, false, PG_LOG_ERROR);
964 26666 : if (type == PGFILETYPE_ERROR)
965 0 : exit(1);
966 :
967 : /*
968 : * If we're processing pg_tblspc, then check whether the filename
969 : * looks like it could be a tablespace OID. If so, and if the
970 : * directory entry is a symbolic link or a directory, skip it.
971 : *
972 : * Our goal here is to ignore anything that would have been considered
973 : * by scan_for_existing_tablespaces to be a tablespace.
974 : */
975 26666 : if (is_pg_tblspc && parse_oid(de->d_name, &oid) &&
976 0 : (type == PGFILETYPE_LNK || type == PGFILETYPE_DIR))
977 2 : continue;
978 :
979 : /* If it's a directory, recurse. */
980 26664 : if (type == PGFILETYPE_DIR)
981 : {
982 : char new_relative_path[MAXPGPATH];
983 :
984 : /* Append new pathname component to relative path. */
985 680 : if (relative_path == NULL)
986 440 : strlcpy(new_relative_path, de->d_name, MAXPGPATH);
987 : else
988 240 : snprintf(new_relative_path, MAXPGPATH, "%s/%s", relative_path,
989 240 : de->d_name);
990 :
991 : /* And recurse. */
992 680 : process_directory_recursively(tsoid,
993 : input_directory, output_directory,
994 : new_relative_path,
995 : n_prior_backups, prior_backup_dirs,
996 : manifests, mwriter, opt);
997 676 : continue;
998 : }
999 :
1000 : /* Skip anything that's not a regular file. */
1001 25984 : if (type != PGFILETYPE_REG)
1002 : {
1003 0 : if (type == PGFILETYPE_LNK)
1004 0 : pg_log_warning("skipping symbolic link \"%s\"", ifullpath);
1005 : else
1006 0 : pg_log_warning("skipping special file \"%s\"", ifullpath);
1007 0 : continue;
1008 : }
1009 :
1010 : /*
1011 : * Skip the backup_label and backup_manifest files; they require
1012 : * special handling and are handled elsewhere.
1013 : */
1014 25984 : if (relative_path == NULL &&
1015 190 : (strcmp(de->d_name, "backup_label") == 0 ||
1016 164 : strcmp(de->d_name, "backup_manifest") == 0))
1017 52 : continue;
1018 :
1019 : /*
1020 : * If it's an incremental file, hand it off to the reconstruction
1021 : * code, which will figure out what to do.
1022 : */
1023 25932 : if (is_incremental_dir &&
1024 25654 : strncmp(de->d_name, INCREMENTAL_PREFIX,
1025 : INCREMENTAL_PREFIX_LENGTH) == 0)
1026 : {
1027 : /* Output path should not include "INCREMENTAL." prefix. */
1028 12926 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
1029 12926 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1030 :
1031 :
1032 : /* Manifest path likewise omits incremental prefix. */
1033 12926 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1034 12926 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1035 :
1036 : /* Reconstruction logic will do the rest. */
1037 12926 : reconstruct_from_incremental_file(ifullpath, ofullpath,
1038 : manifest_prefix,
1039 12926 : de->d_name + INCREMENTAL_PREFIX_LENGTH,
1040 : n_prior_backups,
1041 : prior_backup_dirs,
1042 : manifests,
1043 : manifest_path,
1044 : checksum_type,
1045 : &checksum_length,
1046 : &checksum_payload,
1047 : opt->copy_method,
1048 12926 : opt->debug,
1049 12926 : opt->dry_run);
1050 : }
1051 : else
1052 : {
1053 : /* Construct the path that the backup_manifest will use. */
1054 13006 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1055 13006 : de->d_name);
1056 :
1057 : /*
1058 : * It's not an incremental file, so we need to copy the entire
1059 : * file to the output directory.
1060 : *
1061 : * If a checksum of the required type already exists in the
1062 : * backup_manifest for the final input directory, we can save some
1063 : * work by reusing that checksum instead of computing a new one.
1064 : */
1065 13006 : if (checksum_type != CHECKSUM_TYPE_NONE &&
1066 : latest_manifest != NULL)
1067 : {
1068 : manifest_file *mfile;
1069 :
1070 9094 : mfile = manifest_files_lookup(latest_manifest->files,
1071 : manifest_path);
1072 9094 : if (mfile == NULL)
1073 : {
1074 : char *bmpath;
1075 :
1076 : /*
1077 : * The directory is out of sync with the backup_manifest,
1078 : * so emit a warning.
1079 : */
1080 0 : bmpath = psprintf("%s/%s", input_directory,
1081 : "backup_manifest");
1082 0 : pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
1083 : bmpath, manifest_path);
1084 0 : pfree(bmpath);
1085 : }
1086 9094 : else if (mfile->checksum_type == checksum_type)
1087 : {
1088 7156 : checksum_length = mfile->checksum_length;
1089 7156 : checksum_payload = mfile->checksum_payload;
1090 : }
1091 : }
1092 :
1093 : /*
1094 : * If we're reusing a checksum, then we don't need copy_file() to
1095 : * compute one for us, but otherwise, it needs to compute whatever
1096 : * type of checksum we need.
1097 : */
1098 13006 : if (checksum_length != 0)
1099 7156 : pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_NONE);
1100 : else
1101 5850 : pg_checksum_init(&checksum_ctx, checksum_type);
1102 :
1103 : /* Actually copy the file. */
1104 13006 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
1105 13006 : copy_file(ifullpath, ofullpath, &checksum_ctx,
1106 13006 : opt->copy_method, opt->dry_run);
1107 :
1108 : /*
1109 : * If copy_file() performed a checksum calculation for us, then
1110 : * save the results (except in dry-run mode, when there's no
1111 : * point).
1112 : */
1113 13006 : if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
1114 : {
1115 1938 : checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
1116 1938 : checksum_length = pg_checksum_final(&checksum_ctx,
1117 : checksum_payload);
1118 : }
1119 : }
1120 :
1121 : /* Generate manifest entry, if needed. */
1122 25930 : if (mwriter != NULL)
1123 : {
1124 : struct stat sb;
1125 :
1126 : /*
1127 : * In order to generate a manifest entry, we need the file size
1128 : * and mtime. We have no way to know the correct mtime except to
1129 : * stat() the file, so just do that and get the size as well.
1130 : *
1131 : * If we didn't need the mtime here, we could try to obtain the
1132 : * file size from the reconstruction or file copy process above,
1133 : * although that is actually not convenient in all cases. If we
1134 : * write the file ourselves then clearly we can keep a count of
1135 : * bytes, but if we use something like CopyFile() then it's
1136 : * trickier. Since we have to stat() anyway to get the mtime,
1137 : * there's no point in worrying about it.
1138 : */
1139 23988 : if (stat(ofullpath, &sb) < 0)
1140 0 : pg_fatal("could not stat file \"%s\": %m", ofullpath);
1141 :
1142 : /* OK, now do the work. */
1143 23988 : add_file_to_manifest(mwriter, manifest_path,
1144 23988 : sb.st_size, sb.st_mtime,
1145 : checksum_type, checksum_length,
1146 : checksum_payload);
1147 : }
1148 :
1149 : /* Avoid leaking memory. */
1150 25930 : if (checksum_payload != NULL)
1151 22018 : pfree(checksum_payload);
1152 : }
1153 :
1154 702 : closedir(dir);
1155 702 : }
1156 :
1157 : /*
1158 : * Read the version number from PG_VERSION and convert it to the usual server
1159 : * version number format. (e.g. If PG_VERSION contains "14\n" this function
1160 : * will return 140000)
1161 : */
1162 : static int
1163 40 : read_pg_version_file(char *directory)
1164 : {
1165 : char filename[MAXPGPATH];
1166 : StringInfoData buf;
1167 : int fd;
1168 : int version;
1169 : char *ep;
1170 :
1171 : /* Construct pathname. */
1172 40 : snprintf(filename, MAXPGPATH, "%s/PG_VERSION", directory);
1173 :
1174 : /* Open file. */
1175 40 : if ((fd = open(filename, O_RDONLY, 0)) < 0)
1176 0 : pg_fatal("could not open file \"%s\": %m", filename);
1177 :
1178 : /* Read into memory. Length limit of 128 should be more than generous. */
1179 40 : initStringInfo(&buf);
1180 40 : slurp_file(fd, filename, &buf, 128);
1181 :
1182 : /* Close the file. */
1183 40 : if (close(fd) != 0)
1184 0 : pg_fatal("could not close file \"%s\": %m", filename);
1185 :
1186 : /* Convert to integer. */
1187 40 : errno = 0;
1188 40 : version = strtoul(buf.data, &ep, 10);
1189 40 : if (errno != 0 || *ep != '\n')
1190 : {
1191 : /*
1192 : * Incremental backup is not relevant to very old server versions that
1193 : * used multi-part version number (e.g. 9.6, or 8.4). So if we see
1194 : * what looks like the beginning of such a version number, just bail
1195 : * out.
1196 : */
1197 0 : if (version < 10 && *ep == '.')
1198 0 : pg_fatal("%s: server version too old", filename);
1199 0 : pg_fatal("%s: could not parse version number", filename);
1200 : }
1201 :
1202 : /* Debugging output. */
1203 40 : pg_log_debug("read server version %d from file \"%s\"", version, filename);
1204 :
1205 : /* Release memory and return result. */
1206 40 : pfree(buf.data);
1207 40 : return version * 10000;
1208 : }
1209 :
1210 : /*
1211 : * Add a directory to the list of output directories to clean up.
1212 : */
1213 : static void
1214 28 : remember_to_cleanup_directory(char *target_path, bool rmtopdir)
1215 : {
1216 28 : cb_cleanup_dir *dir = pg_malloc(sizeof(cb_cleanup_dir));
1217 :
1218 28 : dir->target_path = target_path;
1219 28 : dir->rmtopdir = rmtopdir;
1220 28 : dir->next = cleanup_dir_list;
1221 28 : cleanup_dir_list = dir;
1222 28 : }
1223 :
1224 : /*
1225 : * Empty out the list of directories scheduled for cleanup at exit.
1226 : *
1227 : * We want to remove the output directories only on a failure, so call this
1228 : * function when we know that the operation has succeeded.
1229 : *
1230 : * Since we only expect this to be called when we're about to exit, we could
1231 : * just set cleanup_dir_list to NULL and be done with it, but we free the
1232 : * memory to be tidy.
1233 : */
1234 : static void
1235 24 : reset_directory_cleanup_list(void)
1236 : {
1237 50 : while (cleanup_dir_list != NULL)
1238 : {
1239 26 : cb_cleanup_dir *dir = cleanup_dir_list;
1240 :
1241 26 : cleanup_dir_list = cleanup_dir_list->next;
1242 26 : pfree(dir);
1243 : }
1244 24 : }
1245 :
1246 : /*
1247 : * Scan the pg_tblspc directory of the final input backup to get a canonical
1248 : * list of what tablespaces are part of the backup.
1249 : *
1250 : * 'pathname' should be the path to the toplevel backup directory for the
1251 : * final backup in the backup chain.
1252 : */
1253 : static cb_tablespace *
1254 26 : scan_for_existing_tablespaces(char *pathname, cb_options *opt)
1255 : {
1256 : char pg_tblspc[MAXPGPATH];
1257 : DIR *dir;
1258 : struct dirent *de;
1259 26 : cb_tablespace *tslist = NULL;
1260 :
1261 26 : snprintf(pg_tblspc, MAXPGPATH, "%s/%s", pathname, PG_TBLSPC_DIR);
1262 26 : pg_log_debug("scanning \"%s\"", pg_tblspc);
1263 :
1264 26 : if ((dir = opendir(pg_tblspc)) == NULL)
1265 0 : pg_fatal("could not open directory \"%s\": %m", pg_tblspc);
1266 :
1267 80 : while (errno = 0, (de = readdir(dir)) != NULL)
1268 : {
1269 : Oid oid;
1270 : char tblspcdir[MAXPGPATH];
1271 : char link_target[MAXPGPATH];
1272 : int link_length;
1273 : cb_tablespace *ts;
1274 : cb_tablespace *otherts;
1275 : PGFileType type;
1276 :
1277 : /* Silently ignore "." and ".." entries. */
1278 54 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1279 52 : continue;
1280 :
1281 : /* Construct full pathname. */
1282 2 : snprintf(tblspcdir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1283 :
1284 : /* Ignore any file name that doesn't look like a proper OID. */
1285 2 : if (!parse_oid(de->d_name, &oid))
1286 : {
1287 0 : pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
1288 : tblspcdir);
1289 0 : continue;
1290 : }
1291 :
1292 : /* Only symbolic links and directories are tablespaces. */
1293 2 : type = get_dirent_type(tblspcdir, de, false, PG_LOG_ERROR);
1294 2 : if (type == PGFILETYPE_ERROR)
1295 0 : exit(1);
1296 2 : if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
1297 : {
1298 0 : pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
1299 : tblspcdir);
1300 0 : continue;
1301 : }
1302 :
1303 : /* Create a new tablespace object. */
1304 2 : ts = pg_malloc0(sizeof(cb_tablespace));
1305 2 : ts->oid = oid;
1306 :
1307 : /*
1308 : * If it's a link, it's not an in-place tablespace. Otherwise, it must
1309 : * be a directory, and thus an in-place tablespace.
1310 : */
1311 2 : if (type == PGFILETYPE_LNK)
1312 : {
1313 : cb_tablespace_mapping *tsmap;
1314 :
1315 : /* Read the link target. */
1316 2 : link_length = readlink(tblspcdir, link_target, sizeof(link_target));
1317 2 : if (link_length < 0)
1318 0 : pg_fatal("could not read symbolic link \"%s\": %m",
1319 : tblspcdir);
1320 2 : if (link_length >= sizeof(link_target))
1321 0 : pg_fatal("target of symbolic link \"%s\" is too long", tblspcdir);
1322 2 : link_target[link_length] = '\0';
1323 2 : if (!is_absolute_path(link_target))
1324 0 : pg_fatal("target of symbolic link \"%s\" is relative", tblspcdir);
1325 :
1326 : /* Canonicalize the link target. */
1327 2 : canonicalize_path(link_target);
1328 :
1329 : /*
1330 : * Find the corresponding tablespace mapping and copy the relevant
1331 : * details into the new tablespace entry.
1332 : */
1333 2 : for (tsmap = opt->tsmappings; tsmap != NULL; tsmap = tsmap->next)
1334 : {
1335 2 : if (strcmp(tsmap->old_dir, link_target) == 0)
1336 : {
1337 2 : strlcpy(ts->old_dir, tsmap->old_dir, MAXPGPATH);
1338 2 : strlcpy(ts->new_dir, tsmap->new_dir, MAXPGPATH);
1339 2 : ts->in_place = false;
1340 2 : break;
1341 : }
1342 : }
1343 :
1344 : /* Every non-in-place tablespace must be mapped. */
1345 2 : if (tsmap == NULL)
1346 0 : pg_fatal("tablespace at \"%s\" has no tablespace mapping",
1347 : link_target);
1348 : }
1349 : else
1350 : {
1351 : /*
1352 : * For an in-place tablespace, there's no separate directory, so
1353 : * we just record the paths within the data directories.
1354 : */
1355 0 : snprintf(ts->old_dir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1356 0 : snprintf(ts->new_dir, MAXPGPATH, "%s/%s/%s", opt->output,
1357 0 : PG_TBLSPC_DIR, de->d_name);
1358 0 : ts->in_place = true;
1359 : }
1360 :
1361 : /* Tablespaces should not share a directory. */
1362 2 : for (otherts = tslist; otherts != NULL; otherts = otherts->next)
1363 0 : if (strcmp(ts->new_dir, otherts->new_dir) == 0)
1364 0 : pg_fatal("tablespaces with OIDs %u and %u both point at directory \"%s\"",
1365 : otherts->oid, oid, ts->new_dir);
1366 :
1367 : /* Add this tablespace to the list. */
1368 2 : ts->next = tslist;
1369 2 : tslist = ts;
1370 : }
1371 :
1372 26 : if (closedir(dir) != 0)
1373 0 : pg_fatal("could not close directory \"%s\": %m", pg_tblspc);
1374 :
1375 26 : return tslist;
1376 : }
1377 :
1378 : /*
1379 : * Read a file into a StringInfo.
1380 : *
1381 : * fd is used for the actual file I/O, filename for error reporting purposes.
1382 : * A file longer than maxlen is a fatal error.
1383 : */
1384 : static void
1385 114 : slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
1386 : {
1387 : struct stat st;
1388 : ssize_t rb;
1389 :
1390 : /* Check file size, and complain if it's too large. */
1391 114 : if (fstat(fd, &st) != 0)
1392 0 : pg_fatal("could not stat file \"%s\": %m", filename);
1393 114 : if (st.st_size > maxlen)
1394 0 : pg_fatal("file \"%s\" is too large", filename);
1395 :
1396 : /* Make sure we have enough space. */
1397 114 : enlargeStringInfo(buf, st.st_size);
1398 :
1399 : /* Read the data. */
1400 114 : rb = read(fd, &buf->data[buf->len], st.st_size);
1401 :
1402 : /*
1403 : * We don't expect any concurrent changes, so we should read exactly the
1404 : * expected number of bytes.
1405 : */
1406 114 : if (rb != st.st_size)
1407 : {
1408 0 : if (rb < 0)
1409 0 : pg_fatal("could not read file \"%s\": %m", filename);
1410 : else
1411 0 : pg_fatal("could not read file \"%s\": read %zd of %lld",
1412 : filename, rb, (long long int) st.st_size);
1413 : }
1414 :
1415 : /* Adjust buffer length for new data and restore trailing-\0 invariant */
1416 114 : buf->len += rb;
1417 114 : buf->data[buf->len] = '\0';
1418 114 : }
|
