Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_combinebackup.c
4 : * Combine incremental backups with prior backups.
5 : *
6 : * Copyright (c) 2017-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/bin/pg_combinebackup/pg_combinebackup.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres_fe.h"
14 :
15 : #include <dirent.h>
16 : #include <fcntl.h>
17 : #include <limits.h>
18 :
19 : #ifdef HAVE_COPYFILE_H
20 : #include <copyfile.h>
21 : #endif
22 : #ifdef __linux__
23 : #include <sys/ioctl.h>
24 : #include <linux/fs.h>
25 : #endif
26 :
27 : #include "backup_label.h"
28 : #include "common/blkreftable.h"
29 : #include "common/checksum_helper.h"
30 : #include "common/controldata_utils.h"
31 : #include "common/file_perm.h"
32 : #include "common/file_utils.h"
33 : #include "common/logging.h"
34 : #include "copy_file.h"
35 : #include "fe_utils/option_utils.h"
36 : #include "getopt_long.h"
37 : #include "lib/stringinfo.h"
38 : #include "load_manifest.h"
39 : #include "reconstruct.h"
40 : #include "write_manifest.h"
41 :
42 : /* Incremental file naming convention. */
43 : #define INCREMENTAL_PREFIX "INCREMENTAL."
44 : #define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
45 :
46 : /*
47 : * Tracking for directories that need to be removed, or have their contents
48 : * removed, if the operation fails.
49 : */
50 : typedef struct cb_cleanup_dir
51 : {
52 : char *target_path;
53 : bool rmtopdir;
54 : struct cb_cleanup_dir *next;
55 : } cb_cleanup_dir;
56 :
57 : /*
58 : * Stores a tablespace mapping provided using -T, --tablespace-mapping.
59 : */
60 : typedef struct cb_tablespace_mapping
61 : {
62 : char old_dir[MAXPGPATH];
63 : char new_dir[MAXPGPATH];
64 : struct cb_tablespace_mapping *next;
65 : } cb_tablespace_mapping;
66 :
67 : /*
68 : * Stores data parsed from all command-line options.
69 : */
70 : typedef struct cb_options
71 : {
72 : bool debug;
73 : char *output;
74 : bool dry_run;
75 : bool no_sync;
76 : cb_tablespace_mapping *tsmappings;
77 : pg_checksum_type manifest_checksums;
78 : bool no_manifest;
79 : DataDirSyncMethod sync_method;
80 : CopyMethod copy_method;
81 : } cb_options;
82 :
83 : /*
84 : * Data about a tablespace.
85 : *
86 : * Every normal tablespace needs a tablespace mapping, but in-place tablespaces
87 : * don't, so the list of tablespaces can contain more entries than the list of
88 : * tablespace mappings.
89 : */
90 : typedef struct cb_tablespace
91 : {
92 : Oid oid;
93 : bool in_place;
94 : char old_dir[MAXPGPATH];
95 : char new_dir[MAXPGPATH];
96 : struct cb_tablespace *next;
97 : } cb_tablespace;
98 :
99 : /* Directories to be removed if we exit uncleanly. */
100 : static cb_cleanup_dir *cleanup_dir_list = NULL;
101 :
102 : static void add_tablespace_mapping(cb_options *opt, char *arg);
103 : static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
104 : static uint64 check_control_files(int n_backups, char **backup_dirs);
105 : static void check_input_dir_permissions(char *dir);
106 : static void cleanup_directories_atexit(void);
107 : static void create_output_directory(char *dirname, cb_options *opt);
108 : static void help(const char *progname);
109 : static bool parse_oid(char *s, Oid *result);
110 : static void process_directory_recursively(Oid tsoid,
111 : char *input_directory,
112 : char *output_directory,
113 : char *relative_path,
114 : int n_prior_backups,
115 : char **prior_backup_dirs,
116 : manifest_data **manifests,
117 : manifest_writer *mwriter,
118 : cb_options *opt);
119 : static int read_pg_version_file(char *directory);
120 : static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
121 : static void reset_directory_cleanup_list(void);
122 : static cb_tablespace *scan_for_existing_tablespaces(char *pathname,
123 : cb_options *opt);
124 : static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen);
125 :
126 : /*
127 : * Main program.
128 : */
129 : int
130 46 : main(int argc, char *argv[])
131 : {
132 : static struct option long_options[] = {
133 : {"debug", no_argument, NULL, 'd'},
134 : {"dry-run", no_argument, NULL, 'n'},
135 : {"no-sync", no_argument, NULL, 'N'},
136 : {"output", required_argument, NULL, 'o'},
137 : {"tablespace-mapping", required_argument, NULL, 'T'},
138 : {"manifest-checksums", required_argument, NULL, 1},
139 : {"no-manifest", no_argument, NULL, 2},
140 : {"sync-method", required_argument, NULL, 3},
141 : {"clone", no_argument, NULL, 4},
142 : {"copy", no_argument, NULL, 5},
143 : {"copy-file-range", no_argument, NULL, 6},
144 : {NULL, 0, NULL, 0}
145 : };
146 :
147 : const char *progname;
148 : char *last_input_dir;
149 : int i;
150 : int optindex;
151 : int c;
152 : int n_backups;
153 : int n_prior_backups;
154 : int version;
155 : uint64 system_identifier;
156 : char **prior_backup_dirs;
157 : cb_options opt;
158 : cb_tablespace *tablespaces;
159 : cb_tablespace *ts;
160 : StringInfo last_backup_label;
161 : manifest_data **manifests;
162 : manifest_writer *mwriter;
163 :
164 46 : pg_logging_init(argv[0]);
165 46 : progname = get_progname(argv[0]);
166 46 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
167 46 : handle_help_version_opts(argc, argv, progname, help);
168 :
169 42 : memset(&opt, 0, sizeof(opt));
170 42 : opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
171 42 : opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
172 42 : opt.copy_method = COPY_METHOD_COPY;
173 :
174 : /* process command-line options */
175 132 : while ((c = getopt_long(argc, argv, "dnNo:T:",
176 : long_options, &optindex)) != -1)
177 : {
178 92 : switch (c)
179 : {
180 10 : case 'd':
181 10 : opt.debug = true;
182 10 : pg_logging_increase_verbosity();
183 10 : break;
184 0 : case 'n':
185 0 : opt.dry_run = true;
186 0 : break;
187 6 : case 'N':
188 6 : opt.no_sync = true;
189 6 : break;
190 36 : case 'o':
191 36 : opt.output = optarg;
192 36 : break;
193 2 : case 'T':
194 2 : add_tablespace_mapping(&opt, optarg);
195 2 : break;
196 4 : case 1:
197 4 : if (!pg_checksum_parse_type(optarg,
198 : &opt.manifest_checksums))
199 0 : pg_fatal("unrecognized checksum algorithm: \"%s\"",
200 : optarg);
201 4 : break;
202 2 : case 2:
203 2 : opt.no_manifest = true;
204 2 : break;
205 0 : case 3:
206 0 : if (!parse_sync_method(optarg, &opt.sync_method))
207 0 : exit(1);
208 0 : break;
209 0 : case 4:
210 0 : opt.copy_method = COPY_METHOD_CLONE;
211 0 : break;
212 30 : case 5:
213 30 : opt.copy_method = COPY_METHOD_COPY;
214 30 : break;
215 0 : case 6:
216 0 : opt.copy_method = COPY_METHOD_COPY_FILE_RANGE;
217 0 : break;
218 2 : default:
219 : /* getopt_long already emitted a complaint */
220 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
221 2 : exit(1);
222 : }
223 : }
224 :
225 40 : if (optind >= argc)
226 : {
227 2 : pg_log_error("no input directories specified");
228 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
229 2 : exit(1);
230 : }
231 :
232 38 : if (opt.output == NULL)
233 2 : pg_fatal("no output directory specified");
234 :
235 : /* If no manifest is needed, no checksums are needed, either. */
236 36 : if (opt.no_manifest)
237 2 : opt.manifest_checksums = CHECKSUM_TYPE_NONE;
238 :
239 : /* Check that the platform supports the requested copy method. */
240 36 : if (opt.copy_method == COPY_METHOD_CLONE)
241 : {
242 : #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
243 : (defined(__linux__) && defined(FICLONE))
244 :
245 0 : if (opt.dry_run)
246 0 : pg_log_debug("would use cloning to copy files");
247 : else
248 0 : pg_log_debug("will use cloning to copy files");
249 :
250 : #else
251 : pg_fatal("file cloning not supported on this platform");
252 : #endif
253 : }
254 36 : else if (opt.copy_method == COPY_METHOD_COPY_FILE_RANGE)
255 : {
256 : #if defined(HAVE_COPY_FILE_RANGE)
257 :
258 0 : if (opt.dry_run)
259 0 : pg_log_debug("would use copy_file_range to copy blocks");
260 : else
261 0 : pg_log_debug("will use copy_file_range to copy blocks");
262 :
263 : #else
264 : pg_fatal("copy_file_range not supported on this platform");
265 : #endif
266 : }
267 :
268 : /* Read the server version from the final backup. */
269 36 : version = read_pg_version_file(argv[argc - 1]);
270 :
271 : /* Sanity-check control files. */
272 36 : n_backups = argc - optind;
273 36 : system_identifier = check_control_files(n_backups, argv + optind);
274 :
275 : /* Sanity-check backup_label files, and get the contents of the last one. */
276 34 : last_backup_label = check_backup_label_files(n_backups, argv + optind);
277 :
278 : /*
279 : * We'll need the pathnames to the prior backups. By "prior" we mean all
280 : * but the last one listed on the command line.
281 : */
282 24 : n_prior_backups = argc - optind - 1;
283 24 : prior_backup_dirs = argv + optind;
284 :
285 : /* Load backup manifests. */
286 24 : manifests = load_backup_manifests(n_backups, prior_backup_dirs);
287 :
288 : /*
289 : * Validate the manifest system identifier against the backup system
290 : * identifier.
291 : */
292 68 : for (i = 0; i < n_backups; i++)
293 : {
294 46 : if (manifests[i] &&
295 46 : manifests[i]->system_identifier != system_identifier)
296 : {
297 : char *controlpath;
298 :
299 2 : controlpath = psprintf("%s/%s", prior_backup_dirs[i], "global/pg_control");
300 :
301 2 : pg_fatal("%s: manifest system identifier is %llu, but control file has %llu",
302 : controlpath,
303 : (unsigned long long) manifests[i]->system_identifier,
304 : (unsigned long long) system_identifier);
305 : }
306 : }
307 :
308 : /* Figure out which tablespaces are going to be included in the output. */
309 22 : last_input_dir = argv[argc - 1];
310 22 : check_input_dir_permissions(last_input_dir);
311 22 : tablespaces = scan_for_existing_tablespaces(last_input_dir, &opt);
312 :
313 : /*
314 : * Create output directories.
315 : *
316 : * We create one output directory for the main data directory plus one for
317 : * each non-in-place tablespace. create_output_directory() will arrange
318 : * for those directories to be cleaned up on failure. In-place tablespaces
319 : * aren't handled at this stage because they're located beneath the main
320 : * output directory, and thus the cleanup of that directory will get rid
321 : * of them. Plus, the pg_tblspc directory that needs to contain them
322 : * doesn't exist yet.
323 : */
324 22 : atexit(cleanup_directories_atexit);
325 22 : create_output_directory(opt.output, &opt);
326 24 : for (ts = tablespaces; ts != NULL; ts = ts->next)
327 2 : if (!ts->in_place)
328 2 : create_output_directory(ts->new_dir, &opt);
329 :
330 : /* If we need to write a backup_manifest, prepare to do so. */
331 22 : if (!opt.dry_run && !opt.no_manifest)
332 : {
333 20 : mwriter = create_manifest_writer(opt.output, system_identifier);
334 :
335 : /*
336 : * Verify that we have a backup manifest for the final backup; else we
337 : * won't have the WAL ranges for the resulting manifest.
338 : */
339 20 : if (manifests[n_prior_backups] == NULL)
340 0 : pg_fatal("can't generate a manifest because no manifest is available for the final input backup");
341 : }
342 : else
343 2 : mwriter = NULL;
344 :
345 : /* Write backup label into output directory. */
346 22 : if (opt.dry_run)
347 0 : pg_log_debug("would generate \"%s/backup_label\"", opt.output);
348 : else
349 : {
350 22 : pg_log_debug("generating \"%s/backup_label\"", opt.output);
351 22 : last_backup_label->cursor = 0;
352 22 : write_backup_label(opt.output, last_backup_label,
353 : opt.manifest_checksums, mwriter);
354 : }
355 :
356 : /* Process everything that's not part of a user-defined tablespace. */
357 22 : pg_log_debug("processing backup directory \"%s\"", last_input_dir);
358 22 : process_directory_recursively(InvalidOid, last_input_dir, opt.output,
359 : NULL, n_prior_backups, prior_backup_dirs,
360 : manifests, mwriter, &opt);
361 :
362 : /* Process user-defined tablespaces. */
363 24 : for (ts = tablespaces; ts != NULL; ts = ts->next)
364 : {
365 2 : pg_log_debug("processing tablespace directory \"%s\"", ts->old_dir);
366 :
367 : /*
368 : * If it's a normal tablespace, we need to set up a symbolic link from
369 : * pg_tblspc/${OID} to the target directory; if it's an in-place
370 : * tablespace, we need to create a directory at pg_tblspc/${OID}.
371 : */
372 2 : if (!ts->in_place)
373 : {
374 : char linkpath[MAXPGPATH];
375 :
376 2 : snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output,
377 : ts->oid);
378 :
379 2 : if (opt.dry_run)
380 0 : pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
381 : linkpath, ts->new_dir);
382 : else
383 : {
384 2 : pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
385 : linkpath, ts->new_dir);
386 2 : if (symlink(ts->new_dir, linkpath) != 0)
387 0 : pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
388 : linkpath, ts->new_dir);
389 : }
390 : }
391 : else
392 : {
393 0 : if (opt.dry_run)
394 0 : pg_log_debug("would create directory \"%s\"", ts->new_dir);
395 : else
396 : {
397 0 : pg_log_debug("creating directory \"%s\"", ts->new_dir);
398 0 : if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
399 0 : pg_fatal("could not create directory \"%s\": %m",
400 : ts->new_dir);
401 : }
402 : }
403 :
404 : /* OK, now handle the directory contents. */
405 2 : process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
406 : NULL, n_prior_backups, prior_backup_dirs,
407 : manifests, mwriter, &opt);
408 : }
409 :
410 : /* Finalize the backup_manifest, if we're generating one. */
411 22 : if (mwriter != NULL)
412 20 : finalize_manifest(mwriter,
413 20 : manifests[n_prior_backups]->first_wal_range);
414 :
415 : /* fsync that output directory unless we've been told not to do so */
416 22 : if (!opt.no_sync)
417 : {
418 16 : if (opt.dry_run)
419 0 : pg_log_debug("would recursively fsync \"%s\"", opt.output);
420 : else
421 : {
422 16 : pg_log_debug("recursively fsyncing \"%s\"", opt.output);
423 16 : sync_pgdata(opt.output, version * 10000, opt.sync_method);
424 : }
425 : }
426 :
427 : /* It's a success, so don't remove the output directories. */
428 22 : reset_directory_cleanup_list();
429 22 : exit(0);
430 : }
431 :
432 : /*
433 : * Process the option argument for the -T, --tablespace-mapping switch.
434 : */
435 : static void
436 2 : add_tablespace_mapping(cb_options *opt, char *arg)
437 : {
438 2 : cb_tablespace_mapping *tsmap = pg_malloc0(sizeof(cb_tablespace_mapping));
439 : char *dst;
440 : char *dst_ptr;
441 : char *arg_ptr;
442 :
443 : /*
444 : * Basically, we just want to copy everything before the equals sign to
445 : * tsmap->old_dir and everything afterwards to tsmap->new_dir, but if
446 : * there's more or less than one equals sign, that's an error, and if
447 : * there's an equals sign preceded by a backslash, don't treat it as a
448 : * field separator but instead copy a literal equals sign.
449 : */
450 2 : dst_ptr = dst = tsmap->old_dir;
451 100 : for (arg_ptr = arg; *arg_ptr != '\0'; arg_ptr++)
452 : {
453 98 : if (dst_ptr - dst >= MAXPGPATH)
454 0 : pg_fatal("directory name too long");
455 :
456 98 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
457 : ; /* skip backslash escaping = */
458 98 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
459 : {
460 2 : if (tsmap->new_dir[0] != '\0')
461 0 : pg_fatal("multiple \"=\" signs in tablespace mapping");
462 : else
463 2 : dst = dst_ptr = tsmap->new_dir;
464 : }
465 : else
466 96 : *dst_ptr++ = *arg_ptr;
467 : }
468 2 : if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
469 0 : pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
470 :
471 : /*
472 : * All tablespaces are created with absolute directories, so specifying a
473 : * non-absolute path here would never match, possibly confusing users.
474 : *
475 : * In contrast to pg_basebackup, both the old and new directories are on
476 : * the local machine, so the local machine's definition of an absolute
477 : * path is the only relevant one.
478 : */
479 2 : if (!is_absolute_path(tsmap->old_dir))
480 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
481 : tsmap->old_dir);
482 :
483 2 : if (!is_absolute_path(tsmap->new_dir))
484 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
485 : tsmap->new_dir);
486 :
487 : /* Canonicalize paths to avoid spurious failures when comparing. */
488 2 : canonicalize_path(tsmap->old_dir);
489 2 : canonicalize_path(tsmap->new_dir);
490 :
491 : /* Add it to the list. */
492 2 : tsmap->next = opt->tsmappings;
493 2 : opt->tsmappings = tsmap;
494 2 : }
495 :
496 : /*
497 : * Check that the backup_label files form a coherent backup chain, and return
498 : * the contents of the backup_label file from the latest backup.
499 : */
500 : static StringInfo
501 34 : check_backup_label_files(int n_backups, char **backup_dirs)
502 : {
503 34 : StringInfo buf = makeStringInfo();
504 34 : StringInfo lastbuf = buf;
505 : int i;
506 34 : TimeLineID check_tli = 0;
507 34 : XLogRecPtr check_lsn = InvalidXLogRecPtr;
508 :
509 : /* Try to read each backup_label file in turn, last to first. */
510 90 : for (i = n_backups - 1; i >= 0; --i)
511 : {
512 : char pathbuf[MAXPGPATH];
513 : int fd;
514 : TimeLineID start_tli;
515 : TimeLineID previous_tli;
516 : XLogRecPtr start_lsn;
517 : XLogRecPtr previous_lsn;
518 :
519 : /* Open the backup_label file. */
520 66 : snprintf(pathbuf, MAXPGPATH, "%s/backup_label", backup_dirs[i]);
521 66 : pg_log_debug("reading \"%s\"", pathbuf);
522 66 : if ((fd = open(pathbuf, O_RDONLY, 0)) < 0)
523 0 : pg_fatal("could not open file \"%s\": %m", pathbuf);
524 :
525 : /*
526 : * Slurp the whole file into memory.
527 : *
528 : * The exact size limit that we impose here doesn't really matter --
529 : * most of what's supposed to be in the file is fixed size and quite
530 : * short. However, the length of the backup_label is limited (at least
531 : * by some parts of the code) to MAXPGPATH, so include that value in
532 : * the maximum length that we tolerate.
533 : */
534 66 : slurp_file(fd, pathbuf, buf, 10000 + MAXPGPATH);
535 :
536 : /* Close the file. */
537 66 : if (close(fd) != 0)
538 0 : pg_fatal("could not close file \"%s\": %m", pathbuf);
539 :
540 : /* Parse the file contents. */
541 66 : parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
542 : &previous_tli, &previous_lsn);
543 :
544 : /*
545 : * Sanity checks.
546 : *
547 : * XXX. It's actually not required that start_lsn == check_lsn. It
548 : * would be OK if start_lsn > check_lsn provided that start_lsn is
549 : * less than or equal to the relevant switchpoint. But at the moment
550 : * we don't have that information.
551 : */
552 66 : if (i > 0 && previous_tli == 0)
553 2 : pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
554 : backup_dirs[i]);
555 64 : if (i == 0 && previous_tli != 0)
556 2 : pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
557 : backup_dirs[i]);
558 62 : if (i < n_backups - 1 && start_tli != check_tli)
559 0 : pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
560 : backup_dirs[i], start_tli, check_tli);
561 62 : if (i < n_backups - 1 && start_lsn != check_lsn)
562 6 : pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
563 : backup_dirs[i],
564 : LSN_FORMAT_ARGS(start_lsn),
565 : LSN_FORMAT_ARGS(check_lsn));
566 56 : check_tli = previous_tli;
567 56 : check_lsn = previous_lsn;
568 :
569 : /*
570 : * The last backup label in the chain needs to be saved for later use,
571 : * while the others are only needed within this loop.
572 : */
573 56 : if (lastbuf == buf)
574 32 : buf = makeStringInfo();
575 : else
576 24 : resetStringInfo(buf);
577 : }
578 :
579 : /* Free memory that we don't need any more. */
580 24 : if (lastbuf != buf)
581 24 : destroyStringInfo(buf);
582 :
583 : /*
584 : * Return the data from the first backup_info that we read (which is the
585 : * backup_label from the last directory specified on the command line).
586 : */
587 24 : return lastbuf;
588 : }
589 :
590 : /*
591 : * Sanity check control files and return system_identifier.
592 : */
593 : static uint64
594 36 : check_control_files(int n_backups, char **backup_dirs)
595 : {
596 : int i;
597 36 : uint64 system_identifier = 0; /* placate compiler */
598 36 : uint32 data_checksum_version = 0; /* placate compiler */
599 36 : bool data_checksum_mismatch = false;
600 :
601 : /* Try to read each control file in turn, last to first. */
602 108 : for (i = n_backups - 1; i >= 0; --i)
603 : {
604 : ControlFileData *control_file;
605 : bool crc_ok;
606 : char *controlpath;
607 :
608 74 : controlpath = psprintf("%s/%s", backup_dirs[i], "global/pg_control");
609 74 : pg_log_debug("reading \"%s\"", controlpath);
610 74 : control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
611 :
612 : /* Control file contents not meaningful if CRC is bad. */
613 74 : if (!crc_ok)
614 0 : pg_fatal("%s: CRC is incorrect", controlpath);
615 :
616 : /* Can't interpret control file if not current version. */
617 74 : if (control_file->pg_control_version != PG_CONTROL_VERSION)
618 0 : pg_fatal("%s: unexpected control file version",
619 : controlpath);
620 :
621 : /* System identifiers should all match. */
622 74 : if (i == n_backups - 1)
623 36 : system_identifier = control_file->system_identifier;
624 38 : else if (system_identifier != control_file->system_identifier)
625 2 : pg_fatal("%s: expected system identifier %llu, but found %llu",
626 : controlpath, (unsigned long long) system_identifier,
627 : (unsigned long long) control_file->system_identifier);
628 :
629 : /*
630 : * Detect checksum mismatches, but only if the last backup in the
631 : * chain has checksums enabled.
632 : */
633 72 : if (i == n_backups - 1)
634 36 : data_checksum_version = control_file->data_checksum_version;
635 36 : else if (data_checksum_version != 0 &&
636 0 : data_checksum_version != control_file->data_checksum_version)
637 0 : data_checksum_mismatch = true;
638 :
639 : /* Release memory. */
640 72 : pfree(control_file);
641 72 : pfree(controlpath);
642 : }
643 :
644 : /*
645 : * If debug output is enabled, make a note of the system identifier that
646 : * we found in all of the relevant control files.
647 : */
648 34 : pg_log_debug("system identifier is %llu",
649 : (unsigned long long) system_identifier);
650 :
651 : /*
652 : * Warn the user if not all backups are in the same state with regards to
653 : * checksums.
654 : */
655 34 : if (data_checksum_mismatch)
656 : {
657 0 : pg_log_warning("only some backups have checksums enabled");
658 0 : pg_log_warning_hint("disable, and optionally reenable, checksums on the output directory to avoid failures");
659 : }
660 :
661 34 : return system_identifier;
662 : }
663 :
664 : /*
665 : * Set default permissions for new files and directories based on the
666 : * permissions of the given directory. The intent here is that the output
667 : * directory should use the same permissions scheme as the final input
668 : * directory.
669 : */
670 : static void
671 22 : check_input_dir_permissions(char *dir)
672 : {
673 : struct stat st;
674 :
675 22 : if (stat(dir, &st) != 0)
676 0 : pg_fatal("could not stat file \"%s\": %m", dir);
677 :
678 22 : SetDataDirectoryCreatePerm(st.st_mode);
679 22 : }
680 :
681 : /*
682 : * Clean up output directories before exiting.
683 : */
684 : static void
685 22 : cleanup_directories_atexit(void)
686 : {
687 22 : while (cleanup_dir_list != NULL)
688 : {
689 0 : cb_cleanup_dir *dir = cleanup_dir_list;
690 :
691 0 : if (dir->rmtopdir)
692 : {
693 0 : pg_log_info("removing output directory \"%s\"", dir->target_path);
694 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
695 0 : pg_log_error("failed to remove output directory");
696 : }
697 : else
698 : {
699 0 : pg_log_info("removing contents of output directory \"%s\"",
700 : dir->target_path);
701 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
702 0 : pg_log_error("failed to remove contents of output directory");
703 : }
704 :
705 0 : cleanup_dir_list = cleanup_dir_list->next;
706 0 : pfree(dir);
707 : }
708 22 : }
709 :
710 : /*
711 : * Create the named output directory, unless it already exists or we're in
712 : * dry-run mode. If it already exists but is not empty, that's a fatal error.
713 : *
714 : * Adds the created directory to the list of directories to be cleaned up
715 : * at process exit.
716 : */
717 : static void
718 24 : create_output_directory(char *dirname, cb_options *opt)
719 : {
720 24 : switch (pg_check_dir(dirname))
721 : {
722 24 : case 0:
723 24 : if (opt->dry_run)
724 : {
725 0 : pg_log_debug("would create directory \"%s\"", dirname);
726 0 : return;
727 : }
728 24 : pg_log_debug("creating directory \"%s\"", dirname);
729 24 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
730 0 : pg_fatal("could not create directory \"%s\": %m", dirname);
731 24 : remember_to_cleanup_directory(dirname, true);
732 24 : break;
733 :
734 0 : case 1:
735 0 : pg_log_debug("using existing directory \"%s\"", dirname);
736 0 : remember_to_cleanup_directory(dirname, false);
737 0 : break;
738 :
739 0 : case 2:
740 : case 3:
741 : case 4:
742 0 : pg_fatal("directory \"%s\" exists but is not empty", dirname);
743 :
744 0 : case -1:
745 0 : pg_fatal("could not access directory \"%s\": %m", dirname);
746 : }
747 : }
748 :
749 : /*
750 : * help
751 : *
752 : * Prints help page for the program
753 : *
754 : * progname: the name of the executed program, such as "pg_combinebackup"
755 : */
756 : static void
757 2 : help(const char *progname)
758 : {
759 2 : printf(_("%s reconstructs full backups from incrementals.\n\n"), progname);
760 2 : printf(_("Usage:\n"));
761 2 : printf(_(" %s [OPTION]... DIRECTORY...\n"), progname);
762 2 : printf(_("\nOptions:\n"));
763 2 : printf(_(" -d, --debug generate lots of debugging output\n"));
764 2 : printf(_(" -n, --dry-run do not actually do anything\n"));
765 2 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
766 2 : printf(_(" -o, --output=DIRECTORY output directory\n"));
767 2 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
768 : " relocate tablespace in OLDDIR to NEWDIR\n"));
769 2 : printf(_(" --clone clone (reflink) instead of copying files\n"));
770 2 : printf(_(" --copy copy files (default)\n"));
771 2 : printf(_(" --copy-file-range copy using copy_file_range() syscall\n"));
772 2 : printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
773 : " use algorithm for manifest checksums\n"));
774 2 : printf(_(" --no-manifest suppress generation of backup manifest\n"));
775 2 : printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
776 2 : printf(_(" -V, --version output version information, then exit\n"));
777 2 : printf(_(" -?, --help show this help, then exit\n"));
778 :
779 2 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
780 2 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
781 2 : }
782 :
783 : /*
784 : * Try to parse a string as a non-zero OID without leading zeroes.
785 : *
786 : * If it works, return true and set *result to the answer, else return false.
787 : */
788 : static bool
789 4 : parse_oid(char *s, Oid *result)
790 : {
791 : Oid oid;
792 : char *ep;
793 :
794 4 : errno = 0;
795 4 : oid = strtoul(s, &ep, 10);
796 4 : if (errno != 0 || *ep != '\0' || oid < 1 || oid > PG_UINT32_MAX)
797 0 : return false;
798 :
799 4 : *result = oid;
800 4 : return true;
801 : }
802 :
803 : /*
804 : * Copy files from the input directory to the output directory, reconstructing
805 : * full files from incremental files as required.
806 : *
807 : * If processing is a user-defined tablespace, the tsoid should be the OID
808 : * of that tablespace and input_directory and output_directory should be the
809 : * toplevel input and output directories for that tablespace. Otherwise,
810 : * tsoid should be InvalidOid and input_directory and output_directory should
811 : * be the main input and output directories.
812 : *
813 : * relative_path is the path beneath the given input and output directories
814 : * that we are currently processing. If NULL, it indicates that we're
815 : * processing the input and output directories themselves.
816 : *
817 : * n_prior_backups is the number of prior backups that we have available.
818 : * This doesn't count the very last backup, which is referenced by
819 : * output_directory, just the older ones. prior_backup_dirs is an array of
820 : * the locations of those previous backups.
821 : */
822 : static void
823 604 : process_directory_recursively(Oid tsoid,
824 : char *input_directory,
825 : char *output_directory,
826 : char *relative_path,
827 : int n_prior_backups,
828 : char **prior_backup_dirs,
829 : manifest_data **manifests,
830 : manifest_writer *mwriter,
831 : cb_options *opt)
832 : {
833 : char ifulldir[MAXPGPATH];
834 : char ofulldir[MAXPGPATH];
835 : char manifest_prefix[MAXPGPATH];
836 : DIR *dir;
837 : struct dirent *de;
838 604 : bool is_pg_tblspc = false;
839 604 : bool is_pg_wal = false;
840 604 : bool is_incremental_dir = false;
841 604 : manifest_data *latest_manifest = manifests[n_prior_backups];
842 : pg_checksum_type checksum_type;
843 :
844 : /*
845 : * Classify this directory.
846 : *
847 : * We set is_pg_tblspc only for the toplevel pg_tblspc directory, because
848 : * the symlinks in that specific directory require special handling.
849 : *
850 : * We set is_pg_wal for the toplevel WAL directory and all of its
851 : * subdirectories, because those files are not included in the backup
852 : * manifest and hence need special treatement. (Since incremental backup
853 : * does not exist in pre-v10 versions, we don't have to worry about the
854 : * old pg_xlog naming.)
855 : *
856 : * We set is_incremental_dir for directories that can contain incremental
857 : * files requiring reconstruction. If such files occur outside these
858 : * directories, we want to just copy them straight to the output
859 : * directory. This is to protect against a user creating a file with a
860 : * strange name like INCREMENTAL.config and then complaining that
861 : * incremental backups don't work properly. The test here is a bit tricky:
862 : * incremental files occur in subdirectories of base, in pg_global itself,
863 : * and in subdirectories of pg_tblspc only if in-place tablespaces are
864 : * used.
865 : */
866 604 : if (OidIsValid(tsoid))
867 6 : is_incremental_dir = true;
868 598 : else if (relative_path != NULL)
869 : {
870 576 : is_pg_tblspc = strcmp(relative_path, "pg_tblspc") == 0;
871 1130 : is_pg_wal = (strcmp(relative_path, "pg_wal") == 0 ||
872 554 : strncmp(relative_path, "pg_wal/", 7) == 0);
873 576 : is_incremental_dir = strncmp(relative_path, "base/", 5) == 0 ||
874 1060 : strcmp(relative_path, "global") == 0 ||
875 484 : strncmp(relative_path, "pg_tblspc/", 10) == 0;
876 : }
877 :
878 : /*
879 : * If we're under pg_wal, then we don't need checksums, because these
880 : * files aren't included in the backup manifest. Otherwise use whatever
881 : * type of checksum is configured.
882 : */
883 604 : if (!is_pg_wal)
884 538 : checksum_type = opt->manifest_checksums;
885 : else
886 66 : checksum_type = CHECKSUM_TYPE_NONE;
887 :
888 : /*
889 : * Append the relative path to the input and output directories, and
890 : * figure out the appropriate prefix to add to files in this directory
891 : * when looking them up in a backup manifest.
892 : */
893 604 : if (relative_path == NULL)
894 : {
895 24 : strlcpy(ifulldir, input_directory, MAXPGPATH);
896 24 : strlcpy(ofulldir, output_directory, MAXPGPATH);
897 24 : if (OidIsValid(tsoid))
898 2 : snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/", tsoid);
899 : else
900 22 : manifest_prefix[0] = '\0';
901 : }
902 : else
903 : {
904 580 : snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
905 : relative_path);
906 580 : snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
907 : relative_path);
908 580 : if (OidIsValid(tsoid))
909 4 : snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/",
910 : tsoid, relative_path);
911 : else
912 576 : snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
913 : }
914 :
915 : /*
916 : * Toplevel output directories have already been created by the time this
917 : * function is called, but any subdirectories are our responsibility.
918 : */
919 604 : if (relative_path != NULL)
920 : {
921 580 : if (opt->dry_run)
922 0 : pg_log_debug("would create directory \"%s\"", ofulldir);
923 : else
924 : {
925 580 : pg_log_debug("creating directory \"%s\"", ofulldir);
926 580 : if (mkdir(ofulldir, pg_dir_create_mode) == -1)
927 0 : pg_fatal("could not create directory \"%s\": %m", ofulldir);
928 : }
929 : }
930 :
931 : /* It's time to scan the directory. */
932 604 : if ((dir = opendir(ifulldir)) == NULL)
933 0 : pg_fatal("could not open directory \"%s\": %m", ifulldir);
934 24976 : while (errno = 0, (de = readdir(dir)) != NULL)
935 : {
936 : PGFileType type;
937 : char ifullpath[MAXPGPATH];
938 : char ofullpath[MAXPGPATH];
939 : char manifest_path[MAXPGPATH];
940 24372 : Oid oid = InvalidOid;
941 24372 : int checksum_length = 0;
942 24372 : uint8 *checksum_payload = NULL;
943 : pg_checksum_context checksum_ctx;
944 :
945 : /* Ignore "." and ".." entries. */
946 24372 : if (strcmp(de->d_name, ".") == 0 ||
947 23768 : strcmp(de->d_name, "..") == 0)
948 1834 : continue;
949 :
950 : /* Construct input path. */
951 23164 : snprintf(ifullpath, MAXPGPATH, "%s/%s", ifulldir, de->d_name);
952 :
953 : /* Figure out what kind of directory entry this is. */
954 23164 : type = get_dirent_type(ifullpath, de, false, PG_LOG_ERROR);
955 23164 : if (type == PGFILETYPE_ERROR)
956 0 : exit(1);
957 :
958 : /*
959 : * If we're processing pg_tblspc, then check whether the filename
960 : * looks like it could be a tablespace OID. If so, and if the
961 : * directory entry is a symbolic link or a directory, skip it.
962 : *
963 : * Our goal here is to ignore anything that would have been considered
964 : * by scan_for_existing_tablespaces to be a tablespace.
965 : */
966 23164 : if (is_pg_tblspc && parse_oid(de->d_name, &oid) &&
967 0 : (type == PGFILETYPE_LNK || type == PGFILETYPE_DIR))
968 2 : continue;
969 :
970 : /* If it's a directory, recurse. */
971 23162 : if (type == PGFILETYPE_DIR)
972 : {
973 : char new_relative_path[MAXPGPATH];
974 :
975 : /* Append new pathname component to relative path. */
976 580 : if (relative_path == NULL)
977 376 : strlcpy(new_relative_path, de->d_name, MAXPGPATH);
978 : else
979 204 : snprintf(new_relative_path, MAXPGPATH, "%s/%s", relative_path,
980 204 : de->d_name);
981 :
982 : /* And recurse. */
983 580 : process_directory_recursively(tsoid,
984 : input_directory, output_directory,
985 : new_relative_path,
986 : n_prior_backups, prior_backup_dirs,
987 : manifests, mwriter, opt);
988 580 : continue;
989 : }
990 :
991 : /* Skip anything that's not a regular file. */
992 22582 : if (type != PGFILETYPE_REG)
993 : {
994 0 : if (type == PGFILETYPE_LNK)
995 0 : pg_log_warning("skipping symbolic link \"%s\"", ifullpath);
996 : else
997 0 : pg_log_warning("skipping special file \"%s\"", ifullpath);
998 0 : continue;
999 : }
1000 :
1001 : /*
1002 : * Skip the backup_label and backup_manifest files; they require
1003 : * special handling and are handled elsewhere.
1004 : */
1005 22582 : if (relative_path == NULL &&
1006 164 : (strcmp(de->d_name, "backup_label") == 0 ||
1007 142 : strcmp(de->d_name, "backup_manifest") == 0))
1008 44 : continue;
1009 :
1010 : /*
1011 : * If it's an incremental file, hand it off to the reconstruction
1012 : * code, which will figure out what to do.
1013 : */
1014 22538 : if (is_incremental_dir &&
1015 22302 : strncmp(de->d_name, INCREMENTAL_PREFIX,
1016 : INCREMENTAL_PREFIX_LENGTH) == 0)
1017 : {
1018 : /* Output path should not include "INCREMENTAL." prefix. */
1019 10638 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
1020 10638 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1021 :
1022 :
1023 : /* Manifest path likewise omits incremental prefix. */
1024 10638 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1025 10638 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1026 :
1027 : /* Reconstruction logic will do the rest. */
1028 10638 : reconstruct_from_incremental_file(ifullpath, ofullpath,
1029 : manifest_prefix,
1030 10638 : de->d_name + INCREMENTAL_PREFIX_LENGTH,
1031 : n_prior_backups,
1032 : prior_backup_dirs,
1033 : manifests,
1034 : manifest_path,
1035 : checksum_type,
1036 : &checksum_length,
1037 : &checksum_payload,
1038 : opt->copy_method,
1039 10638 : opt->debug,
1040 10638 : opt->dry_run);
1041 : }
1042 : else
1043 : {
1044 : /* Construct the path that the backup_manifest will use. */
1045 11900 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1046 11900 : de->d_name);
1047 :
1048 : /*
1049 : * It's not an incremental file, so we need to copy the entire
1050 : * file to the output directory.
1051 : *
1052 : * If a checksum of the required type already exists in the
1053 : * backup_manifest for the final input directory, we can save some
1054 : * work by reusing that checksum instead of computing a new one.
1055 : */
1056 11900 : if (checksum_type != CHECKSUM_TYPE_NONE &&
1057 : latest_manifest != NULL)
1058 : {
1059 : manifest_file *mfile;
1060 :
1061 8012 : mfile = manifest_files_lookup(latest_manifest->files,
1062 : manifest_path);
1063 8012 : if (mfile == NULL)
1064 : {
1065 : char *bmpath;
1066 :
1067 : /*
1068 : * The directory is out of sync with the backup_manifest,
1069 : * so emit a warning.
1070 : */
1071 0 : bmpath = psprintf("%s/%s", input_directory,
1072 : "backup_manifest");
1073 0 : pg_log_warning("\"%s\" contains no entry for \"%s\"",
1074 : bmpath, manifest_path);
1075 0 : pfree(bmpath);
1076 : }
1077 8012 : else if (mfile->checksum_type == checksum_type)
1078 : {
1079 6082 : checksum_length = mfile->checksum_length;
1080 6082 : checksum_payload = mfile->checksum_payload;
1081 : }
1082 : }
1083 :
1084 : /*
1085 : * If we're reusing a checksum, then we don't need copy_file() to
1086 : * compute one for us, but otherwise, it needs to compute whatever
1087 : * type of checksum we need.
1088 : */
1089 11900 : if (checksum_length != 0)
1090 6082 : pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_NONE);
1091 : else
1092 5818 : pg_checksum_init(&checksum_ctx, checksum_type);
1093 :
1094 : /* Actually copy the file. */
1095 11900 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
1096 11900 : copy_file(ifullpath, ofullpath, &checksum_ctx,
1097 11900 : opt->copy_method, opt->dry_run);
1098 :
1099 : /*
1100 : * If copy_file() performed a checksum calculation for us, then
1101 : * save the results (except in dry-run mode, when there's no
1102 : * point).
1103 : */
1104 11900 : if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
1105 : {
1106 1930 : checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
1107 1930 : checksum_length = pg_checksum_final(&checksum_ctx,
1108 : checksum_payload);
1109 : }
1110 : }
1111 :
1112 : /* Generate manifest entry, if needed. */
1113 22538 : if (mwriter != NULL)
1114 : {
1115 : struct stat sb;
1116 :
1117 : /*
1118 : * In order to generate a manifest entry, we need the file size
1119 : * and mtime. We have no way to know the correct mtime except to
1120 : * stat() the file, so just do that and get the size as well.
1121 : *
1122 : * If we didn't need the mtime here, we could try to obtain the
1123 : * file size from the reconstruction or file copy process above,
1124 : * although that is actually not convenient in all cases. If we
1125 : * write the file ourselves then clearly we can keep a count of
1126 : * bytes, but if we use something like CopyFile() then it's
1127 : * trickier. Since we have to stat() anyway to get the mtime,
1128 : * there's no point in worrying about it.
1129 : */
1130 20606 : if (stat(ofullpath, &sb) < 0)
1131 0 : pg_fatal("could not stat file \"%s\": %m", ofullpath);
1132 :
1133 : /* OK, now do the work. */
1134 20606 : add_file_to_manifest(mwriter, manifest_path,
1135 20606 : sb.st_size, sb.st_mtime,
1136 : checksum_type, checksum_length,
1137 : checksum_payload);
1138 : }
1139 :
1140 : /* Avoid leaking memory. */
1141 22538 : if (checksum_payload != NULL)
1142 18650 : pfree(checksum_payload);
1143 : }
1144 :
1145 604 : closedir(dir);
1146 604 : }
1147 :
1148 : /*
1149 : * Read the version number from PG_VERSION and convert it to the usual server
1150 : * version number format. (e.g. If PG_VERSION contains "14\n" this function
1151 : * will return 140000)
1152 : */
1153 : static int
1154 36 : read_pg_version_file(char *directory)
1155 : {
1156 : char filename[MAXPGPATH];
1157 : StringInfoData buf;
1158 : int fd;
1159 : int version;
1160 : char *ep;
1161 :
1162 : /* Construct pathname. */
1163 36 : snprintf(filename, MAXPGPATH, "%s/PG_VERSION", directory);
1164 :
1165 : /* Open file. */
1166 36 : if ((fd = open(filename, O_RDONLY, 0)) < 0)
1167 0 : pg_fatal("could not open file \"%s\": %m", filename);
1168 :
1169 : /* Read into memory. Length limit of 128 should be more than generous. */
1170 36 : initStringInfo(&buf);
1171 36 : slurp_file(fd, filename, &buf, 128);
1172 :
1173 : /* Close the file. */
1174 36 : if (close(fd) != 0)
1175 0 : pg_fatal("could not close file \"%s\": %m", filename);
1176 :
1177 : /* Convert to integer. */
1178 36 : errno = 0;
1179 36 : version = strtoul(buf.data, &ep, 10);
1180 36 : if (errno != 0 || *ep != '\n')
1181 : {
1182 : /*
1183 : * Incremental backup is not relevant to very old server versions that
1184 : * used multi-part version number (e.g. 9.6, or 8.4). So if we see
1185 : * what looks like the beginning of such a version number, just bail
1186 : * out.
1187 : */
1188 0 : if (version < 10 && *ep == '.')
1189 0 : pg_fatal("%s: server version too old", filename);
1190 0 : pg_fatal("%s: could not parse version number", filename);
1191 : }
1192 :
1193 : /* Debugging output. */
1194 36 : pg_log_debug("read server version %d from file \"%s\"", version, filename);
1195 :
1196 : /* Release memory and return result. */
1197 36 : pfree(buf.data);
1198 36 : return version * 10000;
1199 : }
1200 :
1201 : /*
1202 : * Add a directory to the list of output directories to clean up.
1203 : */
1204 : static void
1205 24 : remember_to_cleanup_directory(char *target_path, bool rmtopdir)
1206 : {
1207 24 : cb_cleanup_dir *dir = pg_malloc(sizeof(cb_cleanup_dir));
1208 :
1209 24 : dir->target_path = target_path;
1210 24 : dir->rmtopdir = rmtopdir;
1211 24 : dir->next = cleanup_dir_list;
1212 24 : cleanup_dir_list = dir;
1213 24 : }
1214 :
1215 : /*
1216 : * Empty out the list of directories scheduled for cleanup at exit.
1217 : *
1218 : * We want to remove the output directories only on a failure, so call this
1219 : * function when we know that the operation has succeeded.
1220 : *
1221 : * Since we only expect this to be called when we're about to exit, we could
1222 : * just set cleanup_dir_list to NULL and be done with it, but we free the
1223 : * memory to be tidy.
1224 : */
1225 : static void
1226 22 : reset_directory_cleanup_list(void)
1227 : {
1228 46 : while (cleanup_dir_list != NULL)
1229 : {
1230 24 : cb_cleanup_dir *dir = cleanup_dir_list;
1231 :
1232 24 : cleanup_dir_list = cleanup_dir_list->next;
1233 24 : pfree(dir);
1234 : }
1235 22 : }
1236 :
1237 : /*
1238 : * Scan the pg_tblspc directory of the final input backup to get a canonical
1239 : * list of what tablespaces are part of the backup.
1240 : *
1241 : * 'pathname' should be the path to the toplevel backup directory for the
1242 : * final backup in the backup chain.
1243 : */
1244 : static cb_tablespace *
1245 22 : scan_for_existing_tablespaces(char *pathname, cb_options *opt)
1246 : {
1247 : char pg_tblspc[MAXPGPATH];
1248 : DIR *dir;
1249 : struct dirent *de;
1250 22 : cb_tablespace *tslist = NULL;
1251 :
1252 22 : snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pathname);
1253 22 : pg_log_debug("scanning \"%s\"", pg_tblspc);
1254 :
1255 22 : if ((dir = opendir(pg_tblspc)) == NULL)
1256 0 : pg_fatal("could not open directory \"%s\": %m", pg_tblspc);
1257 :
1258 68 : while (errno = 0, (de = readdir(dir)) != NULL)
1259 : {
1260 : Oid oid;
1261 : char tblspcdir[MAXPGPATH];
1262 : char link_target[MAXPGPATH];
1263 : int link_length;
1264 : cb_tablespace *ts;
1265 : cb_tablespace *otherts;
1266 : PGFileType type;
1267 :
1268 : /* Silently ignore "." and ".." entries. */
1269 46 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1270 44 : continue;
1271 :
1272 : /* Construct full pathname. */
1273 2 : snprintf(tblspcdir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1274 :
1275 : /* Ignore any file name that doesn't look like a proper OID. */
1276 2 : if (!parse_oid(de->d_name, &oid))
1277 : {
1278 0 : pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
1279 : tblspcdir);
1280 0 : continue;
1281 : }
1282 :
1283 : /* Only symbolic links and directories are tablespaces. */
1284 2 : type = get_dirent_type(tblspcdir, de, false, PG_LOG_ERROR);
1285 2 : if (type == PGFILETYPE_ERROR)
1286 0 : exit(1);
1287 2 : if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
1288 : {
1289 0 : pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
1290 : tblspcdir);
1291 0 : continue;
1292 : }
1293 :
1294 : /* Create a new tablespace object. */
1295 2 : ts = pg_malloc0(sizeof(cb_tablespace));
1296 2 : ts->oid = oid;
1297 :
1298 : /*
1299 : * If it's a link, it's not an in-place tablespace. Otherwise, it must
1300 : * be a directory, and thus an in-place tablespace.
1301 : */
1302 2 : if (type == PGFILETYPE_LNK)
1303 : {
1304 : cb_tablespace_mapping *tsmap;
1305 :
1306 : /* Read the link target. */
1307 2 : link_length = readlink(tblspcdir, link_target, sizeof(link_target));
1308 2 : if (link_length < 0)
1309 0 : pg_fatal("could not read symbolic link \"%s\": %m",
1310 : tblspcdir);
1311 2 : if (link_length >= sizeof(link_target))
1312 0 : pg_fatal("target of symbolic link \"%s\" is too long", tblspcdir);
1313 2 : link_target[link_length] = '\0';
1314 2 : if (!is_absolute_path(link_target))
1315 0 : pg_fatal("target of symbolic link \"%s\" is relative", tblspcdir);
1316 :
1317 : /* Canonicalize the link target. */
1318 2 : canonicalize_path(link_target);
1319 :
1320 : /*
1321 : * Find the corresponding tablespace mapping and copy the relevant
1322 : * details into the new tablespace entry.
1323 : */
1324 2 : for (tsmap = opt->tsmappings; tsmap != NULL; tsmap = tsmap->next)
1325 : {
1326 2 : if (strcmp(tsmap->old_dir, link_target) == 0)
1327 : {
1328 2 : strlcpy(ts->old_dir, tsmap->old_dir, MAXPGPATH);
1329 2 : strlcpy(ts->new_dir, tsmap->new_dir, MAXPGPATH);
1330 2 : ts->in_place = false;
1331 2 : break;
1332 : }
1333 : }
1334 :
1335 : /* Every non-in-place tablespace must be mapped. */
1336 2 : if (tsmap == NULL)
1337 0 : pg_fatal("tablespace at \"%s\" has no tablespace mapping",
1338 : link_target);
1339 : }
1340 : else
1341 : {
1342 : /*
1343 : * For an in-place tablespace, there's no separate directory, so
1344 : * we just record the paths within the data directories.
1345 : */
1346 0 : snprintf(ts->old_dir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1347 0 : snprintf(ts->new_dir, MAXPGPATH, "%s/pg_tblspc/%s", opt->output,
1348 0 : de->d_name);
1349 0 : ts->in_place = true;
1350 : }
1351 :
1352 : /* Tablespaces should not share a directory. */
1353 2 : for (otherts = tslist; otherts != NULL; otherts = otherts->next)
1354 0 : if (strcmp(ts->new_dir, otherts->new_dir) == 0)
1355 0 : pg_fatal("tablespaces with OIDs %u and %u both point at directory \"%s\"",
1356 : otherts->oid, oid, ts->new_dir);
1357 :
1358 : /* Add this tablespace to the list. */
1359 2 : ts->next = tslist;
1360 2 : tslist = ts;
1361 : }
1362 :
1363 22 : if (closedir(dir) != 0)
1364 0 : pg_fatal("could not close directory \"%s\": %m", pg_tblspc);
1365 :
1366 22 : return tslist;
1367 : }
1368 :
1369 : /*
1370 : * Read a file into a StringInfo.
1371 : *
1372 : * fd is used for the actual file I/O, filename for error reporting purposes.
1373 : * A file longer than maxlen is a fatal error.
1374 : */
1375 : static void
1376 102 : slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
1377 : {
1378 : struct stat st;
1379 : ssize_t rb;
1380 :
1381 : /* Check file size, and complain if it's too large. */
1382 102 : if (fstat(fd, &st) != 0)
1383 0 : pg_fatal("could not stat file \"%s\": %m", filename);
1384 102 : if (st.st_size > maxlen)
1385 0 : pg_fatal("file \"%s\" is too large", filename);
1386 :
1387 : /* Make sure we have enough space. */
1388 102 : enlargeStringInfo(buf, st.st_size);
1389 :
1390 : /* Read the data. */
1391 102 : rb = read(fd, &buf->data[buf->len], st.st_size);
1392 :
1393 : /*
1394 : * We don't expect any concurrent changes, so we should read exactly the
1395 : * expected number of bytes.
1396 : */
1397 102 : if (rb != st.st_size)
1398 : {
1399 0 : if (rb < 0)
1400 0 : pg_fatal("could not read file \"%s\": %m", filename);
1401 : else
1402 0 : pg_fatal("could not read file \"%s\": read %zd of %lld",
1403 : filename, rb, (long long int) st.st_size);
1404 : }
1405 :
1406 : /* Adjust buffer length for new data and restore trailing-\0 invariant */
1407 102 : buf->len += rb;
1408 102 : buf->data[buf->len] = '\0';
1409 102 : }
|
