Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_combinebackup.c
4 : * Combine incremental backups with prior backups.
5 : *
6 : * Copyright (c) 2017-2025, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/bin/pg_combinebackup/pg_combinebackup.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres_fe.h"
14 :
15 : #include <dirent.h>
16 : #include <fcntl.h>
17 : #include <limits.h>
18 :
19 : #ifdef HAVE_COPYFILE_H
20 : #include <copyfile.h>
21 : #endif
22 : #ifdef __linux__
23 : #include <sys/ioctl.h>
24 : #include <linux/fs.h>
25 : #endif
26 :
27 : #include "access/xlog_internal.h"
28 : #include "backup_label.h"
29 : #include "common/checksum_helper.h"
30 : #include "common/controldata_utils.h"
31 : #include "common/file_perm.h"
32 : #include "common/file_utils.h"
33 : #include "common/logging.h"
34 : #include "common/relpath.h"
35 : #include "copy_file.h"
36 : #include "fe_utils/option_utils.h"
37 : #include "fe_utils/version.h"
38 : #include "getopt_long.h"
39 : #include "lib/stringinfo.h"
40 : #include "load_manifest.h"
41 : #include "reconstruct.h"
42 : #include "write_manifest.h"
43 :
44 : /* Incremental file naming convention. */
45 : #define INCREMENTAL_PREFIX "INCREMENTAL."
46 : #define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
47 :
48 : /*
49 : * Tracking for directories that need to be removed, or have their contents
50 : * removed, if the operation fails.
51 : */
52 : typedef struct cb_cleanup_dir
53 : {
54 : char *target_path;
55 : bool rmtopdir;
56 : struct cb_cleanup_dir *next;
57 : } cb_cleanup_dir;
58 :
59 : /*
60 : * Stores a tablespace mapping provided using -T, --tablespace-mapping.
61 : */
62 : typedef struct cb_tablespace_mapping
63 : {
64 : char old_dir[MAXPGPATH];
65 : char new_dir[MAXPGPATH];
66 : struct cb_tablespace_mapping *next;
67 : } cb_tablespace_mapping;
68 :
69 : /*
70 : * Stores data parsed from all command-line options.
71 : */
72 : typedef struct cb_options
73 : {
74 : bool debug;
75 : char *output;
76 : bool dry_run;
77 : bool no_sync;
78 : cb_tablespace_mapping *tsmappings;
79 : pg_checksum_type manifest_checksums;
80 : bool no_manifest;
81 : DataDirSyncMethod sync_method;
82 : CopyMethod copy_method;
83 : } cb_options;
84 :
85 : /*
86 : * Data about a tablespace.
87 : *
88 : * Every normal tablespace needs a tablespace mapping, but in-place tablespaces
89 : * don't, so the list of tablespaces can contain more entries than the list of
90 : * tablespace mappings.
91 : */
92 : typedef struct cb_tablespace
93 : {
94 : Oid oid;
95 : bool in_place;
96 : char old_dir[MAXPGPATH];
97 : char new_dir[MAXPGPATH];
98 : struct cb_tablespace *next;
99 : } cb_tablespace;
100 :
101 : /* Directories to be removed if we exit uncleanly. */
102 : static cb_cleanup_dir *cleanup_dir_list = NULL;
103 :
104 : static void add_tablespace_mapping(cb_options *opt, char *arg);
105 : static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
106 : static uint64 check_control_files(int n_backups, char **backup_dirs);
107 : static void check_input_dir_permissions(char *dir);
108 : static void cleanup_directories_atexit(void);
109 : static void create_output_directory(char *dirname, cb_options *opt);
110 : static void help(const char *progname);
111 : static bool parse_oid(char *s, Oid *result);
112 : static void process_directory_recursively(Oid tsoid,
113 : char *input_directory,
114 : char *output_directory,
115 : char *relative_path,
116 : int n_prior_backups,
117 : char **prior_backup_dirs,
118 : manifest_data **manifests,
119 : manifest_writer *mwriter,
120 : cb_options *opt);
121 : static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
122 : static void reset_directory_cleanup_list(void);
123 : static cb_tablespace *scan_for_existing_tablespaces(char *pathname,
124 : cb_options *opt);
125 : static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen);
126 :
127 : /*
128 : * Main program.
129 : */
130 : int
131 50 : main(int argc, char *argv[])
132 : {
133 : static struct option long_options[] = {
134 : {"debug", no_argument, NULL, 'd'},
135 : {"dry-run", no_argument, NULL, 'n'},
136 : {"no-sync", no_argument, NULL, 'N'},
137 : {"output", required_argument, NULL, 'o'},
138 : {"tablespace-mapping", required_argument, NULL, 'T'},
139 : {"link", no_argument, NULL, 'k'},
140 : {"manifest-checksums", required_argument, NULL, 1},
141 : {"no-manifest", no_argument, NULL, 2},
142 : {"sync-method", required_argument, NULL, 3},
143 : {"clone", no_argument, NULL, 4},
144 : {"copy", no_argument, NULL, 5},
145 : {"copy-file-range", no_argument, NULL, 6},
146 : {NULL, 0, NULL, 0}
147 : };
148 :
149 : const char *progname;
150 : char *last_input_dir;
151 : int i;
152 : int optindex;
153 : int c;
154 : int n_backups;
155 : int n_prior_backups;
156 : uint32 version;
157 : uint64 system_identifier;
158 : char **prior_backup_dirs;
159 : cb_options opt;
160 : cb_tablespace *tablespaces;
161 : cb_tablespace *ts;
162 : StringInfo last_backup_label;
163 : manifest_data **manifests;
164 : manifest_writer *mwriter;
165 : char *pgdata;
166 :
167 50 : pg_logging_init(argv[0]);
168 50 : progname = get_progname(argv[0]);
169 50 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
170 50 : handle_help_version_opts(argc, argv, progname, help);
171 :
172 46 : memset(&opt, 0, sizeof(opt));
173 46 : opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
174 46 : opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
175 46 : opt.copy_method = COPY_METHOD_COPY;
176 :
177 : /* process command-line options */
178 144 : while ((c = getopt_long(argc, argv, "dknNo:T:",
179 144 : long_options, &optindex)) != -1)
180 : {
181 100 : switch (c)
182 : {
183 12 : case 'd':
184 12 : opt.debug = true;
185 12 : pg_logging_increase_verbosity();
186 12 : break;
187 2 : case 'k':
188 2 : opt.copy_method = COPY_METHOD_LINK;
189 2 : break;
190 0 : case 'n':
191 0 : opt.dry_run = true;
192 0 : break;
193 6 : case 'N':
194 6 : opt.no_sync = true;
195 6 : break;
196 40 : case 'o':
197 40 : opt.output = optarg;
198 40 : break;
199 2 : case 'T':
200 2 : add_tablespace_mapping(&opt, optarg);
201 2 : break;
202 4 : case 1:
203 4 : if (!pg_checksum_parse_type(optarg,
204 : &opt.manifest_checksums))
205 0 : pg_fatal("unrecognized checksum algorithm: \"%s\"",
206 : optarg);
207 4 : break;
208 2 : case 2:
209 2 : opt.no_manifest = true;
210 2 : break;
211 0 : case 3:
212 0 : if (!parse_sync_method(optarg, &opt.sync_method))
213 0 : exit(1);
214 0 : break;
215 0 : case 4:
216 0 : opt.copy_method = COPY_METHOD_CLONE;
217 0 : break;
218 30 : case 5:
219 30 : opt.copy_method = COPY_METHOD_COPY;
220 30 : break;
221 0 : case 6:
222 0 : opt.copy_method = COPY_METHOD_COPY_FILE_RANGE;
223 0 : break;
224 2 : default:
225 : /* getopt_long already emitted a complaint */
226 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
227 2 : exit(1);
228 : }
229 : }
230 :
231 44 : if (optind >= argc)
232 : {
233 2 : pg_log_error("no input directories specified");
234 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
235 2 : exit(1);
236 : }
237 :
238 42 : if (opt.output == NULL)
239 2 : pg_fatal("no output directory specified");
240 :
241 : /* If no manifest is needed, no checksums are needed, either. */
242 40 : if (opt.no_manifest)
243 2 : opt.manifest_checksums = CHECKSUM_TYPE_NONE;
244 :
245 40 : if (opt.dry_run)
246 0 : pg_log_info("Executing in dry-run mode.\n"
247 : "The target directory will not be modified.");
248 :
249 : /* Check that the platform supports the requested copy method. */
250 40 : if (opt.copy_method == COPY_METHOD_CLONE)
251 : {
252 : #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
253 : (defined(__linux__) && defined(FICLONE))
254 :
255 0 : if (opt.dry_run)
256 0 : pg_log_debug("would use cloning to copy files");
257 : else
258 0 : pg_log_debug("will use cloning to copy files");
259 :
260 : #else
261 : pg_fatal("file cloning not supported on this platform");
262 : #endif
263 : }
264 40 : else if (opt.copy_method == COPY_METHOD_COPY_FILE_RANGE)
265 : {
266 : #if defined(HAVE_COPY_FILE_RANGE)
267 :
268 0 : if (opt.dry_run)
269 0 : pg_log_debug("would use copy_file_range to copy blocks");
270 : else
271 0 : pg_log_debug("will use copy_file_range to copy blocks");
272 :
273 : #else
274 : pg_fatal("copy_file_range not supported on this platform");
275 : #endif
276 : }
277 :
278 : /* Read the server version from the final backup. */
279 40 : pgdata = argv[argc - 1];
280 40 : version = get_pg_version(pgdata, NULL);
281 40 : if (GET_PG_MAJORVERSION_NUM(version) < 10)
282 0 : pg_fatal("server version too old");
283 40 : pg_log_debug("read server version %u from file \"%s/%s\"",
284 : GET_PG_MAJORVERSION_NUM(version), pgdata, "PG_VERSION");
285 :
286 : /* Sanity-check control files. */
287 40 : n_backups = argc - optind;
288 40 : system_identifier = check_control_files(n_backups, argv + optind);
289 :
290 : /* Sanity-check backup_label files, and get the contents of the last one. */
291 38 : last_backup_label = check_backup_label_files(n_backups, argv + optind);
292 :
293 : /*
294 : * We'll need the pathnames to the prior backups. By "prior" we mean all
295 : * but the last one listed on the command line.
296 : */
297 28 : n_prior_backups = argc - optind - 1;
298 28 : prior_backup_dirs = argv + optind;
299 :
300 : /* Load backup manifests. */
301 28 : manifests = load_backup_manifests(n_backups, prior_backup_dirs);
302 :
303 : /*
304 : * Validate the manifest system identifier against the backup system
305 : * identifier.
306 : */
307 80 : for (i = 0; i < n_backups; i++)
308 : {
309 54 : if (manifests[i] &&
310 54 : manifests[i]->system_identifier != system_identifier)
311 : {
312 : char *controlpath;
313 :
314 2 : controlpath = psprintf("%s/%s", prior_backup_dirs[i], XLOG_CONTROL_FILE);
315 :
316 2 : pg_fatal("%s: manifest system identifier is %" PRIu64 ", but control file has %" PRIu64,
317 : controlpath,
318 : manifests[i]->system_identifier,
319 : system_identifier);
320 : }
321 : }
322 :
323 : /* Figure out which tablespaces are going to be included in the output. */
324 26 : last_input_dir = argv[argc - 1];
325 26 : check_input_dir_permissions(last_input_dir);
326 26 : tablespaces = scan_for_existing_tablespaces(last_input_dir, &opt);
327 :
328 : /*
329 : * Create output directories.
330 : *
331 : * We create one output directory for the main data directory plus one for
332 : * each non-in-place tablespace. create_output_directory() will arrange
333 : * for those directories to be cleaned up on failure. In-place tablespaces
334 : * aren't handled at this stage because they're located beneath the main
335 : * output directory, and thus the cleanup of that directory will get rid
336 : * of them. Plus, the pg_tblspc directory that needs to contain them
337 : * doesn't exist yet.
338 : */
339 26 : atexit(cleanup_directories_atexit);
340 26 : create_output_directory(opt.output, &opt);
341 28 : for (ts = tablespaces; ts != NULL; ts = ts->next)
342 2 : if (!ts->in_place)
343 2 : create_output_directory(ts->new_dir, &opt);
344 :
345 : /* If we need to write a backup_manifest, prepare to do so. */
346 26 : if (!opt.dry_run && !opt.no_manifest)
347 : {
348 24 : mwriter = create_manifest_writer(opt.output, system_identifier);
349 :
350 : /*
351 : * Verify that we have a backup manifest for the final backup; else we
352 : * won't have the WAL ranges for the resulting manifest.
353 : */
354 24 : if (manifests[n_prior_backups] == NULL)
355 0 : pg_fatal("cannot generate a manifest because no manifest is available for the final input backup");
356 : }
357 : else
358 2 : mwriter = NULL;
359 :
360 : /* Write backup label into output directory. */
361 26 : if (opt.dry_run)
362 0 : pg_log_debug("would generate \"%s/backup_label\"", opt.output);
363 : else
364 : {
365 26 : pg_log_debug("generating \"%s/backup_label\"", opt.output);
366 26 : last_backup_label->cursor = 0;
367 26 : write_backup_label(opt.output, last_backup_label,
368 : opt.manifest_checksums, mwriter);
369 : }
370 :
371 : /* Process everything that's not part of a user-defined tablespace. */
372 26 : pg_log_debug("processing backup directory \"%s\"", last_input_dir);
373 26 : process_directory_recursively(InvalidOid, last_input_dir, opt.output,
374 : NULL, n_prior_backups, prior_backup_dirs,
375 : manifests, mwriter, &opt);
376 :
377 : /* Process user-defined tablespaces. */
378 26 : for (ts = tablespaces; ts != NULL; ts = ts->next)
379 : {
380 2 : pg_log_debug("processing tablespace directory \"%s\"", ts->old_dir);
381 :
382 : /*
383 : * If it's a normal tablespace, we need to set up a symbolic link from
384 : * pg_tblspc/${OID} to the target directory; if it's an in-place
385 : * tablespace, we need to create a directory at pg_tblspc/${OID}.
386 : */
387 2 : if (!ts->in_place)
388 : {
389 : char linkpath[MAXPGPATH];
390 :
391 2 : snprintf(linkpath, MAXPGPATH, "%s/%s/%u", opt.output, PG_TBLSPC_DIR,
392 : ts->oid);
393 :
394 2 : if (opt.dry_run)
395 0 : pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
396 : linkpath, ts->new_dir);
397 : else
398 : {
399 2 : pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
400 : linkpath, ts->new_dir);
401 2 : if (symlink(ts->new_dir, linkpath) != 0)
402 0 : pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
403 : linkpath, ts->new_dir);
404 : }
405 : }
406 : else
407 : {
408 0 : if (opt.dry_run)
409 0 : pg_log_debug("would create directory \"%s\"", ts->new_dir);
410 : else
411 : {
412 0 : pg_log_debug("creating directory \"%s\"", ts->new_dir);
413 0 : if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
414 0 : pg_fatal("could not create directory \"%s\": %m",
415 : ts->new_dir);
416 : }
417 : }
418 :
419 : /* OK, now handle the directory contents. */
420 2 : process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
421 : NULL, n_prior_backups, prior_backup_dirs,
422 : manifests, mwriter, &opt);
423 : }
424 :
425 : /* Finalize the backup_manifest, if we're generating one. */
426 24 : if (mwriter != NULL)
427 22 : finalize_manifest(mwriter,
428 22 : manifests[n_prior_backups]->first_wal_range);
429 :
430 : /* fsync that output directory unless we've been told not to do so */
431 24 : if (!opt.no_sync)
432 : {
433 18 : if (opt.dry_run)
434 0 : pg_log_debug("would recursively fsync \"%s\"", opt.output);
435 : else
436 : {
437 18 : pg_log_debug("recursively fsyncing \"%s\"", opt.output);
438 18 : sync_pgdata(opt.output, version, opt.sync_method, true);
439 : }
440 : }
441 :
442 : /* Warn about the possibility of compromising the backups, when link mode */
443 24 : if (opt.copy_method == COPY_METHOD_LINK)
444 2 : pg_log_warning("--link mode was used; any modifications to the output "
445 : "directory might destructively modify input directories");
446 :
447 : /* It's a success, so don't remove the output directories. */
448 24 : reset_directory_cleanup_list();
449 24 : exit(0);
450 : }
451 :
452 : /*
453 : * Process the option argument for the -T, --tablespace-mapping switch.
454 : */
455 : static void
456 2 : add_tablespace_mapping(cb_options *opt, char *arg)
457 : {
458 2 : cb_tablespace_mapping *tsmap = pg_malloc0(sizeof(cb_tablespace_mapping));
459 : char *dst;
460 : char *dst_ptr;
461 : char *arg_ptr;
462 :
463 : /*
464 : * Basically, we just want to copy everything before the equals sign to
465 : * tsmap->old_dir and everything afterwards to tsmap->new_dir, but if
466 : * there's more or less than one equals sign, that's an error, and if
467 : * there's an equals sign preceded by a backslash, don't treat it as a
468 : * field separator but instead copy a literal equals sign.
469 : */
470 2 : dst_ptr = dst = tsmap->old_dir;
471 100 : for (arg_ptr = arg; *arg_ptr != '\0'; arg_ptr++)
472 : {
473 98 : if (dst_ptr - dst >= MAXPGPATH)
474 0 : pg_fatal("directory name too long");
475 :
476 98 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
477 : ; /* skip backslash escaping = */
478 98 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
479 : {
480 2 : if (tsmap->new_dir[0] != '\0')
481 0 : pg_fatal("multiple \"=\" signs in tablespace mapping");
482 : else
483 2 : dst = dst_ptr = tsmap->new_dir;
484 : }
485 : else
486 96 : *dst_ptr++ = *arg_ptr;
487 : }
488 2 : if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
489 0 : pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
490 :
491 : /*
492 : * All tablespaces are created with absolute directories, so specifying a
493 : * non-absolute path here would never match, possibly confusing users.
494 : *
495 : * In contrast to pg_basebackup, both the old and new directories are on
496 : * the local machine, so the local machine's definition of an absolute
497 : * path is the only relevant one.
498 : */
499 2 : if (!is_absolute_path(tsmap->old_dir))
500 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
501 : tsmap->old_dir);
502 :
503 2 : if (!is_absolute_path(tsmap->new_dir))
504 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
505 : tsmap->new_dir);
506 :
507 : /* Canonicalize paths to avoid spurious failures when comparing. */
508 2 : canonicalize_path(tsmap->old_dir);
509 2 : canonicalize_path(tsmap->new_dir);
510 :
511 : /* Add it to the list. */
512 2 : tsmap->next = opt->tsmappings;
513 2 : opt->tsmappings = tsmap;
514 2 : }
515 :
516 : /*
517 : * Check that the backup_label files form a coherent backup chain, and return
518 : * the contents of the backup_label file from the latest backup.
519 : */
520 : static StringInfo
521 38 : check_backup_label_files(int n_backups, char **backup_dirs)
522 : {
523 38 : StringInfo buf = makeStringInfo();
524 38 : StringInfo lastbuf = buf;
525 : int i;
526 38 : TimeLineID check_tli = 0;
527 38 : XLogRecPtr check_lsn = InvalidXLogRecPtr;
528 :
529 : /* Try to read each backup_label file in turn, last to first. */
530 102 : for (i = n_backups - 1; i >= 0; --i)
531 : {
532 : char pathbuf[MAXPGPATH];
533 : int fd;
534 : TimeLineID start_tli;
535 : TimeLineID previous_tli;
536 : XLogRecPtr start_lsn;
537 : XLogRecPtr previous_lsn;
538 :
539 : /* Open the backup_label file. */
540 74 : snprintf(pathbuf, MAXPGPATH, "%s/backup_label", backup_dirs[i]);
541 74 : pg_log_debug("reading \"%s\"", pathbuf);
542 74 : if ((fd = open(pathbuf, O_RDONLY, 0)) < 0)
543 0 : pg_fatal("could not open file \"%s\": %m", pathbuf);
544 :
545 : /*
546 : * Slurp the whole file into memory.
547 : *
548 : * The exact size limit that we impose here doesn't really matter --
549 : * most of what's supposed to be in the file is fixed size and quite
550 : * short. However, the length of the backup_label is limited (at least
551 : * by some parts of the code) to MAXPGPATH, so include that value in
552 : * the maximum length that we tolerate.
553 : */
554 74 : slurp_file(fd, pathbuf, buf, 10000 + MAXPGPATH);
555 :
556 : /* Close the file. */
557 74 : if (close(fd) != 0)
558 0 : pg_fatal("could not close file \"%s\": %m", pathbuf);
559 :
560 : /* Parse the file contents. */
561 74 : parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
562 : &previous_tli, &previous_lsn);
563 :
564 : /*
565 : * Sanity checks.
566 : *
567 : * XXX. It's actually not required that start_lsn == check_lsn. It
568 : * would be OK if start_lsn > check_lsn provided that start_lsn is
569 : * less than or equal to the relevant switchpoint. But at the moment
570 : * we don't have that information.
571 : */
572 74 : if (i > 0 && previous_tli == 0)
573 2 : pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
574 : backup_dirs[i]);
575 72 : if (i == 0 && previous_tli != 0)
576 2 : pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
577 : backup_dirs[i]);
578 70 : if (i < n_backups - 1 && start_tli != check_tli)
579 0 : pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
580 : backup_dirs[i], start_tli, check_tli);
581 70 : if (i < n_backups - 1 && start_lsn != check_lsn)
582 6 : pg_fatal("backup at \"%s\" starts at LSN %X/%08X, but expected %X/%08X",
583 : backup_dirs[i],
584 : LSN_FORMAT_ARGS(start_lsn),
585 : LSN_FORMAT_ARGS(check_lsn));
586 64 : check_tli = previous_tli;
587 64 : check_lsn = previous_lsn;
588 :
589 : /*
590 : * The last backup label in the chain needs to be saved for later use,
591 : * while the others are only needed within this loop.
592 : */
593 64 : if (lastbuf == buf)
594 36 : buf = makeStringInfo();
595 : else
596 28 : resetStringInfo(buf);
597 : }
598 :
599 : /* Free memory that we don't need any more. */
600 28 : if (lastbuf != buf)
601 28 : destroyStringInfo(buf);
602 :
603 : /*
604 : * Return the data from the first backup_info that we read (which is the
605 : * backup_label from the last directory specified on the command line).
606 : */
607 28 : return lastbuf;
608 : }
609 :
610 : /*
611 : * Sanity check control files and return system_identifier.
612 : */
613 : static uint64
614 40 : check_control_files(int n_backups, char **backup_dirs)
615 : {
616 : int i;
617 40 : uint64 system_identifier = 0; /* placate compiler */
618 40 : uint32 data_checksum_version = 0; /* placate compiler */
619 40 : bool data_checksum_mismatch = false;
620 :
621 : /* Try to read each control file in turn, last to first. */
622 120 : for (i = n_backups - 1; i >= 0; --i)
623 : {
624 : ControlFileData *control_file;
625 : bool crc_ok;
626 : char *controlpath;
627 :
628 82 : controlpath = psprintf("%s/%s", backup_dirs[i], XLOG_CONTROL_FILE);
629 82 : pg_log_debug("reading \"%s\"", controlpath);
630 82 : control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
631 :
632 : /* Control file contents not meaningful if CRC is bad. */
633 82 : if (!crc_ok)
634 0 : pg_fatal("%s: CRC is incorrect", controlpath);
635 :
636 : /* Can't interpret control file if not current version. */
637 82 : if (control_file->pg_control_version != PG_CONTROL_VERSION)
638 0 : pg_fatal("%s: unexpected control file version",
639 : controlpath);
640 :
641 : /* System identifiers should all match. */
642 82 : if (i == n_backups - 1)
643 40 : system_identifier = control_file->system_identifier;
644 42 : else if (system_identifier != control_file->system_identifier)
645 2 : pg_fatal("%s: expected system identifier %" PRIu64 ", but found %" PRIu64,
646 : controlpath, system_identifier,
647 : control_file->system_identifier);
648 :
649 : /*
650 : * Detect checksum mismatches, but only if the last backup in the
651 : * chain has checksums enabled.
652 : */
653 80 : if (i == n_backups - 1)
654 40 : data_checksum_version = control_file->data_checksum_version;
655 40 : else if (data_checksum_version != 0 &&
656 40 : data_checksum_version != control_file->data_checksum_version)
657 0 : data_checksum_mismatch = true;
658 :
659 : /* Release memory. */
660 80 : pfree(control_file);
661 80 : pfree(controlpath);
662 : }
663 :
664 : /*
665 : * If debug output is enabled, make a note of the system identifier that
666 : * we found in all of the relevant control files.
667 : */
668 38 : pg_log_debug("system identifier is %" PRIu64, system_identifier);
669 :
670 : /*
671 : * Warn the user if not all backups are in the same state with regards to
672 : * checksums.
673 : */
674 38 : if (data_checksum_mismatch)
675 : {
676 0 : pg_log_warning("only some backups have checksums enabled");
677 0 : pg_log_warning_hint("Disable, and optionally reenable, checksums on the output directory to avoid failures.");
678 : }
679 :
680 38 : return system_identifier;
681 : }
682 :
683 : /*
684 : * Set default permissions for new files and directories based on the
685 : * permissions of the given directory. The intent here is that the output
686 : * directory should use the same permissions scheme as the final input
687 : * directory.
688 : */
689 : static void
690 26 : check_input_dir_permissions(char *dir)
691 : {
692 : struct stat st;
693 :
694 26 : if (stat(dir, &st) != 0)
695 0 : pg_fatal("could not stat file \"%s\": %m", dir);
696 :
697 26 : SetDataDirectoryCreatePerm(st.st_mode);
698 26 : }
699 :
700 : /*
701 : * Clean up output directories before exiting.
702 : */
703 : static void
704 26 : cleanup_directories_atexit(void)
705 : {
706 28 : while (cleanup_dir_list != NULL)
707 : {
708 2 : cb_cleanup_dir *dir = cleanup_dir_list;
709 :
710 2 : if (dir->rmtopdir)
711 : {
712 2 : pg_log_info("removing output directory \"%s\"", dir->target_path);
713 2 : if (!rmtree(dir->target_path, dir->rmtopdir))
714 0 : pg_log_error("failed to remove output directory");
715 : }
716 : else
717 : {
718 0 : pg_log_info("removing contents of output directory \"%s\"",
719 : dir->target_path);
720 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
721 0 : pg_log_error("failed to remove contents of output directory");
722 : }
723 :
724 2 : cleanup_dir_list = cleanup_dir_list->next;
725 2 : pfree(dir);
726 : }
727 26 : }
728 :
729 : /*
730 : * Create the named output directory, unless it already exists or we're in
731 : * dry-run mode. If it already exists but is not empty, that's a fatal error.
732 : *
733 : * Adds the created directory to the list of directories to be cleaned up
734 : * at process exit.
735 : */
736 : static void
737 28 : create_output_directory(char *dirname, cb_options *opt)
738 : {
739 28 : switch (pg_check_dir(dirname))
740 : {
741 28 : case 0:
742 28 : if (opt->dry_run)
743 : {
744 0 : pg_log_debug("would create directory \"%s\"", dirname);
745 0 : return;
746 : }
747 28 : pg_log_debug("creating directory \"%s\"", dirname);
748 28 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
749 0 : pg_fatal("could not create directory \"%s\": %m", dirname);
750 28 : remember_to_cleanup_directory(dirname, true);
751 28 : break;
752 :
753 0 : case 1:
754 0 : pg_log_debug("using existing directory \"%s\"", dirname);
755 0 : remember_to_cleanup_directory(dirname, false);
756 0 : break;
757 :
758 0 : case 2:
759 : case 3:
760 : case 4:
761 0 : pg_fatal("directory \"%s\" exists but is not empty", dirname);
762 :
763 0 : case -1:
764 0 : pg_fatal("could not access directory \"%s\": %m", dirname);
765 : }
766 : }
767 :
768 : /*
769 : * help
770 : *
771 : * Prints help page for the program
772 : *
773 : * progname: the name of the executed program, such as "pg_combinebackup"
774 : */
775 : static void
776 2 : help(const char *progname)
777 : {
778 2 : printf(_("%s reconstructs full backups from incrementals.\n\n"), progname);
779 2 : printf(_("Usage:\n"));
780 2 : printf(_(" %s [OPTION]... DIRECTORY...\n"), progname);
781 2 : printf(_("\nOptions:\n"));
782 2 : printf(_(" -d, --debug generate lots of debugging output\n"));
783 2 : printf(_(" -k, --link link files instead of copying\n"));
784 2 : printf(_(" -n, --dry-run do not actually do anything\n"));
785 2 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
786 2 : printf(_(" -o, --output=DIRECTORY output directory\n"));
787 2 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
788 : " relocate tablespace in OLDDIR to NEWDIR\n"));
789 2 : printf(_(" --clone clone (reflink) files instead of copying\n"));
790 2 : printf(_(" --copy copy files (default)\n"));
791 2 : printf(_(" --copy-file-range copy using copy_file_range() system call\n"));
792 2 : printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
793 : " use algorithm for manifest checksums\n"));
794 2 : printf(_(" --no-manifest suppress generation of backup manifest\n"));
795 2 : printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
796 2 : printf(_(" -V, --version output version information, then exit\n"));
797 2 : printf(_(" -?, --help show this help, then exit\n"));
798 :
799 2 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
800 2 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
801 2 : }
802 :
803 : /*
804 : * Try to parse a string as a non-zero OID without leading zeroes.
805 : *
806 : * If it works, return true and set *result to the answer, else return false.
807 : */
808 : static bool
809 4 : parse_oid(char *s, Oid *result)
810 : {
811 : Oid oid;
812 : char *ep;
813 :
814 4 : errno = 0;
815 4 : oid = strtoul(s, &ep, 10);
816 4 : if (errno != 0 || *ep != '\0' || oid < 1 || oid > PG_UINT32_MAX)
817 0 : return false;
818 :
819 4 : *result = oid;
820 4 : return true;
821 : }
822 :
823 : /*
824 : * Copy files from the input directory to the output directory, reconstructing
825 : * full files from incremental files as required.
826 : *
827 : * If processing a user-defined tablespace, the tsoid should be the OID
828 : * of that tablespace and input_directory and output_directory should be the
829 : * toplevel input and output directories for that tablespace. Otherwise,
830 : * tsoid should be InvalidOid and input_directory and output_directory should
831 : * be the main input and output directories.
832 : *
833 : * relative_path is the path beneath the given input and output directories
834 : * that we are currently processing. If NULL, it indicates that we're
835 : * processing the input and output directories themselves.
836 : *
837 : * n_prior_backups is the number of prior backups that we have available.
838 : * This doesn't count the very last backup, which is referenced by
839 : * input_directory, just the older ones. prior_backup_dirs is an array of
840 : * the locations of those previous backups.
841 : */
842 : static void
843 708 : process_directory_recursively(Oid tsoid,
844 : char *input_directory,
845 : char *output_directory,
846 : char *relative_path,
847 : int n_prior_backups,
848 : char **prior_backup_dirs,
849 : manifest_data **manifests,
850 : manifest_writer *mwriter,
851 : cb_options *opt)
852 : {
853 : char ifulldir[MAXPGPATH];
854 : char ofulldir[MAXPGPATH];
855 : char manifest_prefix[MAXPGPATH];
856 : DIR *dir;
857 : struct dirent *de;
858 708 : bool is_pg_tblspc = false;
859 708 : bool is_pg_wal = false;
860 708 : bool is_incremental_dir = false;
861 708 : manifest_data *latest_manifest = manifests[n_prior_backups];
862 : pg_checksum_type checksum_type;
863 :
864 : /*
865 : * Classify this directory.
866 : *
867 : * We set is_pg_tblspc only for the toplevel pg_tblspc directory, because
868 : * the symlinks in that specific directory require special handling.
869 : *
870 : * We set is_pg_wal for the toplevel WAL directory and all of its
871 : * subdirectories, because those files are not included in the backup
872 : * manifest and hence need special treatment. (Since incremental backup
873 : * does not exist in pre-v10 versions, we don't have to worry about the
874 : * old pg_xlog naming.)
875 : *
876 : * We set is_incremental_dir for directories that can contain incremental
877 : * files requiring reconstruction. If such files occur outside these
878 : * directories, we want to just copy them straight to the output
879 : * directory. This is to protect against a user creating a file with a
880 : * strange name like INCREMENTAL.config and then complaining that
881 : * incremental backups don't work properly. The test here is a bit tricky:
882 : * incremental files occur in subdirectories of base, in pg_global itself,
883 : * and in subdirectories of pg_tblspc only if in-place tablespaces are
884 : * used.
885 : */
886 708 : if (OidIsValid(tsoid))
887 6 : is_incremental_dir = true;
888 702 : else if (relative_path != NULL)
889 : {
890 676 : is_pg_tblspc = strcmp(relative_path, PG_TBLSPC_DIR) == 0;
891 1326 : is_pg_wal = (strcmp(relative_path, "pg_wal") == 0 ||
892 650 : strncmp(relative_path, "pg_wal/", 7) == 0);
893 676 : is_incremental_dir = strncmp(relative_path, "base/", 5) == 0 ||
894 1244 : strcmp(relative_path, "global") == 0 ||
895 568 : strncmp(relative_path, PG_TBLSPC_DIR_SLASH, 10) == 0;
896 : }
897 :
898 : /*
899 : * If we're under pg_wal, then we don't need checksums, because these
900 : * files aren't included in the backup manifest. Otherwise use whatever
901 : * type of checksum is configured.
902 : */
903 708 : if (!is_pg_wal)
904 630 : checksum_type = opt->manifest_checksums;
905 : else
906 78 : checksum_type = CHECKSUM_TYPE_NONE;
907 :
908 : /*
909 : * Append the relative path to the input and output directories, and
910 : * figure out the appropriate prefix to add to files in this directory
911 : * when looking them up in a backup manifest.
912 : */
913 708 : if (relative_path == NULL)
914 : {
915 28 : strlcpy(ifulldir, input_directory, MAXPGPATH);
916 28 : strlcpy(ofulldir, output_directory, MAXPGPATH);
917 28 : if (OidIsValid(tsoid))
918 2 : snprintf(manifest_prefix, MAXPGPATH, "%s/%u/", PG_TBLSPC_DIR, tsoid);
919 : else
920 26 : manifest_prefix[0] = '\0';
921 : }
922 : else
923 : {
924 680 : snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
925 : relative_path);
926 680 : snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
927 : relative_path);
928 680 : if (OidIsValid(tsoid))
929 4 : snprintf(manifest_prefix, MAXPGPATH, "%s/%u/%s/",
930 : PG_TBLSPC_DIR, tsoid, relative_path);
931 : else
932 676 : snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
933 : }
934 :
935 : /*
936 : * Toplevel output directories have already been created by the time this
937 : * function is called, but any subdirectories are our responsibility.
938 : */
939 708 : if (relative_path != NULL)
940 : {
941 680 : if (opt->dry_run)
942 0 : pg_log_debug("would create directory \"%s\"", ofulldir);
943 : else
944 : {
945 680 : pg_log_debug("creating directory \"%s\"", ofulldir);
946 680 : if (mkdir(ofulldir, pg_dir_create_mode) == -1)
947 0 : pg_fatal("could not create directory \"%s\": %m", ofulldir);
948 : }
949 : }
950 :
951 : /* It's time to scan the directory. */
952 708 : if ((dir = opendir(ifulldir)) == NULL)
953 0 : pg_fatal("could not open directory \"%s\": %m", ifulldir);
954 28726 : while (errno = 0, (de = readdir(dir)) != NULL)
955 : {
956 : PGFileType type;
957 : char ifullpath[MAXPGPATH];
958 : char ofullpath[MAXPGPATH];
959 : char manifest_path[MAXPGPATH];
960 28024 : Oid oid = InvalidOid;
961 28024 : int checksum_length = 0;
962 28024 : uint8 *checksum_payload = NULL;
963 : pg_checksum_context checksum_ctx;
964 :
965 : /* Ignore "." and ".." entries. */
966 28024 : if (strcmp(de->d_name, ".") == 0 ||
967 27316 : strcmp(de->d_name, "..") == 0)
968 2146 : continue;
969 :
970 : /* Construct input path. */
971 26608 : snprintf(ifullpath, MAXPGPATH, "%s/%s", ifulldir, de->d_name);
972 :
973 : /* Figure out what kind of directory entry this is. */
974 26608 : type = get_dirent_type(ifullpath, de, false, PG_LOG_ERROR);
975 26608 : if (type == PGFILETYPE_ERROR)
976 0 : exit(1);
977 :
978 : /*
979 : * If we're processing pg_tblspc, then check whether the filename
980 : * looks like it could be a tablespace OID. If so, and if the
981 : * directory entry is a symbolic link or a directory, skip it.
982 : *
983 : * Our goal here is to ignore anything that would have been considered
984 : * by scan_for_existing_tablespaces to be a tablespace.
985 : */
986 26608 : if (is_pg_tblspc && parse_oid(de->d_name, &oid) &&
987 0 : (type == PGFILETYPE_LNK || type == PGFILETYPE_DIR))
988 2 : continue;
989 :
990 : /* If it's a directory, recurse. */
991 26606 : if (type == PGFILETYPE_DIR)
992 676 : {
993 : char new_relative_path[MAXPGPATH];
994 :
995 : /* Append new pathname component to relative path. */
996 680 : if (relative_path == NULL)
997 440 : strlcpy(new_relative_path, de->d_name, MAXPGPATH);
998 : else
999 240 : snprintf(new_relative_path, MAXPGPATH, "%s/%s", relative_path,
1000 240 : de->d_name);
1001 :
1002 : /* And recurse. */
1003 680 : process_directory_recursively(tsoid,
1004 : input_directory, output_directory,
1005 : new_relative_path,
1006 : n_prior_backups, prior_backup_dirs,
1007 : manifests, mwriter, opt);
1008 676 : continue;
1009 : }
1010 :
1011 : /* Skip anything that's not a regular file. */
1012 25926 : if (type != PGFILETYPE_REG)
1013 : {
1014 0 : if (type == PGFILETYPE_LNK)
1015 0 : pg_log_warning("skipping symbolic link \"%s\"", ifullpath);
1016 : else
1017 0 : pg_log_warning("skipping special file \"%s\"", ifullpath);
1018 0 : continue;
1019 : }
1020 :
1021 : /*
1022 : * Skip the backup_label and backup_manifest files; they require
1023 : * special handling and are handled elsewhere.
1024 : */
1025 25926 : if (relative_path == NULL &&
1026 190 : (strcmp(de->d_name, "backup_label") == 0 ||
1027 164 : strcmp(de->d_name, "backup_manifest") == 0))
1028 52 : continue;
1029 :
1030 : /*
1031 : * If it's an incremental file, hand it off to the reconstruction
1032 : * code, which will figure out what to do.
1033 : */
1034 25874 : if (is_incremental_dir &&
1035 25602 : strncmp(de->d_name, INCREMENTAL_PREFIX,
1036 : INCREMENTAL_PREFIX_LENGTH) == 0)
1037 : {
1038 : /* Output path should not include "INCREMENTAL." prefix. */
1039 12906 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
1040 12906 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1041 :
1042 :
1043 : /* Manifest path likewise omits incremental prefix. */
1044 12906 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1045 12906 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1046 :
1047 : /* Reconstruction logic will do the rest. */
1048 12906 : reconstruct_from_incremental_file(ifullpath, ofullpath,
1049 : manifest_prefix,
1050 12906 : de->d_name + INCREMENTAL_PREFIX_LENGTH,
1051 : n_prior_backups,
1052 : prior_backup_dirs,
1053 : manifests,
1054 : manifest_path,
1055 : checksum_type,
1056 : &checksum_length,
1057 : &checksum_payload,
1058 : opt->copy_method,
1059 12906 : opt->debug,
1060 12906 : opt->dry_run);
1061 : }
1062 : else
1063 : {
1064 : /* Construct the path that the backup_manifest will use. */
1065 12968 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1066 12968 : de->d_name);
1067 :
1068 : /*
1069 : * It's not an incremental file, so we need to copy the entire
1070 : * file to the output directory.
1071 : *
1072 : * If a checksum of the required type already exists in the
1073 : * backup_manifest for the final input directory, we can save some
1074 : * work by reusing that checksum instead of computing a new one.
1075 : */
1076 12968 : if (checksum_type != CHECKSUM_TYPE_NONE &&
1077 : latest_manifest != NULL)
1078 : {
1079 : manifest_file *mfile;
1080 :
1081 9070 : mfile = manifest_files_lookup(latest_manifest->files,
1082 : manifest_path);
1083 9070 : if (mfile == NULL)
1084 : {
1085 : char *bmpath;
1086 :
1087 : /*
1088 : * The directory is out of sync with the backup_manifest,
1089 : * so emit a warning.
1090 : */
1091 0 : bmpath = psprintf("%s/%s", input_directory,
1092 : "backup_manifest");
1093 0 : pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
1094 : bmpath, manifest_path);
1095 0 : pfree(bmpath);
1096 : }
1097 9070 : else if (mfile->checksum_type == checksum_type)
1098 : {
1099 7136 : checksum_length = mfile->checksum_length;
1100 7136 : checksum_payload = mfile->checksum_payload;
1101 : }
1102 : }
1103 :
1104 : /*
1105 : * If we're reusing a checksum, then we don't need copy_file() to
1106 : * compute one for us, but otherwise, it needs to compute whatever
1107 : * type of checksum we need.
1108 : */
1109 12968 : if (checksum_length != 0)
1110 7136 : pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_NONE);
1111 : else
1112 5832 : pg_checksum_init(&checksum_ctx, checksum_type);
1113 :
1114 : /* Actually copy the file. */
1115 12968 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
1116 12968 : copy_file(ifullpath, ofullpath, &checksum_ctx,
1117 12968 : opt->copy_method, opt->dry_run);
1118 :
1119 : /*
1120 : * If copy_file() performed a checksum calculation for us, then
1121 : * save the results (except in dry-run mode, when there's no
1122 : * point).
1123 : */
1124 12968 : if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
1125 : {
1126 1934 : checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
1127 1934 : checksum_length = pg_checksum_final(&checksum_ctx,
1128 : checksum_payload);
1129 : }
1130 : }
1131 :
1132 : /* Generate manifest entry, if needed. */
1133 25872 : if (mwriter != NULL)
1134 : {
1135 : struct stat sb;
1136 :
1137 : /*
1138 : * In order to generate a manifest entry, we need the file size
1139 : * and mtime. We have no way to know the correct mtime except to
1140 : * stat() the file, so just do that and get the size as well.
1141 : *
1142 : * If we didn't need the mtime here, we could try to obtain the
1143 : * file size from the reconstruction or file copy process above,
1144 : * although that is actually not convenient in all cases. If we
1145 : * write the file ourselves then clearly we can keep a count of
1146 : * bytes, but if we use something like CopyFile() then it's
1147 : * trickier. Since we have to stat() anyway to get the mtime,
1148 : * there's no point in worrying about it.
1149 : */
1150 23936 : if (stat(ofullpath, &sb) < 0)
1151 0 : pg_fatal("could not stat file \"%s\": %m", ofullpath);
1152 :
1153 : /* OK, now do the work. */
1154 23936 : add_file_to_manifest(mwriter, manifest_path,
1155 23936 : sb.st_size, sb.st_mtime,
1156 : checksum_type, checksum_length,
1157 : checksum_payload);
1158 : }
1159 :
1160 : /* Avoid leaking memory. */
1161 25872 : if (checksum_payload != NULL)
1162 21974 : pfree(checksum_payload);
1163 : }
1164 :
1165 702 : closedir(dir);
1166 702 : }
1167 :
1168 : /*
1169 : * Add a directory to the list of output directories to clean up.
1170 : */
1171 : static void
1172 28 : remember_to_cleanup_directory(char *target_path, bool rmtopdir)
1173 : {
1174 28 : cb_cleanup_dir *dir = pg_malloc(sizeof(cb_cleanup_dir));
1175 :
1176 28 : dir->target_path = target_path;
1177 28 : dir->rmtopdir = rmtopdir;
1178 28 : dir->next = cleanup_dir_list;
1179 28 : cleanup_dir_list = dir;
1180 28 : }
1181 :
1182 : /*
1183 : * Empty out the list of directories scheduled for cleanup at exit.
1184 : *
1185 : * We want to remove the output directories only on a failure, so call this
1186 : * function when we know that the operation has succeeded.
1187 : *
1188 : * Since we only expect this to be called when we're about to exit, we could
1189 : * just set cleanup_dir_list to NULL and be done with it, but we free the
1190 : * memory to be tidy.
1191 : */
1192 : static void
1193 24 : reset_directory_cleanup_list(void)
1194 : {
1195 50 : while (cleanup_dir_list != NULL)
1196 : {
1197 26 : cb_cleanup_dir *dir = cleanup_dir_list;
1198 :
1199 26 : cleanup_dir_list = cleanup_dir_list->next;
1200 26 : pfree(dir);
1201 : }
1202 24 : }
1203 :
1204 : /*
1205 : * Scan the pg_tblspc directory of the final input backup to get a canonical
1206 : * list of what tablespaces are part of the backup.
1207 : *
1208 : * 'pathname' should be the path to the toplevel backup directory for the
1209 : * final backup in the backup chain.
1210 : */
1211 : static cb_tablespace *
1212 26 : scan_for_existing_tablespaces(char *pathname, cb_options *opt)
1213 : {
1214 : char pg_tblspc[MAXPGPATH];
1215 : DIR *dir;
1216 : struct dirent *de;
1217 26 : cb_tablespace *tslist = NULL;
1218 :
1219 26 : snprintf(pg_tblspc, MAXPGPATH, "%s/%s", pathname, PG_TBLSPC_DIR);
1220 26 : pg_log_debug("scanning \"%s\"", pg_tblspc);
1221 :
1222 26 : if ((dir = opendir(pg_tblspc)) == NULL)
1223 0 : pg_fatal("could not open directory \"%s\": %m", pg_tblspc);
1224 :
1225 80 : while (errno = 0, (de = readdir(dir)) != NULL)
1226 : {
1227 : Oid oid;
1228 : char tblspcdir[MAXPGPATH];
1229 : char link_target[MAXPGPATH];
1230 : int link_length;
1231 : cb_tablespace *ts;
1232 : cb_tablespace *otherts;
1233 : PGFileType type;
1234 :
1235 : /* Silently ignore "." and ".." entries. */
1236 54 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1237 52 : continue;
1238 :
1239 : /* Construct full pathname. */
1240 2 : snprintf(tblspcdir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1241 :
1242 : /* Ignore any file name that doesn't look like a proper OID. */
1243 2 : if (!parse_oid(de->d_name, &oid))
1244 : {
1245 0 : pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
1246 : tblspcdir);
1247 0 : continue;
1248 : }
1249 :
1250 : /* Only symbolic links and directories are tablespaces. */
1251 2 : type = get_dirent_type(tblspcdir, de, false, PG_LOG_ERROR);
1252 2 : if (type == PGFILETYPE_ERROR)
1253 0 : exit(1);
1254 2 : if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
1255 : {
1256 0 : pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
1257 : tblspcdir);
1258 0 : continue;
1259 : }
1260 :
1261 : /* Create a new tablespace object. */
1262 2 : ts = pg_malloc0(sizeof(cb_tablespace));
1263 2 : ts->oid = oid;
1264 :
1265 : /*
1266 : * If it's a link, it's not an in-place tablespace. Otherwise, it must
1267 : * be a directory, and thus an in-place tablespace.
1268 : */
1269 2 : if (type == PGFILETYPE_LNK)
1270 : {
1271 : cb_tablespace_mapping *tsmap;
1272 :
1273 : /* Read the link target. */
1274 2 : link_length = readlink(tblspcdir, link_target, sizeof(link_target));
1275 2 : if (link_length < 0)
1276 0 : pg_fatal("could not read symbolic link \"%s\": %m",
1277 : tblspcdir);
1278 2 : if (link_length >= sizeof(link_target))
1279 0 : pg_fatal("target of symbolic link \"%s\" is too long", tblspcdir);
1280 2 : link_target[link_length] = '\0';
1281 2 : if (!is_absolute_path(link_target))
1282 0 : pg_fatal("target of symbolic link \"%s\" is relative", tblspcdir);
1283 :
1284 : /* Canonicalize the link target. */
1285 2 : canonicalize_path(link_target);
1286 :
1287 : /*
1288 : * Find the corresponding tablespace mapping and copy the relevant
1289 : * details into the new tablespace entry.
1290 : */
1291 2 : for (tsmap = opt->tsmappings; tsmap != NULL; tsmap = tsmap->next)
1292 : {
1293 2 : if (strcmp(tsmap->old_dir, link_target) == 0)
1294 : {
1295 2 : strlcpy(ts->old_dir, tsmap->old_dir, MAXPGPATH);
1296 2 : strlcpy(ts->new_dir, tsmap->new_dir, MAXPGPATH);
1297 2 : ts->in_place = false;
1298 2 : break;
1299 : }
1300 : }
1301 :
1302 : /* Every non-in-place tablespace must be mapped. */
1303 2 : if (tsmap == NULL)
1304 0 : pg_fatal("tablespace at \"%s\" has no tablespace mapping",
1305 : link_target);
1306 : }
1307 : else
1308 : {
1309 : /*
1310 : * For an in-place tablespace, there's no separate directory, so
1311 : * we just record the paths within the data directories.
1312 : */
1313 0 : snprintf(ts->old_dir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1314 0 : snprintf(ts->new_dir, MAXPGPATH, "%s/%s/%s", opt->output,
1315 0 : PG_TBLSPC_DIR, de->d_name);
1316 0 : ts->in_place = true;
1317 : }
1318 :
1319 : /* Tablespaces should not share a directory. */
1320 2 : for (otherts = tslist; otherts != NULL; otherts = otherts->next)
1321 0 : if (strcmp(ts->new_dir, otherts->new_dir) == 0)
1322 0 : pg_fatal("tablespaces with OIDs %u and %u both point at directory \"%s\"",
1323 : otherts->oid, oid, ts->new_dir);
1324 :
1325 : /* Add this tablespace to the list. */
1326 2 : ts->next = tslist;
1327 2 : tslist = ts;
1328 : }
1329 :
1330 26 : if (closedir(dir) != 0)
1331 0 : pg_fatal("could not close directory \"%s\": %m", pg_tblspc);
1332 :
1333 26 : return tslist;
1334 : }
1335 :
1336 : /*
1337 : * Read a file into a StringInfo.
1338 : *
1339 : * fd is used for the actual file I/O, filename for error reporting purposes.
1340 : * A file longer than maxlen is a fatal error.
1341 : */
1342 : static void
1343 74 : slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
1344 : {
1345 : struct stat st;
1346 : ssize_t rb;
1347 :
1348 : /* Check file size, and complain if it's too large. */
1349 74 : if (fstat(fd, &st) != 0)
1350 0 : pg_fatal("could not stat file \"%s\": %m", filename);
1351 74 : if (st.st_size > maxlen)
1352 0 : pg_fatal("file \"%s\" is too large", filename);
1353 :
1354 : /* Make sure we have enough space. */
1355 74 : enlargeStringInfo(buf, st.st_size);
1356 :
1357 : /* Read the data. */
1358 74 : rb = read(fd, &buf->data[buf->len], st.st_size);
1359 :
1360 : /*
1361 : * We don't expect any concurrent changes, so we should read exactly the
1362 : * expected number of bytes.
1363 : */
1364 74 : if (rb != st.st_size)
1365 : {
1366 0 : if (rb < 0)
1367 0 : pg_fatal("could not read file \"%s\": %m", filename);
1368 : else
1369 0 : pg_fatal("could not read file \"%s\": read %zd of %lld",
1370 : filename, rb, (long long int) st.st_size);
1371 : }
1372 :
1373 : /* Adjust buffer length for new data and restore trailing-\0 invariant */
1374 74 : buf->len += rb;
1375 74 : buf->data[buf->len] = '\0';
1376 74 : }
|
