Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_combinebackup.c
4 : * Combine incremental backups with prior backups.
5 : *
6 : * Copyright (c) 2017-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/bin/pg_combinebackup/pg_combinebackup.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres_fe.h"
14 :
15 : #include <dirent.h>
16 : #include <fcntl.h>
17 : #include <limits.h>
18 :
19 : #include "backup_label.h"
20 : #include "common/blkreftable.h"
21 : #include "common/checksum_helper.h"
22 : #include "common/controldata_utils.h"
23 : #include "common/file_perm.h"
24 : #include "common/file_utils.h"
25 : #include "common/logging.h"
26 : #include "copy_file.h"
27 : #include "fe_utils/option_utils.h"
28 : #include "getopt_long.h"
29 : #include "lib/stringinfo.h"
30 : #include "load_manifest.h"
31 : #include "reconstruct.h"
32 : #include "write_manifest.h"
33 :
34 : /* Incremental file naming convention. */
35 : #define INCREMENTAL_PREFIX "INCREMENTAL."
36 : #define INCREMENTAL_PREFIX_LENGTH (sizeof(INCREMENTAL_PREFIX) - 1)
37 :
38 : /*
39 : * Tracking for directories that need to be removed, or have their contents
40 : * removed, if the operation fails.
41 : */
42 : typedef struct cb_cleanup_dir
43 : {
44 : char *target_path;
45 : bool rmtopdir;
46 : struct cb_cleanup_dir *next;
47 : } cb_cleanup_dir;
48 :
49 : /*
50 : * Stores a tablespace mapping provided using -T, --tablespace-mapping.
51 : */
52 : typedef struct cb_tablespace_mapping
53 : {
54 : char old_dir[MAXPGPATH];
55 : char new_dir[MAXPGPATH];
56 : struct cb_tablespace_mapping *next;
57 : } cb_tablespace_mapping;
58 :
59 : /*
60 : * Stores data parsed from all command-line options.
61 : */
62 : typedef struct cb_options
63 : {
64 : bool debug;
65 : char *output;
66 : bool dry_run;
67 : bool no_sync;
68 : cb_tablespace_mapping *tsmappings;
69 : pg_checksum_type manifest_checksums;
70 : bool no_manifest;
71 : DataDirSyncMethod sync_method;
72 : CopyMethod copy_method;
73 : } cb_options;
74 :
75 : /*
76 : * Data about a tablespace.
77 : *
78 : * Every normal tablespace needs a tablespace mapping, but in-place tablespaces
79 : * don't, so the list of tablespaces can contain more entries than the list of
80 : * tablespace mappings.
81 : */
82 : typedef struct cb_tablespace
83 : {
84 : Oid oid;
85 : bool in_place;
86 : char old_dir[MAXPGPATH];
87 : char new_dir[MAXPGPATH];
88 : struct cb_tablespace *next;
89 : } cb_tablespace;
90 :
91 : /* Directories to be removed if we exit uncleanly. */
92 : cb_cleanup_dir *cleanup_dir_list = NULL;
93 :
94 : static void add_tablespace_mapping(cb_options *opt, char *arg);
95 : static StringInfo check_backup_label_files(int n_backups, char **backup_dirs);
96 : static uint64 check_control_files(int n_backups, char **backup_dirs);
97 : static void check_input_dir_permissions(char *dir);
98 : static void cleanup_directories_atexit(void);
99 : static void create_output_directory(char *dirname, cb_options *opt);
100 : static void help(const char *progname);
101 : static bool parse_oid(char *s, Oid *result);
102 : static void process_directory_recursively(Oid tsoid,
103 : char *input_directory,
104 : char *output_directory,
105 : char *relative_path,
106 : int n_prior_backups,
107 : char **prior_backup_dirs,
108 : manifest_data **manifests,
109 : manifest_writer *mwriter,
110 : cb_options *opt);
111 : static int read_pg_version_file(char *directory);
112 : static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
113 : static void reset_directory_cleanup_list(void);
114 : static cb_tablespace *scan_for_existing_tablespaces(char *pathname,
115 : cb_options *opt);
116 : static void slurp_file(int fd, char *filename, StringInfo buf, int maxlen);
117 :
118 : /*
119 : * Main program.
120 : */
121 : int
122 44 : main(int argc, char *argv[])
123 : {
124 : static struct option long_options[] = {
125 : {"debug", no_argument, NULL, 'd'},
126 : {"dry-run", no_argument, NULL, 'n'},
127 : {"no-sync", no_argument, NULL, 'N'},
128 : {"output", required_argument, NULL, 'o'},
129 : {"tablespace-mapping", required_argument, NULL, 'T'},
130 : {"manifest-checksums", required_argument, NULL, 1},
131 : {"no-manifest", no_argument, NULL, 2},
132 : {"sync-method", required_argument, NULL, 3},
133 : {"clone", no_argument, NULL, 4},
134 : {"copy-file-range", no_argument, NULL, 5},
135 : {NULL, 0, NULL, 0}
136 : };
137 :
138 : const char *progname;
139 : char *last_input_dir;
140 : int i;
141 : int optindex;
142 : int c;
143 : int n_backups;
144 : int n_prior_backups;
145 : int version;
146 : uint64 system_identifier;
147 : char **prior_backup_dirs;
148 : cb_options opt;
149 : cb_tablespace *tablespaces;
150 : cb_tablespace *ts;
151 : StringInfo last_backup_label;
152 : manifest_data **manifests;
153 : manifest_writer *mwriter;
154 :
155 44 : pg_logging_init(argv[0]);
156 44 : progname = get_progname(argv[0]);
157 44 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_combinebackup"));
158 44 : handle_help_version_opts(argc, argv, progname, help);
159 :
160 40 : memset(&opt, 0, sizeof(opt));
161 40 : opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
162 40 : opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
163 40 : opt.copy_method = COPY_METHOD_COPY;
164 :
165 : /* process command-line options */
166 96 : while ((c = getopt_long(argc, argv, "dnNo:T:",
167 : long_options, &optindex)) != -1)
168 : {
169 58 : switch (c)
170 : {
171 8 : case 'd':
172 8 : opt.debug = true;
173 8 : pg_logging_increase_verbosity();
174 8 : break;
175 0 : case 'n':
176 0 : opt.dry_run = true;
177 0 : break;
178 6 : case 'N':
179 6 : opt.no_sync = true;
180 6 : break;
181 34 : case 'o':
182 34 : opt.output = optarg;
183 34 : break;
184 2 : case 'T':
185 2 : add_tablespace_mapping(&opt, optarg);
186 2 : break;
187 4 : case 1:
188 4 : if (!pg_checksum_parse_type(optarg,
189 : &opt.manifest_checksums))
190 0 : pg_fatal("unrecognized checksum algorithm: \"%s\"",
191 : optarg);
192 4 : break;
193 2 : case 2:
194 2 : opt.no_manifest = true;
195 2 : break;
196 0 : case 3:
197 0 : if (!parse_sync_method(optarg, &opt.sync_method))
198 0 : exit(1);
199 0 : break;
200 0 : case 4:
201 0 : opt.copy_method = COPY_METHOD_CLONE;
202 0 : break;
203 0 : case 5:
204 0 : opt.copy_method = COPY_METHOD_COPY_FILE_RANGE;
205 0 : break;
206 2 : default:
207 : /* getopt_long already emitted a complaint */
208 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
209 2 : exit(1);
210 : }
211 : }
212 :
213 38 : if (optind >= argc)
214 : {
215 2 : pg_log_error("%s: no input directories specified", progname);
216 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
217 2 : exit(1);
218 : }
219 :
220 36 : if (opt.output == NULL)
221 2 : pg_fatal("no output directory specified");
222 :
223 : /* If no manifest is needed, no checksums are needed, either. */
224 34 : if (opt.no_manifest)
225 2 : opt.manifest_checksums = CHECKSUM_TYPE_NONE;
226 :
227 : /* Check that the platform supports the requested copy method. */
228 34 : if (opt.copy_method == COPY_METHOD_CLONE)
229 : {
230 : #if (defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)) || \
231 : (defined(__linux__) && defined(FICLONE))
232 :
233 : if (opt.dry_run)
234 : pg_log_debug("would use cloning to copy files");
235 : else
236 : pg_log_debug("will use cloning to copy files");
237 :
238 : #else
239 0 : pg_fatal("file cloning not supported on this platform");
240 : #endif
241 : }
242 34 : else if (opt.copy_method == COPY_METHOD_COPY_FILE_RANGE)
243 : {
244 : #if defined(HAVE_COPY_FILE_RANGE)
245 :
246 0 : if (opt.dry_run)
247 0 : pg_log_debug("would use copy_file_range to copy blocks");
248 : else
249 0 : pg_log_debug("will use copy_file_range to copy blocks");
250 :
251 : #else
252 : pg_fatal("copy_file_range not supported on this platform");
253 : #endif
254 : }
255 :
256 : /* Read the server version from the final backup. */
257 34 : version = read_pg_version_file(argv[argc - 1]);
258 :
259 : /* Sanity-check control files. */
260 34 : n_backups = argc - optind;
261 34 : system_identifier = check_control_files(n_backups, argv + optind);
262 :
263 : /* Sanity-check backup_label files, and get the contents of the last one. */
264 32 : last_backup_label = check_backup_label_files(n_backups, argv + optind);
265 :
266 : /*
267 : * We'll need the pathnames to the prior backups. By "prior" we mean all
268 : * but the last one listed on the command line.
269 : */
270 22 : n_prior_backups = argc - optind - 1;
271 22 : prior_backup_dirs = argv + optind;
272 :
273 : /* Load backup manifests. */
274 22 : manifests = load_backup_manifests(n_backups, prior_backup_dirs);
275 :
276 : /*
277 : * Validate the manifest system identifier against the backup system
278 : * identifier.
279 : */
280 62 : for (i = 0; i < n_backups; i++)
281 : {
282 42 : if (manifests[i] &&
283 42 : manifests[i]->system_identifier != system_identifier)
284 : {
285 : char *controlpath;
286 :
287 2 : controlpath = psprintf("%s/%s", prior_backup_dirs[i], "global/pg_control");
288 :
289 2 : pg_fatal("%s: manifest system identifier is %llu, but control file has %llu",
290 : controlpath,
291 : (unsigned long long) manifests[i]->system_identifier,
292 : (unsigned long long) system_identifier);
293 : }
294 : }
295 :
296 : /* Figure out which tablespaces are going to be included in the output. */
297 20 : last_input_dir = argv[argc - 1];
298 20 : check_input_dir_permissions(last_input_dir);
299 20 : tablespaces = scan_for_existing_tablespaces(last_input_dir, &opt);
300 :
301 : /*
302 : * Create output directories.
303 : *
304 : * We create one output directory for the main data directory plus one for
305 : * each non-in-place tablespace. create_output_directory() will arrange
306 : * for those directories to be cleaned up on failure. In-place tablespaces
307 : * aren't handled at this stage because they're located beneath the main
308 : * output directory, and thus the cleanup of that directory will get rid
309 : * of them. Plus, the pg_tblspc directory that needs to contain them
310 : * doesn't exist yet.
311 : */
312 20 : atexit(cleanup_directories_atexit);
313 20 : create_output_directory(opt.output, &opt);
314 22 : for (ts = tablespaces; ts != NULL; ts = ts->next)
315 2 : if (!ts->in_place)
316 2 : create_output_directory(ts->new_dir, &opt);
317 :
318 : /* If we need to write a backup_manifest, prepare to do so. */
319 20 : if (!opt.dry_run && !opt.no_manifest)
320 : {
321 18 : mwriter = create_manifest_writer(opt.output, system_identifier);
322 :
323 : /*
324 : * Verify that we have a backup manifest for the final backup; else we
325 : * won't have the WAL ranges for the resulting manifest.
326 : */
327 18 : if (manifests[n_prior_backups] == NULL)
328 0 : pg_fatal("can't generate a manifest because no manifest is available for the final input backup");
329 : }
330 : else
331 2 : mwriter = NULL;
332 :
333 : /* Write backup label into output directory. */
334 20 : if (opt.dry_run)
335 0 : pg_log_debug("would generate \"%s/backup_label\"", opt.output);
336 : else
337 : {
338 20 : pg_log_debug("generating \"%s/backup_label\"", opt.output);
339 20 : last_backup_label->cursor = 0;
340 20 : write_backup_label(opt.output, last_backup_label,
341 : opt.manifest_checksums, mwriter);
342 : }
343 :
344 : /* Process everything that's not part of a user-defined tablespace. */
345 20 : pg_log_debug("processing backup directory \"%s\"", last_input_dir);
346 20 : process_directory_recursively(InvalidOid, last_input_dir, opt.output,
347 : NULL, n_prior_backups, prior_backup_dirs,
348 : manifests, mwriter, &opt);
349 :
350 : /* Process user-defined tablespaces. */
351 22 : for (ts = tablespaces; ts != NULL; ts = ts->next)
352 : {
353 2 : pg_log_debug("processing tablespace directory \"%s\"", ts->old_dir);
354 :
355 : /*
356 : * If it's a normal tablespace, we need to set up a symbolic link from
357 : * pg_tblspc/${OID} to the target directory; if it's an in-place
358 : * tablespace, we need to create a directory at pg_tblspc/${OID}.
359 : */
360 2 : if (!ts->in_place)
361 : {
362 : char linkpath[MAXPGPATH];
363 :
364 2 : snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output,
365 : ts->oid);
366 :
367 2 : if (opt.dry_run)
368 0 : pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
369 : linkpath, ts->new_dir);
370 : else
371 : {
372 2 : pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
373 : linkpath, ts->new_dir);
374 2 : if (symlink(ts->new_dir, linkpath) != 0)
375 0 : pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
376 : linkpath, ts->new_dir);
377 : }
378 : }
379 : else
380 : {
381 0 : if (opt.dry_run)
382 0 : pg_log_debug("would create directory \"%s\"", ts->new_dir);
383 : else
384 : {
385 0 : pg_log_debug("creating directory \"%s\"", ts->new_dir);
386 0 : if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
387 0 : pg_fatal("could not create directory \"%s\": %m",
388 : ts->new_dir);
389 : }
390 : }
391 :
392 : /* OK, now handle the directory contents. */
393 2 : process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
394 : NULL, n_prior_backups, prior_backup_dirs,
395 : manifests, mwriter, &opt);
396 : }
397 :
398 : /* Finalize the backup_manifest, if we're generating one. */
399 20 : if (mwriter != NULL)
400 18 : finalize_manifest(mwriter,
401 18 : manifests[n_prior_backups]->first_wal_range);
402 :
403 : /* fsync that output directory unless we've been told not to do so */
404 20 : if (!opt.no_sync)
405 : {
406 14 : if (opt.dry_run)
407 0 : pg_log_debug("would recursively fsync \"%s\"", opt.output);
408 : else
409 : {
410 14 : pg_log_debug("recursively fsyncing \"%s\"", opt.output);
411 14 : sync_pgdata(opt.output, version * 10000, opt.sync_method);
412 : }
413 : }
414 :
415 : /* It's a success, so don't remove the output directories. */
416 20 : reset_directory_cleanup_list();
417 20 : exit(0);
418 : }
419 :
420 : /*
421 : * Process the option argument for the -T, --tablespace-mapping switch.
422 : */
423 : static void
424 2 : add_tablespace_mapping(cb_options *opt, char *arg)
425 : {
426 2 : cb_tablespace_mapping *tsmap = pg_malloc0(sizeof(cb_tablespace_mapping));
427 : char *dst;
428 : char *dst_ptr;
429 : char *arg_ptr;
430 :
431 : /*
432 : * Basically, we just want to copy everything before the equals sign to
433 : * tsmap->old_dir and everything afterwards to tsmap->new_dir, but if
434 : * there's more or less than one equals sign, that's an error, and if
435 : * there's an equals sign preceded by a backslash, don't treat it as a
436 : * field separator but instead copy a literal equals sign.
437 : */
438 2 : dst_ptr = dst = tsmap->old_dir;
439 100 : for (arg_ptr = arg; *arg_ptr != '\0'; arg_ptr++)
440 : {
441 98 : if (dst_ptr - dst >= MAXPGPATH)
442 0 : pg_fatal("directory name too long");
443 :
444 98 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
445 : ; /* skip backslash escaping = */
446 98 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
447 : {
448 2 : if (tsmap->new_dir[0] != '\0')
449 0 : pg_fatal("multiple \"=\" signs in tablespace mapping");
450 : else
451 2 : dst = dst_ptr = tsmap->new_dir;
452 : }
453 : else
454 96 : *dst_ptr++ = *arg_ptr;
455 : }
456 2 : if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
457 0 : pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
458 :
459 : /*
460 : * All tablespaces are created with absolute directories, so specifying a
461 : * non-absolute path here would never match, possibly confusing users.
462 : *
463 : * In contrast to pg_basebackup, both the old and new directories are on
464 : * the local machine, so the local machine's definition of an absolute
465 : * path is the only relevant one.
466 : */
467 2 : if (!is_absolute_path(tsmap->old_dir))
468 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
469 : tsmap->old_dir);
470 :
471 2 : if (!is_absolute_path(tsmap->new_dir))
472 0 : pg_fatal("old directory is not an absolute path in tablespace mapping: %s",
473 : tsmap->new_dir);
474 :
475 : /* Canonicalize paths to avoid spurious failures when comparing. */
476 2 : canonicalize_path(tsmap->old_dir);
477 2 : canonicalize_path(tsmap->new_dir);
478 :
479 : /* Add it to the list. */
480 2 : tsmap->next = opt->tsmappings;
481 2 : opt->tsmappings = tsmap;
482 2 : }
483 :
484 : /*
485 : * Check that the backup_label files form a coherent backup chain, and return
486 : * the contents of the backup_label file from the latest backup.
487 : */
488 : static StringInfo
489 32 : check_backup_label_files(int n_backups, char **backup_dirs)
490 : {
491 32 : StringInfo buf = makeStringInfo();
492 32 : StringInfo lastbuf = buf;
493 : int i;
494 32 : TimeLineID check_tli = 0;
495 32 : XLogRecPtr check_lsn = InvalidXLogRecPtr;
496 :
497 : /* Try to read each backup_label file in turn, last to first. */
498 84 : for (i = n_backups - 1; i >= 0; --i)
499 : {
500 : char pathbuf[MAXPGPATH];
501 : int fd;
502 : TimeLineID start_tli;
503 : TimeLineID previous_tli;
504 : XLogRecPtr start_lsn;
505 : XLogRecPtr previous_lsn;
506 :
507 : /* Open the backup_label file. */
508 62 : snprintf(pathbuf, MAXPGPATH, "%s/backup_label", backup_dirs[i]);
509 62 : pg_log_debug("reading \"%s\"", pathbuf);
510 62 : if ((fd = open(pathbuf, O_RDONLY, 0)) < 0)
511 0 : pg_fatal("could not open file \"%s\": %m", pathbuf);
512 :
513 : /*
514 : * Slurp the whole file into memory.
515 : *
516 : * The exact size limit that we impose here doesn't really matter --
517 : * most of what's supposed to be in the file is fixed size and quite
518 : * short. However, the length of the backup_label is limited (at least
519 : * by some parts of the code) to MAXPGPATH, so include that value in
520 : * the maximum length that we tolerate.
521 : */
522 62 : slurp_file(fd, pathbuf, buf, 10000 + MAXPGPATH);
523 :
524 : /* Close the file. */
525 62 : if (close(fd) != 0)
526 0 : pg_fatal("could not close \"%s\": %m", pathbuf);
527 :
528 : /* Parse the file contents. */
529 62 : parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
530 : &previous_tli, &previous_lsn);
531 :
532 : /*
533 : * Sanity checks.
534 : *
535 : * XXX. It's actually not required that start_lsn == check_lsn. It
536 : * would be OK if start_lsn > check_lsn provided that start_lsn is
537 : * less than or equal to the relevant switchpoint. But at the moment
538 : * we don't have that information.
539 : */
540 62 : if (i > 0 && previous_tli == 0)
541 2 : pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
542 : backup_dirs[i]);
543 60 : if (i == 0 && previous_tli != 0)
544 2 : pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
545 : backup_dirs[i]);
546 58 : if (i < n_backups - 1 && start_tli != check_tli)
547 0 : pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
548 : backup_dirs[i], start_tli, check_tli);
549 58 : if (i < n_backups - 1 && start_lsn != check_lsn)
550 6 : pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
551 : backup_dirs[i],
552 : LSN_FORMAT_ARGS(start_lsn),
553 : LSN_FORMAT_ARGS(check_lsn));
554 52 : check_tli = previous_tli;
555 52 : check_lsn = previous_lsn;
556 :
557 : /*
558 : * The last backup label in the chain needs to be saved for later use,
559 : * while the others are only needed within this loop.
560 : */
561 52 : if (lastbuf == buf)
562 30 : buf = makeStringInfo();
563 : else
564 22 : resetStringInfo(buf);
565 : }
566 :
567 : /* Free memory that we don't need any more. */
568 22 : if (lastbuf != buf)
569 22 : destroyStringInfo(buf);
570 :
571 : /*
572 : * Return the data from the first backup_info that we read (which is the
573 : * backup_label from the last directory specified on the command line).
574 : */
575 22 : return lastbuf;
576 : }
577 :
578 : /*
579 : * Sanity check control files and return system_identifier.
580 : */
581 : static uint64
582 34 : check_control_files(int n_backups, char **backup_dirs)
583 : {
584 : int i;
585 34 : uint64 system_identifier = 0; /* placate compiler */
586 34 : uint32 data_checksum_version = 0; /* placate compiler */
587 34 : bool data_checksum_mismatch = false;
588 :
589 : /* Try to read each control file in turn, last to first. */
590 102 : for (i = n_backups - 1; i >= 0; --i)
591 : {
592 : ControlFileData *control_file;
593 : bool crc_ok;
594 : char *controlpath;
595 :
596 70 : controlpath = psprintf("%s/%s", backup_dirs[i], "global/pg_control");
597 70 : pg_log_debug("reading \"%s\"", controlpath);
598 70 : control_file = get_controlfile_by_exact_path(controlpath, &crc_ok);
599 :
600 : /* Control file contents not meaningful if CRC is bad. */
601 70 : if (!crc_ok)
602 0 : pg_fatal("%s: CRC is incorrect", controlpath);
603 :
604 : /* Can't interpret control file if not current version. */
605 70 : if (control_file->pg_control_version != PG_CONTROL_VERSION)
606 0 : pg_fatal("%s: unexpected control file version",
607 : controlpath);
608 :
609 : /* System identifiers should all match. */
610 70 : if (i == n_backups - 1)
611 34 : system_identifier = control_file->system_identifier;
612 36 : else if (system_identifier != control_file->system_identifier)
613 2 : pg_fatal("%s: expected system identifier %llu, but found %llu",
614 : controlpath, (unsigned long long) system_identifier,
615 : (unsigned long long) control_file->system_identifier);
616 :
617 : /*
618 : * Detect checksum mismatches, but only if the last backup in the
619 : * chain has checksums enabled.
620 : */
621 68 : if (i == n_backups - 1)
622 34 : data_checksum_version = control_file->data_checksum_version;
623 34 : else if (data_checksum_version != 0 &&
624 0 : data_checksum_version != control_file->data_checksum_version)
625 0 : data_checksum_mismatch = true;
626 :
627 : /* Release memory. */
628 68 : pfree(control_file);
629 68 : pfree(controlpath);
630 : }
631 :
632 : /*
633 : * If debug output is enabled, make a note of the system identifier that
634 : * we found in all of the relevant control files.
635 : */
636 32 : pg_log_debug("system identifier is %llu",
637 : (unsigned long long) system_identifier);
638 :
639 : /*
640 : * Warn the user if not all backups are in the same state with regards to
641 : * checksums.
642 : */
643 32 : if (data_checksum_mismatch)
644 : {
645 0 : pg_log_warning("only some backups have checksums enabled");
646 0 : pg_log_warning_hint("disable, and optionally reenable, checksums on the output directory to avoid failures");
647 : }
648 :
649 32 : return system_identifier;
650 : }
651 :
652 : /*
653 : * Set default permissions for new files and directories based on the
654 : * permissions of the given directory. The intent here is that the output
655 : * directory should use the same permissions scheme as the final input
656 : * directory.
657 : */
658 : static void
659 20 : check_input_dir_permissions(char *dir)
660 : {
661 : struct stat st;
662 :
663 20 : if (stat(dir, &st) != 0)
664 0 : pg_fatal("could not stat \"%s\": %m", dir);
665 :
666 20 : SetDataDirectoryCreatePerm(st.st_mode);
667 20 : }
668 :
669 : /*
670 : * Clean up output directories before exiting.
671 : */
672 : static void
673 20 : cleanup_directories_atexit(void)
674 : {
675 20 : while (cleanup_dir_list != NULL)
676 : {
677 0 : cb_cleanup_dir *dir = cleanup_dir_list;
678 :
679 0 : if (dir->rmtopdir)
680 : {
681 0 : pg_log_info("removing output directory \"%s\"", dir->target_path);
682 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
683 0 : pg_log_error("failed to remove output directory");
684 : }
685 : else
686 : {
687 0 : pg_log_info("removing contents of output directory \"%s\"",
688 : dir->target_path);
689 0 : if (!rmtree(dir->target_path, dir->rmtopdir))
690 0 : pg_log_error("failed to remove contents of output directory");
691 : }
692 :
693 0 : cleanup_dir_list = cleanup_dir_list->next;
694 0 : pfree(dir);
695 : }
696 20 : }
697 :
698 : /*
699 : * Create the named output directory, unless it already exists or we're in
700 : * dry-run mode. If it already exists but is not empty, that's a fatal error.
701 : *
702 : * Adds the created directory to the list of directories to be cleaned up
703 : * at process exit.
704 : */
705 : static void
706 22 : create_output_directory(char *dirname, cb_options *opt)
707 : {
708 22 : switch (pg_check_dir(dirname))
709 : {
710 22 : case 0:
711 22 : if (opt->dry_run)
712 : {
713 0 : pg_log_debug("would create directory \"%s\"", dirname);
714 0 : return;
715 : }
716 22 : pg_log_debug("creating directory \"%s\"", dirname);
717 22 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
718 0 : pg_fatal("could not create directory \"%s\": %m", dirname);
719 22 : remember_to_cleanup_directory(dirname, true);
720 22 : break;
721 :
722 0 : case 1:
723 0 : pg_log_debug("using existing directory \"%s\"", dirname);
724 0 : remember_to_cleanup_directory(dirname, false);
725 0 : break;
726 :
727 0 : case 2:
728 : case 3:
729 : case 4:
730 0 : pg_fatal("directory \"%s\" exists but is not empty", dirname);
731 :
732 0 : case -1:
733 0 : pg_fatal("could not access directory \"%s\": %m", dirname);
734 : }
735 : }
736 :
737 : /*
738 : * help
739 : *
740 : * Prints help page for the program
741 : *
742 : * progname: the name of the executed program, such as "pg_combinebackup"
743 : */
744 : static void
745 2 : help(const char *progname)
746 : {
747 2 : printf(_("%s reconstructs full backups from incrementals.\n\n"), progname);
748 2 : printf(_("Usage:\n"));
749 2 : printf(_(" %s [OPTION]... DIRECTORY...\n"), progname);
750 2 : printf(_("\nOptions:\n"));
751 2 : printf(_(" -d, --debug generate lots of debugging output\n"));
752 2 : printf(_(" -n, --dry-run do not actually do anything\n"));
753 2 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
754 2 : printf(_(" -o, --output output directory\n"));
755 2 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
756 : " relocate tablespace in OLDDIR to NEWDIR\n"));
757 2 : printf(_(" --clone clone (reflink) instead of copying files\n"));
758 2 : printf(_(" --copy-file-range copy using copy_file_range() syscall\n"));
759 2 : printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
760 : " use algorithm for manifest checksums\n"));
761 2 : printf(_(" --no-manifest suppress generation of backup manifest\n"));
762 2 : printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
763 2 : printf(_(" -V, --version output version information, then exit\n"));
764 2 : printf(_(" -?, --help show this help, then exit\n"));
765 :
766 2 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
767 2 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
768 2 : }
769 :
770 : /*
771 : * Try to parse a string as a non-zero OID without leading zeroes.
772 : *
773 : * If it works, return true and set *result to the answer, else return false.
774 : */
775 : static bool
776 4 : parse_oid(char *s, Oid *result)
777 : {
778 : Oid oid;
779 : char *ep;
780 :
781 4 : errno = 0;
782 4 : oid = strtoul(s, &ep, 10);
783 4 : if (errno != 0 || *ep != '\0' || oid < 1 || oid > PG_UINT32_MAX)
784 0 : return false;
785 :
786 4 : *result = oid;
787 4 : return true;
788 : }
789 :
790 : /*
791 : * Copy files from the input directory to the output directory, reconstructing
792 : * full files from incremental files as required.
793 : *
794 : * If processing is a user-defined tablespace, the tsoid should be the OID
795 : * of that tablespace and input_directory and output_directory should be the
796 : * toplevel input and output directories for that tablespace. Otherwise,
797 : * tsoid should be InvalidOid and input_directory and output_directory should
798 : * be the main input and output directories.
799 : *
800 : * relative_path is the path beneath the given input and output directories
801 : * that we are currently processing. If NULL, it indicates that we're
802 : * processing the input and output directories themselves.
803 : *
804 : * n_prior_backups is the number of prior backups that we have available.
805 : * This doesn't count the very last backup, which is referenced by
806 : * output_directory, just the older ones. prior_backup_dirs is an array of
807 : * the locations of those previous backups.
808 : */
809 : static void
810 550 : process_directory_recursively(Oid tsoid,
811 : char *input_directory,
812 : char *output_directory,
813 : char *relative_path,
814 : int n_prior_backups,
815 : char **prior_backup_dirs,
816 : manifest_data **manifests,
817 : manifest_writer *mwriter,
818 : cb_options *opt)
819 : {
820 : char ifulldir[MAXPGPATH];
821 : char ofulldir[MAXPGPATH];
822 : char manifest_prefix[MAXPGPATH];
823 : DIR *dir;
824 : struct dirent *de;
825 550 : bool is_pg_tblspc = false;
826 550 : bool is_pg_wal = false;
827 550 : bool is_incremental_dir = false;
828 550 : manifest_data *latest_manifest = manifests[n_prior_backups];
829 : pg_checksum_type checksum_type;
830 :
831 : /*
832 : * Classify this directory.
833 : *
834 : * We set is_pg_tblspc only for the toplevel pg_tblspc directory, because
835 : * the symlinks in that specific directory require special handling.
836 : *
837 : * We set is_pg_wal for the toplevel WAL directory and all of its
838 : * subdirectories, because those files are not included in the backup
839 : * manifest and hence need special treatement. (Since incremental backup
840 : * does not exist in pre-v10 versions, we don't have to worry about the
841 : * old pg_xlog naming.)
842 : *
843 : * We set is_incremental_dir for directories that can contain incremental
844 : * files requiring reconstruction. If such files occur outside these
845 : * directories, we want to just copy them straight to the output
846 : * directory. This is to protect against a user creating a file with a
847 : * strange name like INCREMENTAL.config and then complaining that
848 : * incremental backups don't work properly. The test here is a bit tricky:
849 : * incremental files occur in subdirectories of base, in pg_global itself,
850 : * and in subdirectories of pg_tblspc only if in-place tablespaces are
851 : * used.
852 : */
853 550 : if (OidIsValid(tsoid))
854 6 : is_incremental_dir = true;
855 544 : else if (relative_path != NULL)
856 : {
857 524 : is_pg_tblspc = strcmp(relative_path, "pg_tblspc") == 0;
858 1028 : is_pg_wal = (strcmp(relative_path, "pg_wal") == 0 ||
859 504 : strncmp(relative_path, "pg_wal/", 7) == 0);
860 524 : is_incremental_dir = strncmp(relative_path, "base/", 5) == 0 ||
861 964 : strcmp(relative_path, "global") == 0 ||
862 440 : strncmp(relative_path, "pg_tblspc/", 10) == 0;
863 : }
864 :
865 : /*
866 : * If we're under pg_wal, then we don't need checksums, because these
867 : * files aren't included in the backup manifest. Otherwise use whatever
868 : * type of checksum is configured.
869 : */
870 550 : if (!is_pg_wal)
871 490 : checksum_type = opt->manifest_checksums;
872 : else
873 60 : checksum_type = CHECKSUM_TYPE_NONE;
874 :
875 : /*
876 : * Append the relative path to the input and output directories, and
877 : * figure out the appropriate prefix to add to files in this directory
878 : * when looking them up in a backup manifest.
879 : */
880 550 : if (relative_path == NULL)
881 : {
882 22 : strlcpy(ifulldir, input_directory, MAXPGPATH);
883 22 : strlcpy(ofulldir, output_directory, MAXPGPATH);
884 22 : if (OidIsValid(tsoid))
885 2 : snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/", tsoid);
886 : else
887 20 : manifest_prefix[0] = '\0';
888 : }
889 : else
890 : {
891 528 : snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
892 : relative_path);
893 528 : snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
894 : relative_path);
895 528 : if (OidIsValid(tsoid))
896 4 : snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/",
897 : tsoid, relative_path);
898 : else
899 524 : snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
900 : }
901 :
902 : /*
903 : * Toplevel output directories have already been created by the time this
904 : * function is called, but any subdirectories are our responsibility.
905 : */
906 550 : if (relative_path != NULL)
907 : {
908 528 : if (opt->dry_run)
909 0 : pg_log_debug("would create directory \"%s\"", ofulldir);
910 : else
911 : {
912 528 : pg_log_debug("creating directory \"%s\"", ofulldir);
913 528 : if (mkdir(ofulldir, pg_dir_create_mode) == -1)
914 0 : pg_fatal("could not create directory \"%s\": %m", ofulldir);
915 : }
916 : }
917 :
918 : /* It's time to scan the directory. */
919 550 : if ((dir = opendir(ifulldir)) == NULL)
920 0 : pg_fatal("could not open directory \"%s\": %m", ifulldir);
921 22816 : while (errno = 0, (de = readdir(dir)) != NULL)
922 : {
923 : PGFileType type;
924 : char ifullpath[MAXPGPATH];
925 : char ofullpath[MAXPGPATH];
926 : char manifest_path[MAXPGPATH];
927 22266 : Oid oid = InvalidOid;
928 22266 : int checksum_length = 0;
929 22266 : uint8 *checksum_payload = NULL;
930 : pg_checksum_context checksum_ctx;
931 :
932 : /* Ignore "." and ".." entries. */
933 22266 : if (strcmp(de->d_name, ".") == 0 ||
934 21716 : strcmp(de->d_name, "..") == 0)
935 1670 : continue;
936 :
937 : /* Construct input path. */
938 21166 : snprintf(ifullpath, MAXPGPATH, "%s/%s", ifulldir, de->d_name);
939 :
940 : /* Figure out what kind of directory entry this is. */
941 21166 : type = get_dirent_type(ifullpath, de, false, PG_LOG_ERROR);
942 21166 : if (type == PGFILETYPE_ERROR)
943 0 : exit(1);
944 :
945 : /*
946 : * If we're processing pg_tblspc, then check whether the filename
947 : * looks like it could be a tablespace OID. If so, and if the
948 : * directory entry is a symbolic link or a directory, skip it.
949 : *
950 : * Our goal here is to ignore anything that would have been considered
951 : * by scan_for_existing_tablespaces to be a tablespace.
952 : */
953 21166 : if (is_pg_tblspc && parse_oid(de->d_name, &oid) &&
954 0 : (type == PGFILETYPE_LNK || type == PGFILETYPE_DIR))
955 2 : continue;
956 :
957 : /* If it's a directory, recurse. */
958 21164 : if (type == PGFILETYPE_DIR)
959 : {
960 : char new_relative_path[MAXPGPATH];
961 :
962 : /* Append new pathname component to relative path. */
963 528 : if (relative_path == NULL)
964 342 : strlcpy(new_relative_path, de->d_name, MAXPGPATH);
965 : else
966 186 : snprintf(new_relative_path, MAXPGPATH, "%s/%s", relative_path,
967 186 : de->d_name);
968 :
969 : /* And recurse. */
970 528 : process_directory_recursively(tsoid,
971 : input_directory, output_directory,
972 : new_relative_path,
973 : n_prior_backups, prior_backup_dirs,
974 : manifests, mwriter, opt);
975 528 : continue;
976 : }
977 :
978 : /* Skip anything that's not a regular file. */
979 20636 : if (type != PGFILETYPE_REG)
980 : {
981 0 : if (type == PGFILETYPE_LNK)
982 0 : pg_log_warning("skipping symbolic link \"%s\"", ifullpath);
983 : else
984 0 : pg_log_warning("skipping special file \"%s\"", ifullpath);
985 0 : continue;
986 : }
987 :
988 : /*
989 : * Skip the backup_label and backup_manifest files; they require
990 : * special handling and are handled elsewhere.
991 : */
992 20636 : if (relative_path == NULL &&
993 148 : (strcmp(de->d_name, "backup_label") == 0 ||
994 128 : strcmp(de->d_name, "backup_manifest") == 0))
995 40 : continue;
996 :
997 : /*
998 : * If it's an incremental file, hand it off to the reconstruction
999 : * code, which will figure out what to do.
1000 : */
1001 20596 : if (is_incremental_dir &&
1002 20384 : strncmp(de->d_name, INCREMENTAL_PREFIX,
1003 : INCREMENTAL_PREFIX_LENGTH) == 0)
1004 : {
1005 : /* Output path should not include "INCREMENTAL." prefix. */
1006 9308 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
1007 9308 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1008 :
1009 :
1010 : /* Manifest path likewise omits incremental prefix. */
1011 9308 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1012 9308 : de->d_name + INCREMENTAL_PREFIX_LENGTH);
1013 :
1014 : /* Reconstruction logic will do the rest. */
1015 9308 : reconstruct_from_incremental_file(ifullpath, ofullpath,
1016 : manifest_prefix,
1017 9308 : de->d_name + INCREMENTAL_PREFIX_LENGTH,
1018 : n_prior_backups,
1019 : prior_backup_dirs,
1020 : manifests,
1021 : manifest_path,
1022 : checksum_type,
1023 : &checksum_length,
1024 : &checksum_payload,
1025 : opt->copy_method,
1026 9308 : opt->debug,
1027 9308 : opt->dry_run);
1028 : }
1029 : else
1030 : {
1031 : /* Construct the path that the backup_manifest will use. */
1032 11288 : snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
1033 11288 : de->d_name);
1034 :
1035 : /*
1036 : * It's not an incremental file, so we need to copy the entire
1037 : * file to the output directory.
1038 : *
1039 : * If a checksum of the required type already exists in the
1040 : * backup_manifest for the final input directory, we can save some
1041 : * work by reusing that checksum instead of computing a new one.
1042 : */
1043 11288 : if (checksum_type != CHECKSUM_TYPE_NONE &&
1044 : latest_manifest != NULL)
1045 : {
1046 : manifest_file *mfile;
1047 :
1048 7404 : mfile = manifest_files_lookup(latest_manifest->files,
1049 : manifest_path);
1050 7404 : if (mfile == NULL)
1051 : {
1052 : char *bmpath;
1053 :
1054 : /*
1055 : * The directory is out of sync with the backup_manifest,
1056 : * so emit a warning.
1057 : */
1058 0 : bmpath = psprintf("%s/%s", input_directory,
1059 : "backup_manifest");
1060 0 : pg_log_warning("\"%s\" contains no entry for \"%s\"",
1061 : bmpath, manifest_path);
1062 0 : pfree(bmpath);
1063 : }
1064 7404 : else if (mfile->checksum_type == checksum_type)
1065 : {
1066 5474 : checksum_length = mfile->checksum_length;
1067 5474 : checksum_payload = mfile->checksum_payload;
1068 : }
1069 : }
1070 :
1071 : /*
1072 : * If we're reusing a checksum, then we don't need copy_file() to
1073 : * compute one for us, but otherwise, it needs to compute whatever
1074 : * type of checksum we need.
1075 : */
1076 11288 : if (checksum_length != 0)
1077 5474 : pg_checksum_init(&checksum_ctx, CHECKSUM_TYPE_NONE);
1078 : else
1079 5814 : pg_checksum_init(&checksum_ctx, checksum_type);
1080 :
1081 : /* Actually copy the file. */
1082 11288 : snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
1083 11288 : copy_file(ifullpath, ofullpath, &checksum_ctx,
1084 11288 : opt->copy_method, opt->dry_run);
1085 :
1086 : /*
1087 : * If copy_file() performed a checksum calculation for us, then
1088 : * save the results (except in dry-run mode, when there's no
1089 : * point).
1090 : */
1091 11288 : if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
1092 : {
1093 1930 : checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
1094 1930 : checksum_length = pg_checksum_final(&checksum_ctx,
1095 : checksum_payload);
1096 : }
1097 : }
1098 :
1099 : /* Generate manifest entry, if needed. */
1100 20596 : if (mwriter != NULL)
1101 : {
1102 : struct stat sb;
1103 :
1104 : /*
1105 : * In order to generate a manifest entry, we need the file size
1106 : * and mtime. We have no way to know the correct mtime except to
1107 : * stat() the file, so just do that and get the size as well.
1108 : *
1109 : * If we didn't need the mtime here, we could try to obtain the
1110 : * file size from the reconstruction or file copy process above,
1111 : * although that is actually not convenient in all cases. If we
1112 : * write the file ourselves then clearly we can keep a count of
1113 : * bytes, but if we use something like CopyFile() then it's
1114 : * trickier. Since we have to stat() anyway to get the mtime,
1115 : * there's no point in worrying about it.
1116 : */
1117 18664 : if (stat(ofullpath, &sb) < 0)
1118 0 : pg_fatal("could not stat file \"%s\": %m", ofullpath);
1119 :
1120 : /* OK, now do the work. */
1121 18664 : add_file_to_manifest(mwriter, manifest_path,
1122 18664 : sb.st_size, sb.st_mtime,
1123 : checksum_type, checksum_length,
1124 : checksum_payload);
1125 : }
1126 :
1127 : /* Avoid leaking memory. */
1128 20596 : if (checksum_payload != NULL)
1129 16712 : pfree(checksum_payload);
1130 : }
1131 :
1132 550 : closedir(dir);
1133 550 : }
1134 :
1135 : /*
1136 : * Read the version number from PG_VERSION and convert it to the usual server
1137 : * version number format. (e.g. If PG_VERSION contains "14\n" this function
1138 : * will return 140000)
1139 : */
1140 : static int
1141 34 : read_pg_version_file(char *directory)
1142 : {
1143 : char filename[MAXPGPATH];
1144 : StringInfoData buf;
1145 : int fd;
1146 : int version;
1147 : char *ep;
1148 :
1149 : /* Construct pathname. */
1150 34 : snprintf(filename, MAXPGPATH, "%s/PG_VERSION", directory);
1151 :
1152 : /* Open file. */
1153 34 : if ((fd = open(filename, O_RDONLY, 0)) < 0)
1154 0 : pg_fatal("could not open file \"%s\": %m", filename);
1155 :
1156 : /* Read into memory. Length limit of 128 should be more than generous. */
1157 34 : initStringInfo(&buf);
1158 34 : slurp_file(fd, filename, &buf, 128);
1159 :
1160 : /* Close the file. */
1161 34 : if (close(fd) != 0)
1162 0 : pg_fatal("could not close \"%s\": %m", filename);
1163 :
1164 : /* Convert to integer. */
1165 34 : errno = 0;
1166 34 : version = strtoul(buf.data, &ep, 10);
1167 34 : if (errno != 0 || *ep != '\n')
1168 : {
1169 : /*
1170 : * Incremental backup is not relevant to very old server versions that
1171 : * used multi-part version number (e.g. 9.6, or 8.4). So if we see
1172 : * what looks like the beginning of such a version number, just bail
1173 : * out.
1174 : */
1175 0 : if (version < 10 && *ep == '.')
1176 0 : pg_fatal("%s: server version too old\n", filename);
1177 0 : pg_fatal("%s: could not parse version number\n", filename);
1178 : }
1179 :
1180 : /* Debugging output. */
1181 34 : pg_log_debug("read server version %d from \"%s\"", version, filename);
1182 :
1183 : /* Release memory and return result. */
1184 34 : pfree(buf.data);
1185 34 : return version * 10000;
1186 : }
1187 :
1188 : /*
1189 : * Add a directory to the list of output directories to clean up.
1190 : */
1191 : static void
1192 22 : remember_to_cleanup_directory(char *target_path, bool rmtopdir)
1193 : {
1194 22 : cb_cleanup_dir *dir = pg_malloc(sizeof(cb_cleanup_dir));
1195 :
1196 22 : dir->target_path = target_path;
1197 22 : dir->rmtopdir = rmtopdir;
1198 22 : dir->next = cleanup_dir_list;
1199 22 : cleanup_dir_list = dir;
1200 22 : }
1201 :
1202 : /*
1203 : * Empty out the list of directories scheduled for cleanup at exit.
1204 : *
1205 : * We want to remove the output directories only on a failure, so call this
1206 : * function when we know that the operation has succeeded.
1207 : *
1208 : * Since we only expect this to be called when we're about to exit, we could
1209 : * just set cleanup_dir_list to NULL and be done with it, but we free the
1210 : * memory to be tidy.
1211 : */
1212 : static void
1213 20 : reset_directory_cleanup_list(void)
1214 : {
1215 42 : while (cleanup_dir_list != NULL)
1216 : {
1217 22 : cb_cleanup_dir *dir = cleanup_dir_list;
1218 :
1219 22 : cleanup_dir_list = cleanup_dir_list->next;
1220 22 : pfree(dir);
1221 : }
1222 20 : }
1223 :
1224 : /*
1225 : * Scan the pg_tblspc directory of the final input backup to get a canonical
1226 : * list of what tablespaces are part of the backup.
1227 : *
1228 : * 'pathname' should be the path to the toplevel backup directory for the
1229 : * final backup in the backup chain.
1230 : */
1231 : static cb_tablespace *
1232 20 : scan_for_existing_tablespaces(char *pathname, cb_options *opt)
1233 : {
1234 : char pg_tblspc[MAXPGPATH];
1235 : DIR *dir;
1236 : struct dirent *de;
1237 20 : cb_tablespace *tslist = NULL;
1238 :
1239 20 : snprintf(pg_tblspc, MAXPGPATH, "%s/pg_tblspc", pathname);
1240 20 : pg_log_debug("scanning \"%s\"", pg_tblspc);
1241 :
1242 20 : if ((dir = opendir(pg_tblspc)) == NULL)
1243 0 : pg_fatal("could not open directory \"%s\": %m", pg_tblspc);
1244 :
1245 62 : while (errno = 0, (de = readdir(dir)) != NULL)
1246 : {
1247 : Oid oid;
1248 : char tblspcdir[MAXPGPATH];
1249 : char link_target[MAXPGPATH];
1250 : int link_length;
1251 : cb_tablespace *ts;
1252 : cb_tablespace *otherts;
1253 : PGFileType type;
1254 :
1255 : /* Silently ignore "." and ".." entries. */
1256 42 : if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
1257 40 : continue;
1258 :
1259 : /* Construct full pathname. */
1260 2 : snprintf(tblspcdir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1261 :
1262 : /* Ignore any file name that doesn't look like a proper OID. */
1263 2 : if (!parse_oid(de->d_name, &oid))
1264 : {
1265 0 : pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
1266 : tblspcdir);
1267 0 : continue;
1268 : }
1269 :
1270 : /* Only symbolic links and directories are tablespaces. */
1271 2 : type = get_dirent_type(tblspcdir, de, false, PG_LOG_ERROR);
1272 2 : if (type == PGFILETYPE_ERROR)
1273 0 : exit(1);
1274 2 : if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
1275 : {
1276 0 : pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
1277 : tblspcdir);
1278 0 : continue;
1279 : }
1280 :
1281 : /* Create a new tablespace object. */
1282 2 : ts = pg_malloc0(sizeof(cb_tablespace));
1283 2 : ts->oid = oid;
1284 :
1285 : /*
1286 : * If it's a link, it's not an in-place tablespace. Otherwise, it must
1287 : * be a directory, and thus an in-place tablespace.
1288 : */
1289 2 : if (type == PGFILETYPE_LNK)
1290 : {
1291 : cb_tablespace_mapping *tsmap;
1292 :
1293 : /* Read the link target. */
1294 2 : link_length = readlink(tblspcdir, link_target, sizeof(link_target));
1295 2 : if (link_length < 0)
1296 0 : pg_fatal("could not read symbolic link \"%s\": %m",
1297 : tblspcdir);
1298 2 : if (link_length >= sizeof(link_target))
1299 0 : pg_fatal("symbolic link \"%s\" is too long", tblspcdir);
1300 2 : link_target[link_length] = '\0';
1301 2 : if (!is_absolute_path(link_target))
1302 0 : pg_fatal("symbolic link \"%s\" is relative", tblspcdir);
1303 :
1304 : /* Canonicalize the link target. */
1305 2 : canonicalize_path(link_target);
1306 :
1307 : /*
1308 : * Find the corresponding tablespace mapping and copy the relevant
1309 : * details into the new tablespace entry.
1310 : */
1311 2 : for (tsmap = opt->tsmappings; tsmap != NULL; tsmap = tsmap->next)
1312 : {
1313 2 : if (strcmp(tsmap->old_dir, link_target) == 0)
1314 : {
1315 2 : strlcpy(ts->old_dir, tsmap->old_dir, MAXPGPATH);
1316 2 : strlcpy(ts->new_dir, tsmap->new_dir, MAXPGPATH);
1317 2 : ts->in_place = false;
1318 2 : break;
1319 : }
1320 : }
1321 :
1322 : /* Every non-in-place tablespace must be mapped. */
1323 2 : if (tsmap == NULL)
1324 0 : pg_fatal("tablespace at \"%s\" has no tablespace mapping",
1325 : link_target);
1326 : }
1327 : else
1328 : {
1329 : /*
1330 : * For an in-place tablespace, there's no separate directory, so
1331 : * we just record the paths within the data directories.
1332 : */
1333 0 : snprintf(ts->old_dir, MAXPGPATH, "%s/%s", pg_tblspc, de->d_name);
1334 0 : snprintf(ts->new_dir, MAXPGPATH, "%s/pg_tblspc/%s", opt->output,
1335 0 : de->d_name);
1336 0 : ts->in_place = true;
1337 : }
1338 :
1339 : /* Tablespaces should not share a directory. */
1340 2 : for (otherts = tslist; otherts != NULL; otherts = otherts->next)
1341 0 : if (strcmp(ts->new_dir, otherts->new_dir) == 0)
1342 0 : pg_fatal("tablespaces with OIDs %u and %u both point at \"%s\"",
1343 : otherts->oid, oid, ts->new_dir);
1344 :
1345 : /* Add this tablespace to the list. */
1346 2 : ts->next = tslist;
1347 2 : tslist = ts;
1348 : }
1349 :
1350 20 : if (closedir(dir) != 0)
1351 0 : pg_fatal("could not close directory \"%s\": %m", pg_tblspc);
1352 :
1353 20 : return tslist;
1354 : }
1355 :
1356 : /*
1357 : * Read a file into a StringInfo.
1358 : *
1359 : * fd is used for the actual file I/O, filename for error reporting purposes.
1360 : * A file longer than maxlen is a fatal error.
1361 : */
1362 : static void
1363 96 : slurp_file(int fd, char *filename, StringInfo buf, int maxlen)
1364 : {
1365 : struct stat st;
1366 : ssize_t rb;
1367 :
1368 : /* Check file size, and complain if it's too large. */
1369 96 : if (fstat(fd, &st) != 0)
1370 0 : pg_fatal("could not stat \"%s\": %m", filename);
1371 96 : if (st.st_size > maxlen)
1372 0 : pg_fatal("file \"%s\" is too large", filename);
1373 :
1374 : /* Make sure we have enough space. */
1375 96 : enlargeStringInfo(buf, st.st_size);
1376 :
1377 : /* Read the data. */
1378 96 : rb = read(fd, &buf->data[buf->len], st.st_size);
1379 :
1380 : /*
1381 : * We don't expect any concurrent changes, so we should read exactly the
1382 : * expected number of bytes.
1383 : */
1384 96 : if (rb != st.st_size)
1385 : {
1386 0 : if (rb < 0)
1387 0 : pg_fatal("could not read file \"%s\": %m", filename);
1388 : else
1389 0 : pg_fatal("could not read file \"%s\": read only %zd of %lld bytes",
1390 : filename, rb, (long long int) st.st_size);
1391 : }
1392 :
1393 : /* Adjust buffer length for new data and restore trailing-\0 invariant */
1394 96 : buf->len += rb;
1395 96 : buf->data[buf->len] = '\0';
1396 96 : }
|
