Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_basebackup.c - receive a base backup using streaming replication protocol
4 : *
5 : * Author: Magnus Hagander <magnus@hagander.net>
6 : *
7 : * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
8 : *
9 : * IDENTIFICATION
10 : * src/bin/pg_basebackup/pg_basebackup.c
11 : *-------------------------------------------------------------------------
12 : */
13 :
14 : #include "postgres_fe.h"
15 :
16 : #include <unistd.h>
17 : #include <dirent.h>
18 : #include <sys/stat.h>
19 : #include <sys/wait.h>
20 : #include <signal.h>
21 : #include <time.h>
22 : #ifdef HAVE_SYS_SELECT_H
23 : #include <sys/select.h>
24 : #endif
25 : #ifdef HAVE_LIBZ
26 : #include <zlib.h>
27 : #endif
28 :
29 : #include "access/xlog_internal.h"
30 : #include "common/file_perm.h"
31 : #include "common/file_utils.h"
32 : #include "common/logging.h"
33 : #include "common/string.h"
34 : #include "fe_utils/string_utils.h"
35 : #include "getopt_long.h"
36 : #include "libpq-fe.h"
37 : #include "pqexpbuffer.h"
38 : #include "pgtar.h"
39 : #include "pgtime.h"
40 : #include "receivelog.h"
41 : #include "replication/basebackup.h"
42 : #include "streamutil.h"
43 :
44 : #define ERRCODE_DATA_CORRUPTED "XX001"
45 :
46 : typedef struct TablespaceListCell
47 : {
48 : struct TablespaceListCell *next;
49 : char old_dir[MAXPGPATH];
50 : char new_dir[MAXPGPATH];
51 : } TablespaceListCell;
52 :
53 : typedef struct TablespaceList
54 : {
55 : TablespaceListCell *head;
56 : TablespaceListCell *tail;
57 : } TablespaceList;
58 :
59 : /*
60 : * pg_xlog has been renamed to pg_wal in version 10. This version number
61 : * should be compared with PQserverVersion().
62 : */
63 : #define MINIMUM_VERSION_FOR_PG_WAL 100000
64 :
65 : /*
66 : * Temporary replication slots are supported from version 10.
67 : */
68 : #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
69 :
70 : /*
71 : * recovery.conf is integrated into postgresql.conf from version 12.
72 : */
73 : #define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000
74 :
75 : /*
76 : * Different ways to include WAL
77 : */
78 : typedef enum
79 : {
80 : NO_WAL,
81 : FETCH_WAL,
82 : STREAM_WAL
83 : } IncludeWal;
84 :
85 : /* Global options */
86 : static char *basedir = NULL;
87 : static TablespaceList tablespace_dirs = {NULL, NULL};
88 : static char *xlog_dir = NULL;
89 : static char format = 'p'; /* p(lain)/t(ar) */
90 : static char *label = "pg_basebackup base backup";
91 : static bool noclean = false;
92 : static bool checksum_failure = false;
93 : static bool showprogress = false;
94 : static int verbose = 0;
95 : static int compresslevel = 0;
96 : static IncludeWal includewal = STREAM_WAL;
97 : static bool fastcheckpoint = false;
98 : static bool writerecoveryconf = false;
99 : static bool do_sync = true;
100 : static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
101 : static pg_time_t last_progress_report = 0;
102 : static int32 maxrate = 0; /* no limit by default */
103 : static char *replication_slot = NULL;
104 : static bool temp_replication_slot = true;
105 : static bool create_slot = false;
106 : static bool no_slot = false;
107 : static bool verify_checksums = true;
108 :
109 : static bool success = false;
110 : static bool made_new_pgdata = false;
111 : static bool found_existing_pgdata = false;
112 : static bool made_new_xlogdir = false;
113 : static bool found_existing_xlogdir = false;
114 : static bool made_tablespace_dirs = false;
115 : static bool found_tablespace_dirs = false;
116 :
117 : /* Progress counters */
118 : static uint64 totalsize;
119 : static uint64 totaldone;
120 : static int tablespacecount;
121 :
122 : /* Pipe to communicate with background wal receiver process */
123 : #ifndef WIN32
124 : static int bgpipe[2] = {-1, -1};
125 : #endif
126 :
127 : /* Handle to child process */
128 : static pid_t bgchild = -1;
129 : static bool in_log_streamer = false;
130 :
131 : /* End position for xlog streaming, empty string if unknown yet */
132 : static XLogRecPtr xlogendptr;
133 :
134 : #ifndef WIN32
135 : static int has_xlogendptr = 0;
136 : #else
137 : static volatile LONG has_xlogendptr = 0;
138 : #endif
139 :
140 : /* Contents of configuration file to be generated */
141 : static PQExpBuffer recoveryconfcontents = NULL;
142 :
143 : /* Function headers */
144 : static void usage(void);
145 : static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
146 : static void progress_report(int tablespacenum, const char *filename, bool force);
147 :
148 : static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
149 : static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
150 : static void GenerateRecoveryConf(PGconn *conn);
151 : static void WriteRecoveryConf(void);
152 : static void BaseBackup(void);
153 :
154 : static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
155 : bool segment_finished);
156 :
157 : static const char *get_tablespace_mapping(const char *dir);
158 : static void tablespace_list_append(const char *arg);
159 :
160 :
161 : static void
162 206 : cleanup_directories_atexit(void)
163 : {
164 206 : if (success || in_log_streamer)
165 166 : return;
166 :
167 40 : if (!noclean && !checksum_failure)
168 : {
169 32 : if (made_new_pgdata)
170 : {
171 8 : pg_log_info("removing data directory \"%s\"", basedir);
172 8 : if (!rmtree(basedir, true))
173 0 : pg_log_error("failed to remove data directory");
174 : }
175 24 : else if (found_existing_pgdata)
176 : {
177 0 : pg_log_info("removing contents of data directory \"%s\"", basedir);
178 0 : if (!rmtree(basedir, false))
179 0 : pg_log_error("failed to remove contents of data directory");
180 : }
181 :
182 64 : if (made_new_xlogdir)
183 : {
184 0 : pg_log_info("removing WAL directory \"%s\"", xlog_dir);
185 0 : if (!rmtree(xlog_dir, true))
186 0 : pg_log_error("failed to remove WAL directory");
187 : }
188 32 : else if (found_existing_xlogdir)
189 : {
190 0 : pg_log_info("removing contents of WAL directory \"%s\"", xlog_dir);
191 0 : if (!rmtree(xlog_dir, false))
192 0 : pg_log_error("failed to remove contents of WAL directory");
193 : }
194 : }
195 : else
196 : {
197 8 : if ((made_new_pgdata || found_existing_pgdata) && !checksum_failure)
198 0 : pg_log_info("data directory \"%s\" not removed at user's request", basedir);
199 :
200 8 : if (made_new_xlogdir || found_existing_xlogdir)
201 0 : pg_log_info("WAL directory \"%s\" not removed at user's request", xlog_dir);
202 : }
203 :
204 40 : if ((made_tablespace_dirs || found_tablespace_dirs) && !checksum_failure)
205 0 : pg_log_info("changes to tablespace directories will not be undone");
206 : }
207 :
208 : static void
209 180 : disconnect_atexit(void)
210 : {
211 180 : if (conn != NULL)
212 96 : PQfinish(conn);
213 180 : }
214 :
215 : #ifndef WIN32
216 : /*
217 : * On windows, our background thread dies along with the process. But on
218 : * Unix, if we have started a subprocess, we want to kill it off so it
219 : * doesn't remain running trying to stream data.
220 : */
221 : static void
222 90 : kill_bgchild_atexit(void)
223 : {
224 90 : if (bgchild > 0)
225 90 : kill(bgchild, SIGTERM);
226 90 : }
227 : #endif
228 :
229 : /*
230 : * Split argument into old_dir and new_dir and append to tablespace mapping
231 : * list.
232 : */
233 : static void
234 16 : tablespace_list_append(const char *arg)
235 : {
236 16 : TablespaceListCell *cell = (TablespaceListCell *) pg_malloc0(sizeof(TablespaceListCell));
237 : char *dst;
238 : char *dst_ptr;
239 : const char *arg_ptr;
240 :
241 16 : dst_ptr = dst = cell->old_dir;
242 580 : for (arg_ptr = arg; *arg_ptr; arg_ptr++)
243 : {
244 566 : if (dst_ptr - dst >= MAXPGPATH)
245 : {
246 0 : pg_log_error("directory name too long");
247 0 : exit(1);
248 : }
249 :
250 566 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
251 : ; /* skip backslash escaping = */
252 562 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
253 : {
254 30 : if (*cell->new_dir)
255 : {
256 2 : pg_log_error("multiple \"=\" signs in tablespace mapping");
257 2 : exit(1);
258 : }
259 : else
260 14 : dst = dst_ptr = cell->new_dir;
261 : }
262 : else
263 546 : *dst_ptr++ = *arg_ptr;
264 : }
265 :
266 14 : if (!*cell->old_dir || !*cell->new_dir)
267 : {
268 6 : pg_log_error("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
269 6 : exit(1);
270 : }
271 :
272 : /*
273 : * This check isn't absolutely necessary. But all tablespaces are created
274 : * with absolute directories, so specifying a non-absolute path here would
275 : * just never match, possibly confusing users. It's also good to be
276 : * consistent with the new_dir check.
277 : */
278 8 : if (!is_absolute_path(cell->old_dir))
279 : {
280 2 : pg_log_error("old directory is not an absolute path in tablespace mapping: %s",
281 : cell->old_dir);
282 2 : exit(1);
283 : }
284 :
285 6 : if (!is_absolute_path(cell->new_dir))
286 : {
287 2 : pg_log_error("new directory is not an absolute path in tablespace mapping: %s",
288 : cell->new_dir);
289 2 : exit(1);
290 : }
291 :
292 : /*
293 : * Comparisons done with these values should involve similarly
294 : * canonicalized path values. This is particularly sensitive on Windows
295 : * where path values may not necessarily use Unix slashes.
296 : */
297 4 : canonicalize_path(cell->old_dir);
298 4 : canonicalize_path(cell->new_dir);
299 :
300 4 : if (tablespace_dirs.tail)
301 0 : tablespace_dirs.tail->next = cell;
302 : else
303 4 : tablespace_dirs.head = cell;
304 4 : tablespace_dirs.tail = cell;
305 4 : }
306 :
307 :
308 : #ifdef HAVE_LIBZ
309 : static const char *
310 0 : get_gz_error(gzFile gzf)
311 : {
312 : int errnum;
313 : const char *errmsg;
314 :
315 0 : errmsg = gzerror(gzf, &errnum);
316 0 : if (errnum == Z_ERRNO)
317 0 : return strerror(errno);
318 : else
319 0 : return errmsg;
320 : }
321 : #endif
322 :
323 : static void
324 2 : usage(void)
325 : {
326 2 : printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
327 : progname);
328 2 : printf(_("Usage:\n"));
329 2 : printf(_(" %s [OPTION]...\n"), progname);
330 2 : printf(_("\nOptions controlling the output:\n"));
331 2 : printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
332 2 : printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
333 2 : printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
334 : " (in kB/s, or use suffix \"k\" or \"M\")\n"));
335 2 : printf(_(" -R, --write-recovery-conf\n"
336 : " write configuration for replication\n"));
337 2 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
338 : " relocate tablespace in OLDDIR to NEWDIR\n"));
339 2 : printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
340 2 : printf(_(" -X, --wal-method=none|fetch|stream\n"
341 : " include required WAL files with specified method\n"));
342 2 : printf(_(" -z, --gzip compress tar output\n"));
343 2 : printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
344 2 : printf(_("\nGeneral options:\n"));
345 2 : printf(_(" -c, --checkpoint=fast|spread\n"
346 : " set fast or spread checkpointing\n"));
347 2 : printf(_(" -C, --create-slot create replication slot\n"));
348 2 : printf(_(" -l, --label=LABEL set backup label\n"));
349 2 : printf(_(" -n, --no-clean do not clean up after errors\n"));
350 2 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
351 2 : printf(_(" -P, --progress show progress information\n"));
352 2 : printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
353 2 : printf(_(" -v, --verbose output verbose messages\n"));
354 2 : printf(_(" -V, --version output version information, then exit\n"));
355 2 : printf(_(" --no-slot prevent creation of temporary replication slot\n"));
356 2 : printf(_(" --no-verify-checksums\n"
357 : " do not verify checksums\n"));
358 2 : printf(_(" -?, --help show this help, then exit\n"));
359 2 : printf(_("\nConnection options:\n"));
360 2 : printf(_(" -d, --dbname=CONNSTR connection string\n"));
361 2 : printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
362 2 : printf(_(" -p, --port=PORT database server port number\n"));
363 2 : printf(_(" -s, --status-interval=INTERVAL\n"
364 : " time between status packets sent to server (in seconds)\n"));
365 2 : printf(_(" -U, --username=NAME connect as specified database user\n"));
366 2 : printf(_(" -w, --no-password never prompt for password\n"));
367 2 : printf(_(" -W, --password force password prompt (should happen automatically)\n"));
368 2 : printf(_("\nReport bugs to <pgsql-bugs@lists.postgresql.org>.\n"));
369 2 : }
370 :
371 :
372 : /*
373 : * Called in the background process every time data is received.
374 : * On Unix, we check to see if there is any data on our pipe
375 : * (which would mean we have a stop position), and if it is, check if
376 : * it is time to stop.
377 : * On Windows, we are in a single process, so we can just check if it's
378 : * time to stop.
379 : */
380 : static bool
381 5636 : reached_end_position(XLogRecPtr segendpos, uint32 timeline,
382 : bool segment_finished)
383 : {
384 5636 : if (!has_xlogendptr)
385 : {
386 : #ifndef WIN32
387 : fd_set fds;
388 : struct timeval tv;
389 : int r;
390 :
391 : /*
392 : * Don't have the end pointer yet - check our pipe to see if it has
393 : * been sent yet.
394 : */
395 5516 : FD_ZERO(&fds);
396 5516 : FD_SET(bgpipe[0], &fds);
397 :
398 5516 : MemSet(&tv, 0, sizeof(tv));
399 :
400 5516 : r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
401 5516 : if (r == 1)
402 : {
403 : char xlogend[64];
404 : uint32 hi,
405 : lo;
406 :
407 80 : MemSet(xlogend, 0, sizeof(xlogend));
408 80 : r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
409 80 : if (r < 0)
410 : {
411 0 : pg_log_error("could not read from ready pipe: %m");
412 0 : exit(1);
413 : }
414 :
415 80 : if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
416 : {
417 0 : pg_log_error("could not parse write-ahead log location \"%s\"",
418 : xlogend);
419 0 : exit(1);
420 : }
421 80 : xlogendptr = ((uint64) hi) << 32 | lo;
422 80 : has_xlogendptr = 1;
423 :
424 : /*
425 : * Fall through to check if we've reached the point further
426 : * already.
427 : */
428 : }
429 : else
430 : {
431 : /*
432 : * No data received on the pipe means we don't know the end
433 : * position yet - so just say it's not time to stop yet.
434 : */
435 5436 : return false;
436 : }
437 : #else
438 :
439 : /*
440 : * On win32, has_xlogendptr is set by the main thread, so if it's not
441 : * set here, we just go back and wait until it shows up.
442 : */
443 : return false;
444 : #endif
445 : }
446 :
447 : /*
448 : * At this point we have an end pointer, so compare it to the current
449 : * position to figure out if it's time to stop.
450 : */
451 200 : if (segendpos >= xlogendptr)
452 160 : return true;
453 :
454 : /*
455 : * Have end pointer, but haven't reached it yet - so tell the caller to
456 : * keep streaming.
457 : */
458 40 : return false;
459 : }
460 :
461 : typedef struct
462 : {
463 : PGconn *bgconn;
464 : XLogRecPtr startptr;
465 : char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
466 : char *sysidentifier;
467 : int timeline;
468 : } logstreamer_param;
469 :
470 : static int
471 82 : LogStreamerMain(logstreamer_param *param)
472 : {
473 : StreamCtl stream;
474 :
475 82 : in_log_streamer = true;
476 :
477 82 : MemSet(&stream, 0, sizeof(stream));
478 82 : stream.startpos = param->startptr;
479 82 : stream.timeline = param->timeline;
480 82 : stream.sysidentifier = param->sysidentifier;
481 82 : stream.stream_stop = reached_end_position;
482 : #ifndef WIN32
483 82 : stream.stop_socket = bgpipe[0];
484 : #else
485 : stream.stop_socket = PGINVALID_SOCKET;
486 : #endif
487 82 : stream.standby_message_timeout = standby_message_timeout;
488 82 : stream.synchronous = false;
489 82 : stream.do_sync = do_sync;
490 82 : stream.mark_done = true;
491 82 : stream.partial_suffix = NULL;
492 82 : stream.replication_slot = replication_slot;
493 :
494 82 : if (format == 'p')
495 74 : stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0, do_sync);
496 : else
497 8 : stream.walmethod = CreateWalTarMethod(param->xlog, compresslevel, do_sync);
498 :
499 82 : if (!ReceiveXlogStream(param->bgconn, &stream))
500 :
501 : /*
502 : * Any errors will already have been reported in the function process,
503 : * but we need to tell the parent that we didn't shutdown in a nice
504 : * way.
505 : */
506 2 : return 1;
507 :
508 80 : if (!stream.walmethod->finish())
509 : {
510 0 : pg_log_error("could not finish writing WAL files: %m");
511 0 : return 1;
512 : }
513 :
514 80 : PQfinish(param->bgconn);
515 :
516 80 : if (format == 'p')
517 72 : FreeWalDirectoryMethod();
518 : else
519 8 : FreeWalTarMethod();
520 80 : pg_free(stream.walmethod);
521 :
522 80 : return 0;
523 : }
524 :
525 : /*
526 : * Initiate background process for receiving xlog during the backup.
527 : * The background stream will use its own database connection so we can
528 : * stream the logfile in parallel with the backups.
529 : */
530 : static void
531 92 : StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
532 : {
533 : logstreamer_param *param;
534 : uint32 hi,
535 : lo;
536 : char statusdir[MAXPGPATH];
537 :
538 92 : param = pg_malloc0(sizeof(logstreamer_param));
539 92 : param->timeline = timeline;
540 92 : param->sysidentifier = sysidentifier;
541 :
542 : /* Convert the starting position */
543 92 : if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
544 : {
545 0 : pg_log_error("could not parse write-ahead log location \"%s\"",
546 : startpos);
547 0 : exit(1);
548 : }
549 92 : param->startptr = ((uint64) hi) << 32 | lo;
550 : /* Round off to even segment position */
551 92 : param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
552 :
553 : #ifndef WIN32
554 : /* Create our background pipe */
555 92 : if (pipe(bgpipe) < 0)
556 : {
557 0 : pg_log_error("could not create pipe for background process: %m");
558 0 : exit(1);
559 : }
560 : #endif
561 :
562 : /* Get a second connection */
563 92 : param->bgconn = GetConnection();
564 92 : if (!param->bgconn)
565 : /* Error message already written in GetConnection() */
566 0 : exit(1);
567 :
568 : /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
569 92 : snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
570 : basedir,
571 92 : PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
572 : "pg_xlog" : "pg_wal");
573 :
574 : /* Temporary replication slots are only supported in 10 and newer */
575 92 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_TEMP_SLOTS)
576 0 : temp_replication_slot = false;
577 :
578 : /*
579 : * Create replication slot if requested
580 : */
581 92 : if (temp_replication_slot && !replication_slot)
582 80 : replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
583 92 : if (temp_replication_slot || create_slot)
584 : {
585 84 : if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
586 : temp_replication_slot, true, true, false))
587 2 : exit(1);
588 :
589 82 : if (verbose)
590 : {
591 0 : if (temp_replication_slot)
592 0 : pg_log_info("created temporary replication slot \"%s\"",
593 : replication_slot);
594 : else
595 0 : pg_log_info("created replication slot \"%s\"",
596 : replication_slot);
597 : }
598 : }
599 :
600 90 : if (format == 'p')
601 : {
602 : /*
603 : * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
604 : * pg_wal or pg_xlog) depending on the target server so we can write
605 : * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
606 : * tar file may arrive later.
607 : */
608 80 : snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
609 : basedir,
610 80 : PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
611 : "pg_xlog" : "pg_wal");
612 :
613 80 : if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
614 : {
615 0 : pg_log_error("could not create directory \"%s\": %m", statusdir);
616 0 : exit(1);
617 : }
618 : }
619 :
620 : /*
621 : * Start a child process and tell it to start streaming. On Unix, this is
622 : * a fork(). On Windows, we create a thread.
623 : */
624 : #ifndef WIN32
625 90 : bgchild = fork();
626 172 : if (bgchild == 0)
627 : {
628 : /* in child process */
629 82 : exit(LogStreamerMain(param));
630 : }
631 90 : else if (bgchild < 0)
632 : {
633 0 : pg_log_error("could not create background process: %m");
634 0 : exit(1);
635 : }
636 :
637 : /*
638 : * Else we are in the parent process and all is well.
639 : */
640 90 : atexit(kill_bgchild_atexit);
641 : #else /* WIN32 */
642 : bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
643 : if (bgchild == 0)
644 : {
645 : pg_log_error("could not create background thread: %m");
646 : exit(1);
647 : }
648 : #endif
649 90 : }
650 :
651 : /*
652 : * Verify that the given directory exists and is empty. If it does not
653 : * exist, it is created. If it exists but is not empty, an error will
654 : * be given and the process ended.
655 : */
656 : static void
657 106 : verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
658 : {
659 106 : switch (pg_check_dir(dirname))
660 : {
661 : case 0:
662 :
663 : /*
664 : * Does not exist, so create
665 : */
666 104 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
667 : {
668 0 : pg_log_error("could not create directory \"%s\": %m", dirname);
669 0 : exit(1);
670 : }
671 104 : if (created)
672 104 : *created = true;
673 104 : return;
674 : case 1:
675 :
676 : /*
677 : * Exists, empty
678 : */
679 0 : if (found)
680 0 : *found = true;
681 0 : return;
682 : case 2:
683 : case 3:
684 : case 4:
685 :
686 : /*
687 : * Exists, not empty
688 : */
689 2 : pg_log_error("directory \"%s\" exists but is not empty", dirname);
690 2 : exit(1);
691 : case -1:
692 :
693 : /*
694 : * Access problem
695 : */
696 0 : pg_log_error("could not access directory \"%s\": %m", dirname);
697 0 : exit(1);
698 : }
699 : }
700 :
701 :
702 : /*
703 : * Print a progress report based on the global variables. If verbose output
704 : * is enabled, also print the current file name.
705 : *
706 : * Progress report is written at maximum once per second, unless the
707 : * force parameter is set to true.
708 : */
709 : static void
710 120116 : progress_report(int tablespacenum, const char *filename, bool force)
711 : {
712 : int percent;
713 : char totaldone_str[32];
714 : char totalsize_str[32];
715 : pg_time_t now;
716 :
717 120116 : if (!showprogress)
718 240232 : return;
719 :
720 0 : now = time(NULL);
721 0 : if (now == last_progress_report && !force)
722 0 : return; /* Max once per second */
723 :
724 0 : last_progress_report = now;
725 0 : percent = totalsize ? (int) ((totaldone / 1024) * 100 / totalsize) : 0;
726 :
727 : /*
728 : * Avoid overflowing past 100% or the full size. This may make the total
729 : * size number change as we approach the end of the backup (the estimate
730 : * will always be wrong if WAL is included), but that's better than having
731 : * the done column be bigger than the total.
732 : */
733 0 : if (percent > 100)
734 0 : percent = 100;
735 0 : if (totaldone / 1024 > totalsize)
736 0 : totalsize = totaldone / 1024;
737 :
738 : /*
739 : * Separate step to keep platform-dependent format code out of
740 : * translatable strings. And we only test for INT64_FORMAT availability
741 : * in snprintf, not fprintf.
742 : */
743 0 : snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
744 : totaldone / 1024);
745 0 : snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize);
746 :
747 : #define VERBOSE_FILENAME_LENGTH 35
748 0 : if (verbose)
749 : {
750 0 : if (!filename)
751 :
752 : /*
753 : * No filename given, so clear the status line (used for last
754 : * call)
755 : */
756 0 : fprintf(stderr,
757 0 : ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
758 : "%*s/%s kB (100%%), %d/%d tablespaces %*s",
759 : tablespacecount),
760 0 : (int) strlen(totalsize_str),
761 : totaldone_str, totalsize_str,
762 : tablespacenum, tablespacecount,
763 : VERBOSE_FILENAME_LENGTH + 5, "");
764 : else
765 : {
766 0 : bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
767 :
768 0 : fprintf(stderr,
769 0 : ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
770 : "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
771 : tablespacecount),
772 0 : (int) strlen(totalsize_str),
773 : totaldone_str, totalsize_str, percent,
774 : tablespacenum, tablespacecount,
775 : /* Prefix with "..." if we do leading truncation */
776 : truncate ? "..." : "",
777 : truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
778 : truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
779 : /* Truncate filename at beginning if it's too long */
780 0 : truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
781 : }
782 : }
783 : else
784 0 : fprintf(stderr,
785 0 : ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
786 : "%*s/%s kB (%d%%), %d/%d tablespaces",
787 : tablespacecount),
788 0 : (int) strlen(totalsize_str),
789 : totaldone_str, totalsize_str, percent,
790 : tablespacenum, tablespacecount);
791 :
792 0 : if (isatty(fileno(stderr)))
793 0 : fprintf(stderr, "\r");
794 : else
795 0 : fprintf(stderr, "\n");
796 : }
797 :
798 : static int32
799 0 : parse_max_rate(char *src)
800 : {
801 : double result;
802 : char *after_num;
803 0 : char *suffix = NULL;
804 :
805 0 : errno = 0;
806 0 : result = strtod(src, &after_num);
807 0 : if (src == after_num)
808 : {
809 0 : pg_log_error("transfer rate \"%s\" is not a valid value", src);
810 0 : exit(1);
811 : }
812 0 : if (errno != 0)
813 : {
814 0 : pg_log_error("invalid transfer rate \"%s\": %m", src);
815 0 : exit(1);
816 : }
817 :
818 0 : if (result <= 0)
819 : {
820 : /*
821 : * Reject obviously wrong values here.
822 : */
823 0 : pg_log_error("transfer rate must be greater than zero");
824 0 : exit(1);
825 : }
826 :
827 : /*
828 : * Evaluate suffix, after skipping over possible whitespace. Lack of
829 : * suffix means kilobytes.
830 : */
831 0 : while (*after_num != '\0' && isspace((unsigned char) *after_num))
832 0 : after_num++;
833 :
834 0 : if (*after_num != '\0')
835 : {
836 0 : suffix = after_num;
837 0 : if (*after_num == 'k')
838 : {
839 : /* kilobyte is the expected unit. */
840 0 : after_num++;
841 : }
842 0 : else if (*after_num == 'M')
843 : {
844 0 : after_num++;
845 0 : result *= 1024.0;
846 : }
847 : }
848 :
849 : /* The rest can only consist of white space. */
850 0 : while (*after_num != '\0' && isspace((unsigned char) *after_num))
851 0 : after_num++;
852 :
853 0 : if (*after_num != '\0')
854 : {
855 0 : pg_log_error("invalid --max-rate unit: \"%s\"", suffix);
856 0 : exit(1);
857 : }
858 :
859 : /* Valid integer? */
860 0 : if ((uint64) result != (uint64) ((uint32) result))
861 : {
862 0 : pg_log_error("transfer rate \"%s\" exceeds integer range", src);
863 0 : exit(1);
864 : }
865 :
866 : /*
867 : * The range is checked on the server side too, but avoid the server
868 : * connection if a nonsensical value was passed.
869 : */
870 0 : if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
871 : {
872 0 : pg_log_error("transfer rate \"%s\" is out of range", src);
873 0 : exit(1);
874 : }
875 :
876 0 : return (int32) result;
877 : }
878 :
879 : /*
880 : * Write a piece of tar data
881 : */
882 : static void
883 17734 : writeTarData(
884 : #ifdef HAVE_LIBZ
885 : gzFile ztarfile,
886 : #endif
887 : FILE *tarfile, char *buf, int r, char *current_file)
888 : {
889 : #ifdef HAVE_LIBZ
890 17734 : if (ztarfile != NULL)
891 : {
892 0 : if (gzwrite(ztarfile, buf, r) != r)
893 : {
894 0 : pg_log_error("could not write to compressed file \"%s\": %s",
895 : current_file, get_gz_error(ztarfile));
896 0 : exit(1);
897 : }
898 : }
899 : else
900 : #endif
901 : {
902 17734 : if (fwrite(buf, r, 1, tarfile) != 1)
903 : {
904 0 : pg_log_error("could not write to file \"%s\": %m", current_file);
905 0 : exit(1);
906 : }
907 : }
908 17734 : }
909 :
910 : #ifdef HAVE_LIBZ
911 : #define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename)
912 : #else
913 : #define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename)
914 : #endif
915 :
916 : /*
917 : * Receive a tar format file from the connection to the server, and write
918 : * the data from this file directly into a tar file. If compression is
919 : * enabled, the data will be compressed while written to the file.
920 : *
921 : * The file will be named base.tar[.gz] if it's for the main data directory
922 : * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
923 : *
924 : * No attempt to inspect or validate the contents of the file is done.
925 : */
926 : static void
927 14 : ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
928 : {
929 : char filename[MAXPGPATH];
930 14 : char *copybuf = NULL;
931 14 : FILE *tarfile = NULL;
932 : char tarhdr[512];
933 14 : bool basetablespace = PQgetisnull(res, rownum, 0);
934 14 : bool in_tarhdr = true;
935 14 : bool skip_file = false;
936 14 : bool is_recovery_guc_supported = true;
937 14 : bool is_postgresql_auto_conf = false;
938 14 : bool found_postgresql_auto_conf = false;
939 14 : int file_padding_len = 0;
940 14 : size_t tarhdrsz = 0;
941 14 : pgoff_t filesz = 0;
942 :
943 : #ifdef HAVE_LIBZ
944 14 : gzFile ztarfile = NULL;
945 : #endif
946 :
947 : /* recovery.conf is integrated into postgresql.conf in 12 and newer */
948 14 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
949 0 : is_recovery_guc_supported = false;
950 :
951 14 : if (basetablespace)
952 : {
953 : /*
954 : * Base tablespaces
955 : */
956 10 : if (strcmp(basedir, "-") == 0)
957 : {
958 : #ifdef WIN32
959 : _setmode(fileno(stdout), _O_BINARY);
960 : #endif
961 :
962 : #ifdef HAVE_LIBZ
963 0 : if (compresslevel != 0)
964 : {
965 0 : ztarfile = gzdopen(dup(fileno(stdout)), "wb");
966 0 : if (gzsetparams(ztarfile, compresslevel,
967 : Z_DEFAULT_STRATEGY) != Z_OK)
968 : {
969 0 : pg_log_error("could not set compression level %d: %s",
970 : compresslevel, get_gz_error(ztarfile));
971 0 : exit(1);
972 : }
973 : }
974 : else
975 : #endif
976 0 : tarfile = stdout;
977 0 : strcpy(filename, "-");
978 : }
979 : else
980 : {
981 : #ifdef HAVE_LIBZ
982 10 : if (compresslevel != 0)
983 : {
984 0 : snprintf(filename, sizeof(filename), "%s/base.tar.gz", basedir);
985 0 : ztarfile = gzopen(filename, "wb");
986 0 : if (gzsetparams(ztarfile, compresslevel,
987 : Z_DEFAULT_STRATEGY) != Z_OK)
988 : {
989 0 : pg_log_error("could not set compression level %d: %s",
990 : compresslevel, get_gz_error(ztarfile));
991 0 : exit(1);
992 : }
993 : }
994 : else
995 : #endif
996 : {
997 10 : snprintf(filename, sizeof(filename), "%s/base.tar", basedir);
998 10 : tarfile = fopen(filename, "wb");
999 : }
1000 : }
1001 : }
1002 : else
1003 : {
1004 : /*
1005 : * Specific tablespace
1006 : */
1007 : #ifdef HAVE_LIBZ
1008 4 : if (compresslevel != 0)
1009 : {
1010 0 : snprintf(filename, sizeof(filename), "%s/%s.tar.gz", basedir,
1011 : PQgetvalue(res, rownum, 0));
1012 0 : ztarfile = gzopen(filename, "wb");
1013 0 : if (gzsetparams(ztarfile, compresslevel,
1014 : Z_DEFAULT_STRATEGY) != Z_OK)
1015 : {
1016 0 : pg_log_error("could not set compression level %d: %s",
1017 : compresslevel, get_gz_error(ztarfile));
1018 0 : exit(1);
1019 : }
1020 : }
1021 : else
1022 : #endif
1023 : {
1024 4 : snprintf(filename, sizeof(filename), "%s/%s.tar", basedir,
1025 : PQgetvalue(res, rownum, 0));
1026 4 : tarfile = fopen(filename, "wb");
1027 : }
1028 : }
1029 :
1030 : #ifdef HAVE_LIBZ
1031 14 : if (compresslevel != 0)
1032 : {
1033 0 : if (!ztarfile)
1034 : {
1035 : /* Compression is in use */
1036 0 : pg_log_error("could not create compressed file \"%s\": %s",
1037 : filename, get_gz_error(ztarfile));
1038 0 : exit(1);
1039 : }
1040 : }
1041 : else
1042 : #endif
1043 : {
1044 : /* Either no zlib support, or zlib support but compresslevel = 0 */
1045 14 : if (!tarfile)
1046 : {
1047 0 : pg_log_error("could not create file \"%s\": %m", filename);
1048 0 : exit(1);
1049 : }
1050 : }
1051 :
1052 : /*
1053 : * Get the COPY data stream
1054 : */
1055 14 : res = PQgetResult(conn);
1056 14 : if (PQresultStatus(res) != PGRES_COPY_OUT)
1057 : {
1058 0 : pg_log_error("could not get COPY data stream: %s",
1059 : PQerrorMessage(conn));
1060 0 : exit(1);
1061 : }
1062 :
1063 : while (1)
1064 17720 : {
1065 : int r;
1066 :
1067 17734 : if (copybuf != NULL)
1068 : {
1069 17720 : PQfreemem(copybuf);
1070 17720 : copybuf = NULL;
1071 : }
1072 :
1073 17734 : r = PQgetCopyData(conn, ©buf, 0);
1074 17734 : if (r == -1)
1075 : {
1076 : /*
1077 : * End of chunk. If requested, and this is the base tablespace,
1078 : * write configuration file into the tarfile. When done, close the
1079 : * file (but not stdout).
1080 : *
1081 : * Also, write two completely empty blocks at the end of the tar
1082 : * file, as required by some tar programs.
1083 : */
1084 : char zerobuf[1024];
1085 :
1086 14 : MemSet(zerobuf, 0, sizeof(zerobuf));
1087 :
1088 14 : if (basetablespace && writerecoveryconf)
1089 : {
1090 : char header[512];
1091 :
1092 : /*
1093 : * If postgresql.auto.conf has not been found in the streamed
1094 : * data, add recovery configuration to postgresql.auto.conf if
1095 : * recovery parameters are GUCs. If the instance connected to
1096 : * is older than 12, create recovery.conf with this data
1097 : * otherwise.
1098 : */
1099 0 : if (!found_postgresql_auto_conf || !is_recovery_guc_supported)
1100 : {
1101 : int padding;
1102 :
1103 0 : tarCreateHeader(header,
1104 : is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf",
1105 : NULL,
1106 0 : recoveryconfcontents->len,
1107 : pg_file_create_mode, 04000, 02000,
1108 : time(NULL));
1109 :
1110 0 : padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1111 :
1112 0 : WRITE_TAR_DATA(header, sizeof(header));
1113 0 : WRITE_TAR_DATA(recoveryconfcontents->data,
1114 : recoveryconfcontents->len);
1115 0 : if (padding)
1116 0 : WRITE_TAR_DATA(zerobuf, padding);
1117 : }
1118 :
1119 : /*
1120 : * standby.signal is supported only if recovery parameters are
1121 : * GUCs.
1122 : */
1123 0 : if (is_recovery_guc_supported)
1124 : {
1125 0 : tarCreateHeader(header, "standby.signal", NULL,
1126 : 0, /* zero-length file */
1127 : pg_file_create_mode, 04000, 02000,
1128 : time(NULL));
1129 :
1130 0 : WRITE_TAR_DATA(header, sizeof(header));
1131 0 : WRITE_TAR_DATA(zerobuf, 511);
1132 : }
1133 : }
1134 :
1135 : /* 2 * 512 bytes empty data at end of file */
1136 14 : WRITE_TAR_DATA(zerobuf, sizeof(zerobuf));
1137 :
1138 : #ifdef HAVE_LIBZ
1139 14 : if (ztarfile != NULL)
1140 : {
1141 0 : if (gzclose(ztarfile) != 0)
1142 : {
1143 0 : pg_log_error("could not close compressed file \"%s\": %s",
1144 : filename, get_gz_error(ztarfile));
1145 0 : exit(1);
1146 : }
1147 : }
1148 : else
1149 : #endif
1150 : {
1151 14 : if (strcmp(basedir, "-") != 0)
1152 : {
1153 14 : if (fclose(tarfile) != 0)
1154 : {
1155 0 : pg_log_error("could not close file \"%s\": %m",
1156 : filename);
1157 0 : exit(1);
1158 : }
1159 : }
1160 : }
1161 :
1162 14 : break;
1163 : }
1164 17720 : else if (r == -2)
1165 : {
1166 0 : pg_log_error("could not read COPY data: %s",
1167 : PQerrorMessage(conn));
1168 0 : exit(1);
1169 : }
1170 :
1171 17720 : if (!writerecoveryconf || !basetablespace)
1172 : {
1173 : /*
1174 : * When not writing config file, or when not working on the base
1175 : * tablespace, we never have to look for an existing configuration
1176 : * file in the stream.
1177 : */
1178 17720 : WRITE_TAR_DATA(copybuf, r);
1179 : }
1180 : else
1181 : {
1182 : /*
1183 : * Look for a config file in the existing tar stream. If it's
1184 : * there, we must skip it so we can later overwrite it with our
1185 : * own version of the file.
1186 : *
1187 : * To do this, we have to process the individual files inside the
1188 : * TAR stream. The stream consists of a header and zero or more
1189 : * chunks, all 512 bytes long. The stream from the server is
1190 : * broken up into smaller pieces, so we have to track the size of
1191 : * the files to find the next header structure.
1192 : */
1193 0 : int rr = r;
1194 0 : int pos = 0;
1195 :
1196 0 : while (rr > 0)
1197 : {
1198 0 : if (in_tarhdr)
1199 : {
1200 : /*
1201 : * We're currently reading a header structure inside the
1202 : * TAR stream, i.e. the file metadata.
1203 : */
1204 0 : if (tarhdrsz < 512)
1205 : {
1206 : /*
1207 : * Copy the header structure into tarhdr in case the
1208 : * header is not aligned to 512 bytes or it's not
1209 : * returned in whole by the last PQgetCopyData call.
1210 : */
1211 : int hdrleft;
1212 : int bytes2copy;
1213 :
1214 0 : hdrleft = 512 - tarhdrsz;
1215 0 : bytes2copy = (rr > hdrleft ? hdrleft : rr);
1216 :
1217 0 : memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy);
1218 :
1219 0 : rr -= bytes2copy;
1220 0 : pos += bytes2copy;
1221 0 : tarhdrsz += bytes2copy;
1222 : }
1223 : else
1224 : {
1225 : /*
1226 : * We have the complete header structure in tarhdr,
1227 : * look at the file metadata: we may want append
1228 : * recovery info into postgresql.auto.conf and skip
1229 : * standby.signal file if recovery parameters are
1230 : * integrated as GUCs, and recovery.conf otherwise. In
1231 : * both cases we must calculate tar padding.
1232 : */
1233 0 : if (is_recovery_guc_supported)
1234 : {
1235 0 : skip_file = (strcmp(&tarhdr[0], "standby.signal") == 0);
1236 0 : is_postgresql_auto_conf = (strcmp(&tarhdr[0], "postgresql.auto.conf") == 0);
1237 : }
1238 : else
1239 0 : skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
1240 :
1241 0 : filesz = read_tar_number(&tarhdr[124], 12);
1242 0 : file_padding_len = ((filesz + 511) & ~511) - filesz;
1243 :
1244 0 : if (is_recovery_guc_supported &&
1245 0 : is_postgresql_auto_conf &&
1246 : writerecoveryconf)
1247 0 : {
1248 : /* replace tar header */
1249 : char header[512];
1250 :
1251 0 : tarCreateHeader(header, "postgresql.auto.conf", NULL,
1252 0 : filesz + recoveryconfcontents->len,
1253 : pg_file_create_mode, 04000, 02000,
1254 : time(NULL));
1255 :
1256 0 : WRITE_TAR_DATA(header, sizeof(header));
1257 : }
1258 : else
1259 : {
1260 : /* copy stream with padding */
1261 0 : filesz += file_padding_len;
1262 :
1263 0 : if (!skip_file)
1264 : {
1265 : /*
1266 : * If we're not skipping the file, write the
1267 : * tar header unmodified.
1268 : */
1269 0 : WRITE_TAR_DATA(tarhdr, 512);
1270 : }
1271 : }
1272 :
1273 : /* Next part is the file, not the header */
1274 0 : in_tarhdr = false;
1275 : }
1276 : }
1277 : else
1278 : {
1279 : /*
1280 : * We're processing a file's contents.
1281 : */
1282 0 : if (filesz > 0)
1283 : {
1284 : /*
1285 : * We still have data to read (and possibly write).
1286 : */
1287 : int bytes2write;
1288 :
1289 0 : bytes2write = (filesz > rr ? rr : filesz);
1290 :
1291 0 : if (!skip_file)
1292 0 : WRITE_TAR_DATA(copybuf + pos, bytes2write);
1293 :
1294 0 : rr -= bytes2write;
1295 0 : pos += bytes2write;
1296 0 : filesz -= bytes2write;
1297 : }
1298 0 : else if (is_recovery_guc_supported &&
1299 0 : is_postgresql_auto_conf &&
1300 : writerecoveryconf)
1301 0 : {
1302 : /* append recovery config to postgresql.auto.conf */
1303 : int padding;
1304 : int tailsize;
1305 :
1306 0 : tailsize = (512 - file_padding_len) + recoveryconfcontents->len;
1307 0 : padding = ((tailsize + 511) & ~511) - tailsize;
1308 :
1309 0 : WRITE_TAR_DATA(recoveryconfcontents->data, recoveryconfcontents->len);
1310 :
1311 0 : if (padding)
1312 : {
1313 : char zerobuf[512];
1314 :
1315 0 : MemSet(zerobuf, 0, sizeof(zerobuf));
1316 0 : WRITE_TAR_DATA(zerobuf, padding);
1317 : }
1318 :
1319 : /* skip original file padding */
1320 0 : is_postgresql_auto_conf = false;
1321 0 : skip_file = true;
1322 0 : filesz += file_padding_len;
1323 :
1324 0 : found_postgresql_auto_conf = true;
1325 : }
1326 : else
1327 : {
1328 : /*
1329 : * No more data in the current file, the next piece of
1330 : * data (if any) will be a new file header structure.
1331 : */
1332 0 : in_tarhdr = true;
1333 0 : skip_file = false;
1334 0 : is_postgresql_auto_conf = false;
1335 0 : tarhdrsz = 0;
1336 0 : filesz = 0;
1337 : }
1338 : }
1339 : }
1340 : }
1341 17720 : totaldone += r;
1342 17720 : progress_report(rownum, filename, false);
1343 : } /* while (1) */
1344 14 : progress_report(rownum, filename, true);
1345 :
1346 14 : if (copybuf != NULL)
1347 0 : PQfreemem(copybuf);
1348 :
1349 : /* sync the resulting tar file, errors are not considered fatal */
1350 14 : if (do_sync && strcmp(basedir, "-") != 0)
1351 14 : (void) fsync_fname(filename, false);
1352 14 : }
1353 :
1354 :
1355 : /*
1356 : * Retrieve tablespace path, either relocated or original depending on whether
1357 : * -T was passed or not.
1358 : */
1359 : static const char *
1360 14 : get_tablespace_mapping(const char *dir)
1361 : {
1362 : TablespaceListCell *cell;
1363 : char canon_dir[MAXPGPATH];
1364 :
1365 : /* Canonicalize path for comparison consistency */
1366 14 : strlcpy(canon_dir, dir, sizeof(canon_dir));
1367 14 : canonicalize_path(canon_dir);
1368 :
1369 14 : for (cell = tablespace_dirs.head; cell; cell = cell->next)
1370 12 : if (strcmp(canon_dir, cell->old_dir) == 0)
1371 12 : return cell->new_dir;
1372 :
1373 2 : return dir;
1374 : }
1375 :
1376 :
1377 : /*
1378 : * Receive a tar format stream from the connection to the server, and unpack
1379 : * the contents of it into a directory. Only files, directories and
1380 : * symlinks are supported, no other kinds of special files.
1381 : *
1382 : * If the data is for the main data directory, it will be restored in the
1383 : * specified directory. If it's for another tablespace, it will be restored
1384 : * in the original or mapped directory.
1385 : */
1386 : static void
1387 88 : ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
1388 : {
1389 : char current_path[MAXPGPATH];
1390 : char filename[MAXPGPATH];
1391 : const char *mapped_tblspc_path;
1392 88 : pgoff_t current_len_left = 0;
1393 88 : int current_padding = 0;
1394 : bool basetablespace;
1395 88 : char *copybuf = NULL;
1396 88 : FILE *file = NULL;
1397 :
1398 88 : basetablespace = PQgetisnull(res, rownum, 0);
1399 88 : if (basetablespace)
1400 84 : strlcpy(current_path, basedir, sizeof(current_path));
1401 : else
1402 4 : strlcpy(current_path,
1403 4 : get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1404 : sizeof(current_path));
1405 :
1406 : /*
1407 : * Get the COPY data
1408 : */
1409 88 : res = PQgetResult(conn);
1410 88 : if (PQresultStatus(res) != PGRES_COPY_OUT)
1411 : {
1412 0 : pg_log_error("could not get COPY data stream: %s",
1413 : PQerrorMessage(conn));
1414 0 : exit(1);
1415 : }
1416 :
1417 : while (1)
1418 188982 : {
1419 : int r;
1420 :
1421 189070 : if (copybuf != NULL)
1422 : {
1423 188982 : PQfreemem(copybuf);
1424 188982 : copybuf = NULL;
1425 : }
1426 :
1427 189070 : r = PQgetCopyData(conn, ©buf, 0);
1428 :
1429 189070 : if (r == -1)
1430 : {
1431 : /*
1432 : * End of chunk
1433 : */
1434 88 : if (file)
1435 0 : fclose(file);
1436 :
1437 88 : break;
1438 : }
1439 188982 : else if (r == -2)
1440 : {
1441 0 : pg_log_error("could not read COPY data: %s",
1442 : PQerrorMessage(conn));
1443 0 : exit(1);
1444 : }
1445 :
1446 188982 : if (file == NULL)
1447 : {
1448 : int filemode;
1449 :
1450 : /*
1451 : * No current file, so this must be the header for a new file
1452 : */
1453 85764 : if (r != 512)
1454 : {
1455 0 : pg_log_error("invalid tar block header size: %d", r);
1456 0 : exit(1);
1457 : }
1458 85764 : totaldone += 512;
1459 :
1460 85764 : current_len_left = read_tar_number(©buf[124], 12);
1461 :
1462 : /* Set permissions on the file */
1463 85764 : filemode = read_tar_number(©buf[100], 8);
1464 :
1465 : /*
1466 : * All files are padded up to 512 bytes
1467 : */
1468 85764 : current_padding =
1469 85764 : ((current_len_left + 511) & ~511) - current_len_left;
1470 :
1471 : /*
1472 : * First part of header is zero terminated filename
1473 : */
1474 85764 : snprintf(filename, sizeof(filename), "%s/%s", current_path,
1475 : copybuf);
1476 85764 : if (filename[strlen(filename) - 1] == '/')
1477 : {
1478 : /*
1479 : * Ends in a slash means directory or symlink to directory
1480 : */
1481 2122 : if (copybuf[156] == '5')
1482 : {
1483 : /*
1484 : * Directory
1485 : */
1486 2118 : filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1487 2118 : if (mkdir(filename, pg_dir_create_mode) != 0)
1488 : {
1489 : /*
1490 : * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1491 : * clusters) will have been created by the wal
1492 : * receiver process. Also, when the WAL directory
1493 : * location was specified, pg_wal (or pg_xlog) has
1494 : * already been created as a symbolic link before
1495 : * starting the actual backup. So just ignore creation
1496 : * failures on related directories.
1497 : */
1498 400 : if (!((pg_str_endswith(filename, "/pg_wal") ||
1499 160 : pg_str_endswith(filename, "/pg_xlog") ||
1500 80 : pg_str_endswith(filename, "/archive_status")) &&
1501 160 : errno == EEXIST))
1502 : {
1503 0 : pg_log_error("could not create directory \"%s\": %m",
1504 : filename);
1505 0 : exit(1);
1506 : }
1507 : }
1508 : #ifndef WIN32
1509 2118 : if (chmod(filename, (mode_t) filemode))
1510 0 : pg_log_error("could not set permissions on directory \"%s\": %m",
1511 : filename);
1512 : #endif
1513 : }
1514 4 : else if (copybuf[156] == '2')
1515 : {
1516 : /*
1517 : * Symbolic link
1518 : *
1519 : * It's most likely a link in pg_tblspc directory, to the
1520 : * location of a tablespace. Apply any tablespace mapping
1521 : * given on the command line (--tablespace-mapping). (We
1522 : * blindly apply the mapping without checking that the
1523 : * link really is inside pg_tblspc. We don't expect there
1524 : * to be other symlinks in a data directory, but if there
1525 : * are, you can call it an undocumented feature that you
1526 : * can map them too.)
1527 : */
1528 4 : filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1529 :
1530 4 : mapped_tblspc_path = get_tablespace_mapping(©buf[157]);
1531 4 : if (symlink(mapped_tblspc_path, filename) != 0)
1532 : {
1533 0 : pg_log_error("could not create symbolic link from \"%s\" to \"%s\": %m",
1534 : filename, mapped_tblspc_path);
1535 0 : exit(1);
1536 : }
1537 : }
1538 : else
1539 : {
1540 0 : pg_log_error("unrecognized link indicator \"%c\"",
1541 : copybuf[156]);
1542 0 : exit(1);
1543 : }
1544 2122 : continue; /* directory or link handled */
1545 : }
1546 :
1547 : /*
1548 : * regular file
1549 : */
1550 83642 : file = fopen(filename, "wb");
1551 83642 : if (!file)
1552 : {
1553 0 : pg_log_error("could not create file \"%s\": %m", filename);
1554 0 : exit(1);
1555 : }
1556 :
1557 : #ifndef WIN32
1558 83642 : if (chmod(filename, (mode_t) filemode))
1559 0 : pg_log_error("could not set permissions on file \"%s\": %m",
1560 : filename);
1561 : #endif
1562 :
1563 83642 : if (current_len_left == 0)
1564 : {
1565 : /*
1566 : * Done with this file, next one will be a new tar header
1567 : */
1568 14976 : fclose(file);
1569 14976 : file = NULL;
1570 14976 : continue;
1571 : }
1572 : } /* new file */
1573 : else
1574 : {
1575 : /*
1576 : * Continuing blocks in existing file
1577 : */
1578 103218 : if (current_len_left == 0 && r == current_padding)
1579 : {
1580 : /*
1581 : * Received the padding block for this file, ignore it and
1582 : * close the file, then move on to the next tar header.
1583 : */
1584 924 : fclose(file);
1585 924 : file = NULL;
1586 924 : totaldone += r;
1587 924 : continue;
1588 : }
1589 :
1590 102294 : if (fwrite(copybuf, r, 1, file) != 1)
1591 : {
1592 0 : pg_log_error("could not write to file \"%s\": %m", filename);
1593 0 : exit(1);
1594 : }
1595 102294 : totaldone += r;
1596 102294 : progress_report(rownum, filename, false);
1597 :
1598 102294 : current_len_left -= r;
1599 102294 : if (current_len_left == 0 && current_padding == 0)
1600 : {
1601 : /*
1602 : * Received the last block, and there is no padding to be
1603 : * expected. Close the file and move on to the next tar
1604 : * header.
1605 : */
1606 67742 : fclose(file);
1607 67742 : file = NULL;
1608 67742 : continue;
1609 : }
1610 : } /* continuing data in existing file */
1611 : } /* loop over all data blocks */
1612 88 : progress_report(rownum, filename, true);
1613 :
1614 88 : if (file != NULL)
1615 : {
1616 0 : pg_log_error("COPY stream ended before last file was finished");
1617 0 : exit(1);
1618 : }
1619 :
1620 88 : if (copybuf != NULL)
1621 0 : PQfreemem(copybuf);
1622 :
1623 88 : if (basetablespace && writerecoveryconf)
1624 4 : WriteRecoveryConf();
1625 :
1626 : /*
1627 : * No data is synced here, everything is done for all tablespaces at the
1628 : * end.
1629 : */
1630 88 : }
1631 :
1632 : /*
1633 : * Escape a string so that it can be used as a value in a key-value pair
1634 : * a configuration file.
1635 : */
1636 : static char *
1637 6 : escape_quotes(const char *src)
1638 : {
1639 6 : char *result = escape_single_quotes_ascii(src);
1640 :
1641 6 : if (!result)
1642 : {
1643 0 : pg_log_error("out of memory");
1644 0 : exit(1);
1645 : }
1646 6 : return result;
1647 : }
1648 :
1649 : /*
1650 : * Create a configuration file in memory using a PQExpBuffer
1651 : */
1652 : static void
1653 4 : GenerateRecoveryConf(PGconn *conn)
1654 : {
1655 : PQconninfoOption *connOptions;
1656 : PQconninfoOption *option;
1657 : PQExpBufferData conninfo_buf;
1658 : char *escaped;
1659 :
1660 4 : recoveryconfcontents = createPQExpBuffer();
1661 4 : if (!recoveryconfcontents)
1662 : {
1663 0 : pg_log_error("out of memory");
1664 0 : exit(1);
1665 : }
1666 :
1667 : /*
1668 : * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by
1669 : * standby.signal to trigger a standby state at recovery.
1670 : */
1671 4 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
1672 0 : appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
1673 :
1674 4 : connOptions = PQconninfo(conn);
1675 4 : if (connOptions == NULL)
1676 : {
1677 0 : pg_log_error("out of memory");
1678 0 : exit(1);
1679 : }
1680 :
1681 4 : initPQExpBuffer(&conninfo_buf);
1682 124 : for (option = connOptions; option && option->keyword; option++)
1683 : {
1684 : /* Omit empty settings and those libpqwalreceiver overrides. */
1685 236 : if (strcmp(option->keyword, "replication") == 0 ||
1686 228 : strcmp(option->keyword, "dbname") == 0 ||
1687 220 : strcmp(option->keyword, "fallback_application_name") == 0 ||
1688 152 : (option->val == NULL) ||
1689 88 : (option->val != NULL && option->val[0] == '\0'))
1690 84 : continue;
1691 :
1692 : /* Separate key-value pairs with spaces */
1693 36 : if (conninfo_buf.len != 0)
1694 32 : appendPQExpBufferChar(&conninfo_buf, ' ');
1695 :
1696 : /*
1697 : * Write "keyword=value" pieces, the value string is escaped and/or
1698 : * quoted if necessary.
1699 : */
1700 36 : appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword);
1701 36 : appendConnStrVal(&conninfo_buf, option->val);
1702 : }
1703 :
1704 : /*
1705 : * Escape the connection string, so that it can be put in the config file.
1706 : * Note that this is different from the escaping of individual connection
1707 : * options above!
1708 : */
1709 4 : escaped = escape_quotes(conninfo_buf.data);
1710 4 : appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
1711 4 : free(escaped);
1712 :
1713 4 : if (replication_slot)
1714 : {
1715 2 : escaped = escape_quotes(replication_slot);
1716 2 : appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n", replication_slot);
1717 2 : free(escaped);
1718 : }
1719 :
1720 8 : if (PQExpBufferBroken(recoveryconfcontents) ||
1721 4 : PQExpBufferDataBroken(conninfo_buf))
1722 : {
1723 0 : pg_log_error("out of memory");
1724 0 : exit(1);
1725 : }
1726 :
1727 4 : termPQExpBuffer(&conninfo_buf);
1728 :
1729 4 : PQconninfoFree(connOptions);
1730 4 : }
1731 :
1732 :
1733 : /*
1734 : * Write the configuration file into the directory specified in basedir,
1735 : * with the contents already collected in memory appended. Then write
1736 : * the signal file into the basedir. If the server does not support
1737 : * recovery parameters as GUCs, the signal file is not necessary, and
1738 : * configuration is written to recovery.conf.
1739 : */
1740 : static void
1741 4 : WriteRecoveryConf(void)
1742 : {
1743 : char filename[MAXPGPATH];
1744 : FILE *cf;
1745 4 : bool is_recovery_guc_supported = true;
1746 :
1747 4 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
1748 0 : is_recovery_guc_supported = false;
1749 :
1750 4 : snprintf(filename, MAXPGPATH, "%s/%s", basedir,
1751 : is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf");
1752 :
1753 4 : cf = fopen(filename, is_recovery_guc_supported ? "a" : "w");
1754 4 : if (cf == NULL)
1755 : {
1756 0 : pg_log_error("could not open file \"%s\": %m", filename);
1757 0 : exit(1);
1758 : }
1759 :
1760 4 : if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
1761 : {
1762 0 : pg_log_error("could not write to file \"%s\": %m", filename);
1763 0 : exit(1);
1764 : }
1765 :
1766 4 : fclose(cf);
1767 :
1768 4 : if (is_recovery_guc_supported)
1769 : {
1770 4 : snprintf(filename, MAXPGPATH, "%s/%s", basedir, "standby.signal");
1771 4 : cf = fopen(filename, "w");
1772 4 : if (cf == NULL)
1773 : {
1774 0 : pg_log_error("could not create file \"%s\": %m", filename);
1775 0 : exit(1);
1776 : }
1777 :
1778 4 : fclose(cf);
1779 : }
1780 4 : }
1781 :
1782 :
1783 : static void
1784 98 : BaseBackup(void)
1785 : {
1786 : PGresult *res;
1787 : char *sysidentifier;
1788 : TimeLineID latesttli;
1789 : TimeLineID starttli;
1790 : char *basebkp;
1791 : char escaped_label[MAXPGPATH];
1792 98 : char *maxrate_clause = NULL;
1793 : int i;
1794 : char xlogstart[64];
1795 : char xlogend[64];
1796 : int minServerMajor,
1797 : maxServerMajor;
1798 : int serverVersion,
1799 : serverMajor;
1800 :
1801 : Assert(conn != NULL);
1802 :
1803 : /*
1804 : * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1805 : * can't work with servers older than 9.1.
1806 : */
1807 98 : minServerMajor = 901;
1808 98 : maxServerMajor = PG_VERSION_NUM / 100;
1809 98 : serverVersion = PQserverVersion(conn);
1810 98 : serverMajor = serverVersion / 100;
1811 98 : if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1812 : {
1813 0 : const char *serverver = PQparameterStatus(conn, "server_version");
1814 :
1815 0 : pg_log_error("incompatible server version %s",
1816 : serverver ? serverver : "'unknown'");
1817 0 : exit(1);
1818 : }
1819 :
1820 : /*
1821 : * If WAL streaming was requested, also check that the server is new
1822 : * enough for that.
1823 : */
1824 98 : if (includewal == STREAM_WAL && !CheckServerVersionForStreaming(conn))
1825 : {
1826 : /*
1827 : * Error message already written in CheckServerVersionForStreaming(),
1828 : * but add a hint about using -X none.
1829 : */
1830 0 : pg_log_info("HINT: use -X none or -X fetch to disable log streaming");
1831 0 : exit(1);
1832 : }
1833 :
1834 : /*
1835 : * Build contents of configuration file if requested
1836 : */
1837 98 : if (writerecoveryconf)
1838 4 : GenerateRecoveryConf(conn);
1839 :
1840 : /*
1841 : * Run IDENTIFY_SYSTEM so we can get the timeline
1842 : */
1843 98 : if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1844 0 : exit(1);
1845 :
1846 : /*
1847 : * Start the actual backup
1848 : */
1849 98 : PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1850 :
1851 98 : if (maxrate > 0)
1852 0 : maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1853 :
1854 98 : if (verbose)
1855 0 : pg_log_info("initiating base backup, waiting for checkpoint to complete");
1856 :
1857 98 : if (showprogress && !verbose)
1858 : {
1859 0 : fprintf(stderr, "waiting for checkpoint");
1860 0 : if (isatty(fileno(stderr)))
1861 0 : fprintf(stderr, "\r");
1862 : else
1863 0 : fprintf(stderr, "\n");
1864 : }
1865 :
1866 98 : basebkp =
1867 588 : psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
1868 : escaped_label,
1869 98 : showprogress ? "PROGRESS" : "",
1870 98 : includewal == FETCH_WAL ? "WAL" : "",
1871 98 : fastcheckpoint ? "FAST" : "",
1872 98 : includewal == NO_WAL ? "" : "NOWAIT",
1873 : maxrate_clause ? maxrate_clause : "",
1874 98 : format == 't' ? "TABLESPACE_MAP" : "",
1875 98 : verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
1876 :
1877 98 : if (PQsendQuery(conn, basebkp) == 0)
1878 : {
1879 0 : pg_log_error("could not send replication command \"%s\": %s",
1880 : "BASE_BACKUP", PQerrorMessage(conn));
1881 0 : exit(1);
1882 : }
1883 :
1884 : /*
1885 : * Get the starting WAL location
1886 : */
1887 98 : res = PQgetResult(conn);
1888 98 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
1889 : {
1890 0 : pg_log_error("could not initiate base backup: %s",
1891 : PQerrorMessage(conn));
1892 0 : exit(1);
1893 : }
1894 98 : if (PQntuples(res) != 1)
1895 : {
1896 0 : pg_log_error("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields",
1897 : PQntuples(res), PQnfields(res), 1, 2);
1898 0 : exit(1);
1899 : }
1900 :
1901 98 : strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1902 :
1903 98 : if (verbose)
1904 0 : pg_log_info("checkpoint completed");
1905 :
1906 : /*
1907 : * 9.3 and later sends the TLI of the starting point. With older servers,
1908 : * assume it's the same as the latest timeline reported by
1909 : * IDENTIFY_SYSTEM.
1910 : */
1911 98 : if (PQnfields(res) >= 2)
1912 98 : starttli = atoi(PQgetvalue(res, 0, 1));
1913 : else
1914 0 : starttli = latesttli;
1915 98 : PQclear(res);
1916 98 : MemSet(xlogend, 0, sizeof(xlogend));
1917 :
1918 98 : if (verbose && includewal != NO_WAL)
1919 0 : pg_log_info("write-ahead log start point: %s on timeline %u",
1920 : xlogstart, starttli);
1921 :
1922 : /*
1923 : * Get the header
1924 : */
1925 98 : res = PQgetResult(conn);
1926 98 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
1927 : {
1928 0 : pg_log_error("could not get backup header: %s",
1929 : PQerrorMessage(conn));
1930 0 : exit(1);
1931 : }
1932 98 : if (PQntuples(res) < 1)
1933 : {
1934 0 : pg_log_error("no data returned from server");
1935 0 : exit(1);
1936 : }
1937 :
1938 : /*
1939 : * Sum up the total size, for progress reporting
1940 : */
1941 98 : totalsize = totaldone = 0;
1942 98 : tablespacecount = PQntuples(res);
1943 202 : for (i = 0; i < PQntuples(res); i++)
1944 : {
1945 106 : totalsize += atol(PQgetvalue(res, i, 2));
1946 :
1947 : /*
1948 : * Verify tablespace directories are empty. Don't bother with the
1949 : * first once since it can be relocated, and it will be checked before
1950 : * we do anything anyway.
1951 : */
1952 106 : if (format == 'p' && !PQgetisnull(res, i, 1))
1953 : {
1954 6 : char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1)));
1955 :
1956 6 : verify_dir_is_empty_or_create(path, &made_tablespace_dirs, &found_tablespace_dirs);
1957 : }
1958 : }
1959 :
1960 : /*
1961 : * When writing to stdout, require a single tablespace
1962 : */
1963 96 : if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1964 : {
1965 0 : pg_log_error("can only write single tablespace to stdout, database has %d",
1966 : PQntuples(res));
1967 0 : exit(1);
1968 : }
1969 :
1970 : /*
1971 : * If we're streaming WAL, start the streaming session before we start
1972 : * receiving the actual data chunks.
1973 : */
1974 96 : if (includewal == STREAM_WAL)
1975 : {
1976 92 : if (verbose)
1977 0 : pg_log_info("starting background WAL receiver");
1978 92 : StartLogStreamer(xlogstart, starttli, sysidentifier);
1979 : }
1980 :
1981 : /*
1982 : * Start receiving chunks
1983 : */
1984 196 : for (i = 0; i < PQntuples(res); i++)
1985 : {
1986 102 : if (format == 't')
1987 14 : ReceiveTarFile(conn, res, i);
1988 : else
1989 88 : ReceiveAndUnpackTarFile(conn, res, i);
1990 : } /* Loop over all tablespaces */
1991 :
1992 94 : if (showprogress)
1993 : {
1994 0 : progress_report(PQntuples(res), NULL, true);
1995 0 : if (isatty(fileno(stderr)))
1996 0 : fprintf(stderr, "\n"); /* Need to move to next line */
1997 : }
1998 :
1999 94 : PQclear(res);
2000 :
2001 : /*
2002 : * Get the stop position
2003 : */
2004 94 : res = PQgetResult(conn);
2005 94 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
2006 : {
2007 2 : pg_log_error("could not get write-ahead log end position from server: %s",
2008 : PQerrorMessage(conn));
2009 2 : exit(1);
2010 : }
2011 92 : if (PQntuples(res) != 1)
2012 : {
2013 0 : pg_log_error("no write-ahead log end position returned from server");
2014 0 : exit(1);
2015 : }
2016 92 : strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
2017 92 : if (verbose && includewal != NO_WAL)
2018 0 : pg_log_info("write-ahead log end point: %s", xlogend);
2019 92 : PQclear(res);
2020 :
2021 92 : res = PQgetResult(conn);
2022 92 : if (PQresultStatus(res) != PGRES_COMMAND_OK)
2023 : {
2024 6 : const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
2025 :
2026 12 : if (sqlstate &&
2027 6 : strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
2028 : {
2029 6 : pg_log_error("checksum error occurred");
2030 6 : checksum_failure = true;
2031 : }
2032 : else
2033 : {
2034 0 : pg_log_error("final receive failed: %s",
2035 : PQerrorMessage(conn));
2036 : }
2037 6 : exit(1);
2038 : }
2039 :
2040 86 : if (bgchild > 0)
2041 : {
2042 : #ifndef WIN32
2043 : int status;
2044 : pid_t r;
2045 : #else
2046 : DWORD status;
2047 :
2048 : /*
2049 : * get a pointer sized version of bgchild to avoid warnings about
2050 : * casting to a different size on WIN64.
2051 : */
2052 : intptr_t bgchild_handle = bgchild;
2053 : uint32 hi,
2054 : lo;
2055 : #endif
2056 :
2057 82 : if (verbose)
2058 0 : pg_log_info("waiting for background process to finish streaming ...");
2059 :
2060 : #ifndef WIN32
2061 82 : if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
2062 : {
2063 0 : pg_log_info("could not send command to background pipe: %m");
2064 0 : exit(1);
2065 : }
2066 :
2067 : /* Just wait for the background process to exit */
2068 82 : r = waitpid(bgchild, &status, 0);
2069 82 : if (r == (pid_t) -1)
2070 : {
2071 0 : pg_log_error("could not wait for child process: %m");
2072 0 : exit(1);
2073 : }
2074 82 : if (r != bgchild)
2075 : {
2076 0 : pg_log_error("child %d died, expected %d", (int) r, (int) bgchild);
2077 0 : exit(1);
2078 : }
2079 82 : if (status != 0)
2080 : {
2081 2 : pg_log_error("%s", wait_result_to_str(status));
2082 2 : exit(1);
2083 : }
2084 : /* Exited normally, we're happy! */
2085 : #else /* WIN32 */
2086 :
2087 : /*
2088 : * On Windows, since we are in the same process, we can just store the
2089 : * value directly in the variable, and then set the flag that says
2090 : * it's there.
2091 : */
2092 : if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
2093 : {
2094 : pg_log_error("could not parse write-ahead log location \"%s\"",
2095 : xlogend);
2096 : exit(1);
2097 : }
2098 : xlogendptr = ((uint64) hi) << 32 | lo;
2099 : InterlockedIncrement(&has_xlogendptr);
2100 :
2101 : /* First wait for the thread to exit */
2102 : if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
2103 : WAIT_OBJECT_0)
2104 : {
2105 : _dosmaperr(GetLastError());
2106 : pg_log_error("could not wait for child thread: %m");
2107 : exit(1);
2108 : }
2109 : if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
2110 : {
2111 : _dosmaperr(GetLastError());
2112 : pg_log_error("could not get child thread exit status: %m");
2113 : exit(1);
2114 : }
2115 : if (status != 0)
2116 : {
2117 : pg_log_error("child thread exited with error %u",
2118 : (unsigned int) status);
2119 : exit(1);
2120 : }
2121 : /* Exited normally, we're happy */
2122 : #endif
2123 : }
2124 :
2125 : /* Free the configuration file contents */
2126 84 : destroyPQExpBuffer(recoveryconfcontents);
2127 :
2128 : /*
2129 : * End of copy data. Final result is already checked inside the loop.
2130 : */
2131 84 : PQclear(res);
2132 84 : PQfinish(conn);
2133 84 : conn = NULL;
2134 :
2135 : /*
2136 : * Make data persistent on disk once backup is completed. For tar format
2137 : * once syncing the parent directory is fine, each tar file created per
2138 : * tablespace has been already synced. In plain format, all the data of
2139 : * the base directory is synced, taking into account all the tablespaces.
2140 : * Errors are not considered fatal.
2141 : */
2142 84 : if (do_sync)
2143 : {
2144 34 : if (verbose)
2145 0 : pg_log_info("syncing data to disk ...");
2146 34 : if (format == 't')
2147 : {
2148 8 : if (strcmp(basedir, "-") != 0)
2149 8 : (void) fsync_fname(basedir, true);
2150 : }
2151 : else
2152 : {
2153 26 : (void) fsync_pgdata(basedir, serverVersion);
2154 : }
2155 : }
2156 :
2157 84 : if (verbose)
2158 0 : pg_log_info("base backup completed");
2159 84 : }
2160 :
2161 :
2162 : int
2163 128 : main(int argc, char **argv)
2164 : {
2165 : static struct option long_options[] = {
2166 : {"help", no_argument, NULL, '?'},
2167 : {"version", no_argument, NULL, 'V'},
2168 : {"pgdata", required_argument, NULL, 'D'},
2169 : {"format", required_argument, NULL, 'F'},
2170 : {"checkpoint", required_argument, NULL, 'c'},
2171 : {"create-slot", no_argument, NULL, 'C'},
2172 : {"max-rate", required_argument, NULL, 'r'},
2173 : {"write-recovery-conf", no_argument, NULL, 'R'},
2174 : {"slot", required_argument, NULL, 'S'},
2175 : {"tablespace-mapping", required_argument, NULL, 'T'},
2176 : {"wal-method", required_argument, NULL, 'X'},
2177 : {"gzip", no_argument, NULL, 'z'},
2178 : {"compress", required_argument, NULL, 'Z'},
2179 : {"label", required_argument, NULL, 'l'},
2180 : {"no-clean", no_argument, NULL, 'n'},
2181 : {"no-sync", no_argument, NULL, 'N'},
2182 : {"dbname", required_argument, NULL, 'd'},
2183 : {"host", required_argument, NULL, 'h'},
2184 : {"port", required_argument, NULL, 'p'},
2185 : {"username", required_argument, NULL, 'U'},
2186 : {"no-password", no_argument, NULL, 'w'},
2187 : {"password", no_argument, NULL, 'W'},
2188 : {"status-interval", required_argument, NULL, 's'},
2189 : {"verbose", no_argument, NULL, 'v'},
2190 : {"progress", no_argument, NULL, 'P'},
2191 : {"waldir", required_argument, NULL, 1},
2192 : {"no-slot", no_argument, NULL, 2},
2193 : {"no-verify-checksums", no_argument, NULL, 3},
2194 : {NULL, 0, NULL, 0}
2195 : };
2196 : int c;
2197 :
2198 : int option_index;
2199 :
2200 128 : pg_logging_init(argv[0]);
2201 128 : progname = get_progname(argv[0]);
2202 128 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2203 :
2204 128 : if (argc > 1)
2205 : {
2206 126 : if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2207 : {
2208 2 : usage();
2209 2 : exit(0);
2210 : }
2211 124 : else if (strcmp(argv[1], "-V") == 0
2212 124 : || strcmp(argv[1], "--version") == 0)
2213 : {
2214 2 : puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2215 2 : exit(0);
2216 : }
2217 : }
2218 :
2219 124 : atexit(cleanup_directories_atexit);
2220 :
2221 124 : while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
2222 : long_options, &option_index)) != -1)
2223 : {
2224 370 : switch (c)
2225 : {
2226 : case 'C':
2227 8 : create_slot = true;
2228 8 : break;
2229 : case 'D':
2230 120 : basedir = pg_strdup(optarg);
2231 120 : break;
2232 : case 'F':
2233 28 : if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2234 18 : format = 'p';
2235 10 : else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2236 10 : format = 't';
2237 : else
2238 : {
2239 0 : pg_log_error("invalid output format \"%s\", must be \"plain\" or \"tar\"\n",
2240 : optarg);
2241 0 : exit(1);
2242 : }
2243 28 : break;
2244 : case 'r':
2245 0 : maxrate = parse_max_rate(optarg);
2246 0 : break;
2247 : case 'R':
2248 4 : writerecoveryconf = true;
2249 4 : break;
2250 : case 'S':
2251 :
2252 : /*
2253 : * When specifying replication slot name, use a permanent
2254 : * slot.
2255 : */
2256 14 : replication_slot = pg_strdup(optarg);
2257 14 : temp_replication_slot = false;
2258 14 : break;
2259 : case 2:
2260 4 : no_slot = true;
2261 4 : break;
2262 : case 'T':
2263 16 : tablespace_list_append(optarg);
2264 4 : break;
2265 : case 'X':
2266 36 : if (strcmp(optarg, "n") == 0 ||
2267 18 : strcmp(optarg, "none") == 0)
2268 : {
2269 4 : includewal = NO_WAL;
2270 : }
2271 28 : else if (strcmp(optarg, "f") == 0 ||
2272 14 : strcmp(optarg, "fetch") == 0)
2273 : {
2274 2 : includewal = FETCH_WAL;
2275 : }
2276 24 : else if (strcmp(optarg, "s") == 0 ||
2277 12 : strcmp(optarg, "stream") == 0)
2278 : {
2279 12 : includewal = STREAM_WAL;
2280 : }
2281 : else
2282 : {
2283 0 : pg_log_error("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
2284 : optarg);
2285 0 : exit(1);
2286 : }
2287 18 : break;
2288 : case 1:
2289 2 : xlog_dir = pg_strdup(optarg);
2290 2 : break;
2291 : case 'l':
2292 0 : label = pg_strdup(optarg);
2293 0 : break;
2294 : case 'n':
2295 2 : noclean = true;
2296 2 : break;
2297 : case 'N':
2298 50 : do_sync = false;
2299 50 : break;
2300 : case 'z':
2301 : #ifdef HAVE_LIBZ
2302 0 : compresslevel = Z_DEFAULT_COMPRESSION;
2303 : #else
2304 : compresslevel = 1; /* will be rejected below */
2305 : #endif
2306 0 : break;
2307 : case 'Z':
2308 0 : compresslevel = atoi(optarg);
2309 0 : if (compresslevel < 0 || compresslevel > 9)
2310 : {
2311 0 : pg_log_error("invalid compression level \"%s\"\n", optarg);
2312 0 : exit(1);
2313 : }
2314 0 : break;
2315 : case 'c':
2316 0 : if (pg_strcasecmp(optarg, "fast") == 0)
2317 0 : fastcheckpoint = true;
2318 0 : else if (pg_strcasecmp(optarg, "spread") == 0)
2319 0 : fastcheckpoint = false;
2320 : else
2321 : {
2322 0 : pg_log_error("invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"",
2323 : optarg);
2324 0 : exit(1);
2325 : }
2326 0 : break;
2327 : case 'd':
2328 0 : connection_string = pg_strdup(optarg);
2329 0 : break;
2330 : case 'h':
2331 50 : dbhost = pg_strdup(optarg);
2332 50 : break;
2333 : case 'p':
2334 50 : dbport = pg_strdup(optarg);
2335 50 : break;
2336 : case 'U':
2337 0 : dbuser = pg_strdup(optarg);
2338 0 : break;
2339 : case 'w':
2340 0 : dbgetpassword = -1;
2341 0 : break;
2342 : case 'W':
2343 0 : dbgetpassword = 1;
2344 0 : break;
2345 : case 's':
2346 0 : standby_message_timeout = atoi(optarg) * 1000;
2347 0 : if (standby_message_timeout < 0)
2348 : {
2349 0 : pg_log_error("invalid status interval \"%s\"", optarg);
2350 0 : exit(1);
2351 : }
2352 0 : break;
2353 : case 'v':
2354 0 : verbose++;
2355 0 : break;
2356 : case 'P':
2357 0 : showprogress = true;
2358 0 : break;
2359 : case 3:
2360 2 : verify_checksums = false;
2361 2 : break;
2362 : default:
2363 :
2364 : /*
2365 : * getopt_long already emitted a complaint
2366 : */
2367 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2368 : progname);
2369 2 : exit(1);
2370 : }
2371 : }
2372 :
2373 : /*
2374 : * Any non-option arguments?
2375 : */
2376 110 : if (optind < argc)
2377 : {
2378 0 : pg_log_error("too many command-line arguments (first is \"%s\")",
2379 : argv[optind]);
2380 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2381 : progname);
2382 0 : exit(1);
2383 : }
2384 :
2385 : /*
2386 : * Required arguments
2387 : */
2388 110 : if (basedir == NULL)
2389 : {
2390 2 : pg_log_error("no target directory specified");
2391 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2392 : progname);
2393 2 : exit(1);
2394 : }
2395 :
2396 : /*
2397 : * Mutually exclusive arguments
2398 : */
2399 108 : if (format == 'p' && compresslevel != 0)
2400 : {
2401 0 : pg_log_error("only tar mode backups can be compressed");
2402 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2403 : progname);
2404 0 : exit(1);
2405 : }
2406 :
2407 108 : if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2408 : {
2409 0 : pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
2410 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2411 : progname);
2412 0 : exit(1);
2413 : }
2414 :
2415 108 : if (replication_slot && includewal != STREAM_WAL)
2416 : {
2417 2 : pg_log_error("replication slots can only be used with WAL streaming");
2418 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2419 : progname);
2420 2 : exit(1);
2421 : }
2422 :
2423 106 : if (no_slot)
2424 : {
2425 4 : if (replication_slot)
2426 : {
2427 2 : pg_log_error("--no-slot cannot be used with slot name");
2428 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2429 : progname);
2430 2 : exit(1);
2431 : }
2432 2 : temp_replication_slot = false;
2433 : }
2434 :
2435 104 : if (create_slot)
2436 : {
2437 6 : if (!replication_slot)
2438 : {
2439 2 : pg_log_error("%s needs a slot to be specified using --slot",
2440 : "--create-slot");
2441 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2442 : progname);
2443 2 : exit(1);
2444 : }
2445 :
2446 4 : if (no_slot)
2447 : {
2448 0 : pg_log_error("--create-slot and --no-slot are incompatible options");
2449 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2450 : progname);
2451 0 : exit(1);
2452 : }
2453 : }
2454 :
2455 102 : if (xlog_dir)
2456 : {
2457 2 : if (format != 'p')
2458 : {
2459 0 : pg_log_error("WAL directory location can only be specified in plain mode");
2460 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2461 : progname);
2462 0 : exit(1);
2463 : }
2464 :
2465 : /* clean up xlog directory name, check it's absolute */
2466 2 : canonicalize_path(xlog_dir);
2467 2 : if (!is_absolute_path(xlog_dir))
2468 : {
2469 0 : pg_log_error("WAL directory location must be an absolute path");
2470 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2471 : progname);
2472 0 : exit(1);
2473 : }
2474 : }
2475 :
2476 : #ifndef HAVE_LIBZ
2477 : if (compresslevel != 0)
2478 : {
2479 : pg_log_error("this build does not support compression");
2480 : exit(1);
2481 : }
2482 : #endif
2483 :
2484 : /* connection in replication mode to server */
2485 102 : conn = GetConnection();
2486 102 : if (!conn)
2487 : {
2488 : /* Error message already written in GetConnection() */
2489 4 : exit(1);
2490 : }
2491 98 : atexit(disconnect_atexit);
2492 :
2493 : /*
2494 : * Set umask so that directories/files are created with the same
2495 : * permissions as directories/files in the source data directory.
2496 : *
2497 : * pg_mode_mask is set to owner-only by default and then updated in
2498 : * GetConnection() where we get the mode from the server-side with
2499 : * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
2500 : */
2501 98 : umask(pg_mode_mask);
2502 :
2503 : /*
2504 : * Verify that the target directory exists, or create it. For plaintext
2505 : * backups, always require the directory. For tar backups, require it
2506 : * unless we are writing to stdout.
2507 : */
2508 98 : if (format == 'p' || strcmp(basedir, "-") != 0)
2509 98 : verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata);
2510 :
2511 : /* determine remote server's xlog segment size */
2512 98 : if (!RetrieveWalSegSize(conn))
2513 0 : exit(1);
2514 :
2515 : /* Create pg_wal symlink, if required */
2516 98 : if (xlog_dir)
2517 : {
2518 : char *linkloc;
2519 :
2520 2 : verify_dir_is_empty_or_create(xlog_dir, &made_new_xlogdir, &found_existing_xlogdir);
2521 :
2522 : /*
2523 : * Form name of the place where the symlink must go. pg_xlog has been
2524 : * renamed to pg_wal in post-10 clusters.
2525 : */
2526 2 : linkloc = psprintf("%s/%s", basedir,
2527 2 : PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
2528 : "pg_xlog" : "pg_wal");
2529 :
2530 : #ifdef HAVE_SYMLINK
2531 2 : if (symlink(xlog_dir, linkloc) != 0)
2532 : {
2533 0 : pg_log_error("could not create symbolic link \"%s\": %m", linkloc);
2534 0 : exit(1);
2535 : }
2536 : #else
2537 : pg_log_error("symlinks are not supported on this platform");
2538 : exit(1);
2539 : #endif
2540 2 : free(linkloc);
2541 : }
2542 :
2543 98 : BaseBackup();
2544 :
2545 84 : success = true;
2546 84 : return 0;
2547 : }
|
