Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_basebackup.c - receive a base backup using streaming replication protocol
4 : *
5 : * Author: Magnus Hagander <magnus@hagander.net>
6 : *
7 : * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
8 : *
9 : * IDENTIFICATION
10 : * src/bin/pg_basebackup/pg_basebackup.c
11 : *-------------------------------------------------------------------------
12 : */
13 :
14 : #include "postgres_fe.h"
15 :
16 : #include <unistd.h>
17 : #include <dirent.h>
18 : #include <sys/stat.h>
19 : #include <sys/wait.h>
20 : #include <signal.h>
21 : #include <time.h>
22 : #ifdef HAVE_SYS_SELECT_H
23 : #include <sys/select.h>
24 : #endif
25 : #ifdef HAVE_LIBZ
26 : #include <zlib.h>
27 : #endif
28 :
29 : #include "access/xlog_internal.h"
30 : #include "common/file_perm.h"
31 : #include "common/file_utils.h"
32 : #include "common/logging.h"
33 : #include "common/string.h"
34 : #include "fe_utils/string_utils.h"
35 : #include "getopt_long.h"
36 : #include "libpq-fe.h"
37 : #include "pqexpbuffer.h"
38 : #include "pgtar.h"
39 : #include "pgtime.h"
40 : #include "receivelog.h"
41 : #include "replication/basebackup.h"
42 : #include "streamutil.h"
43 :
44 : #define ERRCODE_DATA_CORRUPTED "XX001"
45 :
46 : typedef struct TablespaceListCell
47 : {
48 : struct TablespaceListCell *next;
49 : char old_dir[MAXPGPATH];
50 : char new_dir[MAXPGPATH];
51 : } TablespaceListCell;
52 :
53 : typedef struct TablespaceList
54 : {
55 : TablespaceListCell *head;
56 : TablespaceListCell *tail;
57 : } TablespaceList;
58 :
59 : /*
60 : * pg_xlog has been renamed to pg_wal in version 10. This version number
61 : * should be compared with PQserverVersion().
62 : */
63 : #define MINIMUM_VERSION_FOR_PG_WAL 100000
64 :
65 : /*
66 : * Temporary replication slots are supported from version 10.
67 : */
68 : #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000
69 :
70 : /*
71 : * recovery.conf is integrated into postgresql.conf from version 12.
72 : */
73 : #define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000
74 :
75 : /*
76 : * Different ways to include WAL
77 : */
78 : typedef enum
79 : {
80 : NO_WAL,
81 : FETCH_WAL,
82 : STREAM_WAL
83 : } IncludeWal;
84 :
85 : /* Global options */
86 : static char *basedir = NULL;
87 : static TablespaceList tablespace_dirs = {NULL, NULL};
88 : static char *xlog_dir = NULL;
89 : static char format = 'p'; /* p(lain)/t(ar) */
90 : static char *label = "pg_basebackup base backup";
91 : static bool noclean = false;
92 : static bool checksum_failure = false;
93 : static bool showprogress = false;
94 : static int verbose = 0;
95 : static int compresslevel = 0;
96 : static IncludeWal includewal = STREAM_WAL;
97 : static bool fastcheckpoint = false;
98 : static bool writerecoveryconf = false;
99 : static bool do_sync = true;
100 : static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
101 : static pg_time_t last_progress_report = 0;
102 : static int32 maxrate = 0; /* no limit by default */
103 : static char *replication_slot = NULL;
104 : static bool temp_replication_slot = true;
105 : static bool create_slot = false;
106 : static bool no_slot = false;
107 : static bool verify_checksums = true;
108 :
109 : static bool success = false;
110 : static bool made_new_pgdata = false;
111 : static bool found_existing_pgdata = false;
112 : static bool made_new_xlogdir = false;
113 : static bool found_existing_xlogdir = false;
114 : static bool made_tablespace_dirs = false;
115 : static bool found_tablespace_dirs = false;
116 :
117 : /* Progress counters */
118 : static uint64 totalsize;
119 : static uint64 totaldone;
120 : static int tablespacecount;
121 :
122 : /* Pipe to communicate with background wal receiver process */
123 : #ifndef WIN32
124 : static int bgpipe[2] = {-1, -1};
125 : #endif
126 :
127 : /* Handle to child process */
128 : static pid_t bgchild = -1;
129 : static bool in_log_streamer = false;
130 :
131 : /* End position for xlog streaming, empty string if unknown yet */
132 : static XLogRecPtr xlogendptr;
133 :
134 : #ifndef WIN32
135 : static int has_xlogendptr = 0;
136 : #else
137 : static volatile LONG has_xlogendptr = 0;
138 : #endif
139 :
140 : /* Contents of configuration file to be generated */
141 : static PQExpBuffer recoveryconfcontents = NULL;
142 :
143 : /* Function headers */
144 : static void usage(void);
145 : static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found);
146 : static void progress_report(int tablespacenum, const char *filename, bool force);
147 :
148 : static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
149 : static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
150 : static void GenerateRecoveryConf(PGconn *conn);
151 : static void WriteRecoveryConf(void);
152 : static void BaseBackup(void);
153 :
154 : static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline,
155 : bool segment_finished);
156 :
157 : static const char *get_tablespace_mapping(const char *dir);
158 : static void tablespace_list_append(const char *arg);
159 :
160 :
161 : static void
162 206 : cleanup_directories_atexit(void)
163 : {
164 206 : if (success || in_log_streamer)
165 166 : return;
166 :
167 40 : if (!noclean && !checksum_failure)
168 : {
169 32 : if (made_new_pgdata)
170 : {
171 8 : pg_log_info("removing data directory \"%s\"", basedir);
172 8 : if (!rmtree(basedir, true))
173 0 : pg_log_error("failed to remove data directory");
174 : }
175 24 : else if (found_existing_pgdata)
176 : {
177 0 : pg_log_info("removing contents of data directory \"%s\"", basedir);
178 0 : if (!rmtree(basedir, false))
179 0 : pg_log_error("failed to remove contents of data directory");
180 : }
181 :
182 64 : if (made_new_xlogdir)
183 : {
184 0 : pg_log_info("removing WAL directory \"%s\"", xlog_dir);
185 0 : if (!rmtree(xlog_dir, true))
186 0 : pg_log_error("failed to remove WAL directory");
187 : }
188 32 : else if (found_existing_xlogdir)
189 : {
190 0 : pg_log_info("removing contents of WAL directory \"%s\"", xlog_dir);
191 0 : if (!rmtree(xlog_dir, false))
192 0 : pg_log_error("failed to remove contents of WAL directory");
193 : }
194 : }
195 : else
196 : {
197 8 : if ((made_new_pgdata || found_existing_pgdata) && !checksum_failure)
198 0 : pg_log_info("data directory \"%s\" not removed at user's request", basedir);
199 :
200 8 : if (made_new_xlogdir || found_existing_xlogdir)
201 0 : pg_log_info("WAL directory \"%s\" not removed at user's request", xlog_dir);
202 : }
203 :
204 40 : if ((made_tablespace_dirs || found_tablespace_dirs) && !checksum_failure)
205 0 : pg_log_info("changes to tablespace directories will not be undone");
206 : }
207 :
208 : static void
209 180 : disconnect_atexit(void)
210 : {
211 180 : if (conn != NULL)
212 96 : PQfinish(conn);
213 180 : }
214 :
215 : #ifndef WIN32
216 : /*
217 : * On windows, our background thread dies along with the process. But on
218 : * Unix, if we have started a subprocess, we want to kill it off so it
219 : * doesn't remain running trying to stream data.
220 : */
221 : static void
222 90 : kill_bgchild_atexit(void)
223 : {
224 90 : if (bgchild > 0)
225 90 : kill(bgchild, SIGTERM);
226 90 : }
227 : #endif
228 :
229 : /*
230 : * Split argument into old_dir and new_dir and append to tablespace mapping
231 : * list.
232 : */
233 : static void
234 16 : tablespace_list_append(const char *arg)
235 : {
236 16 : TablespaceListCell *cell = (TablespaceListCell *) pg_malloc0(sizeof(TablespaceListCell));
237 : char *dst;
238 : char *dst_ptr;
239 : const char *arg_ptr;
240 :
241 16 : dst_ptr = dst = cell->old_dir;
242 580 : for (arg_ptr = arg; *arg_ptr; arg_ptr++)
243 : {
244 566 : if (dst_ptr - dst >= MAXPGPATH)
245 : {
246 0 : pg_log_error("directory name too long");
247 0 : exit(1);
248 : }
249 :
250 566 : if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=')
251 : ; /* skip backslash escaping = */
252 562 : else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\'))
253 : {
254 30 : if (*cell->new_dir)
255 : {
256 2 : pg_log_error("multiple \"=\" signs in tablespace mapping");
257 2 : exit(1);
258 : }
259 : else
260 14 : dst = dst_ptr = cell->new_dir;
261 : }
262 : else
263 546 : *dst_ptr++ = *arg_ptr;
264 : }
265 :
266 14 : if (!*cell->old_dir || !*cell->new_dir)
267 : {
268 6 : pg_log_error("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
269 6 : exit(1);
270 : }
271 :
272 : /*
273 : * This check isn't absolutely necessary. But all tablespaces are created
274 : * with absolute directories, so specifying a non-absolute path here would
275 : * just never match, possibly confusing users. It's also good to be
276 : * consistent with the new_dir check.
277 : */
278 8 : if (!is_absolute_path(cell->old_dir))
279 : {
280 2 : pg_log_error("old directory is not an absolute path in tablespace mapping: %s",
281 : cell->old_dir);
282 2 : exit(1);
283 : }
284 :
285 6 : if (!is_absolute_path(cell->new_dir))
286 : {
287 2 : pg_log_error("new directory is not an absolute path in tablespace mapping: %s",
288 : cell->new_dir);
289 2 : exit(1);
290 : }
291 :
292 : /*
293 : * Comparisons done with these values should involve similarly
294 : * canonicalized path values. This is particularly sensitive on Windows
295 : * where path values may not necessarily use Unix slashes.
296 : */
297 4 : canonicalize_path(cell->old_dir);
298 4 : canonicalize_path(cell->new_dir);
299 :
300 4 : if (tablespace_dirs.tail)
301 0 : tablespace_dirs.tail->next = cell;
302 : else
303 4 : tablespace_dirs.head = cell;
304 4 : tablespace_dirs.tail = cell;
305 4 : }
306 :
307 :
308 : #ifdef HAVE_LIBZ
309 : static const char *
310 0 : get_gz_error(gzFile gzf)
311 : {
312 : int errnum;
313 : const char *errmsg;
314 :
315 0 : errmsg = gzerror(gzf, &errnum);
316 0 : if (errnum == Z_ERRNO)
317 0 : return strerror(errno);
318 : else
319 0 : return errmsg;
320 : }
321 : #endif
322 :
323 : static void
324 2 : usage(void)
325 : {
326 2 : printf(_("%s takes a base backup of a running PostgreSQL server.\n\n"),
327 : progname);
328 2 : printf(_("Usage:\n"));
329 2 : printf(_(" %s [OPTION]...\n"), progname);
330 2 : printf(_("\nOptions controlling the output:\n"));
331 2 : printf(_(" -D, --pgdata=DIRECTORY receive base backup into directory\n"));
332 2 : printf(_(" -F, --format=p|t output format (plain (default), tar)\n"));
333 2 : printf(_(" -r, --max-rate=RATE maximum transfer rate to transfer data directory\n"
334 : " (in kB/s, or use suffix \"k\" or \"M\")\n"));
335 2 : printf(_(" -R, --write-recovery-conf\n"
336 : " write configuration for replication\n"));
337 2 : printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
338 : " relocate tablespace in OLDDIR to NEWDIR\n"));
339 2 : printf(_(" --waldir=WALDIR location for the write-ahead log directory\n"));
340 2 : printf(_(" -X, --wal-method=none|fetch|stream\n"
341 : " include required WAL files with specified method\n"));
342 2 : printf(_(" -z, --gzip compress tar output\n"));
343 2 : printf(_(" -Z, --compress=0-9 compress tar output with given compression level\n"));
344 2 : printf(_("\nGeneral options:\n"));
345 2 : printf(_(" -c, --checkpoint=fast|spread\n"
346 : " set fast or spread checkpointing\n"));
347 2 : printf(_(" -C, --create-slot create replication slot\n"));
348 2 : printf(_(" -l, --label=LABEL set backup label\n"));
349 2 : printf(_(" -n, --no-clean do not clean up after errors\n"));
350 2 : printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
351 2 : printf(_(" -P, --progress show progress information\n"));
352 2 : printf(_(" -S, --slot=SLOTNAME replication slot to use\n"));
353 2 : printf(_(" -v, --verbose output verbose messages\n"));
354 2 : printf(_(" -V, --version output version information, then exit\n"));
355 2 : printf(_(" --no-slot prevent creation of temporary replication slot\n"));
356 2 : printf(_(" --no-verify-checksums\n"
357 : " do not verify checksums\n"));
358 2 : printf(_(" -?, --help show this help, then exit\n"));
359 2 : printf(_("\nConnection options:\n"));
360 2 : printf(_(" -d, --dbname=CONNSTR connection string\n"));
361 2 : printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
362 2 : printf(_(" -p, --port=PORT database server port number\n"));
363 2 : printf(_(" -s, --status-interval=INTERVAL\n"
364 : " time between status packets sent to server (in seconds)\n"));
365 2 : printf(_(" -U, --username=NAME connect as specified database user\n"));
366 2 : printf(_(" -w, --no-password never prompt for password\n"));
367 2 : printf(_(" -W, --password force password prompt (should happen automatically)\n"));
368 2 : printf(_("\nReport bugs to <pgsql-bugs@lists.postgresql.org>.\n"));
369 2 : }
370 :
371 :
372 : /*
373 : * Called in the background process every time data is received.
374 : * On Unix, we check to see if there is any data on our pipe
375 : * (which would mean we have a stop position), and if it is, check if
376 : * it is time to stop.
377 : * On Windows, we are in a single process, so we can just check if it's
378 : * time to stop.
379 : */
380 : static bool
381 4442 : reached_end_position(XLogRecPtr segendpos, uint32 timeline,
382 : bool segment_finished)
383 : {
384 4442 : if (!has_xlogendptr)
385 : {
386 : #ifndef WIN32
387 : fd_set fds;
388 : struct timeval tv;
389 : int r;
390 :
391 : /*
392 : * Don't have the end pointer yet - check our pipe to see if it has
393 : * been sent yet.
394 : */
395 4316 : FD_ZERO(&fds);
396 4316 : FD_SET(bgpipe[0], &fds);
397 :
398 4316 : MemSet(&tv, 0, sizeof(tv));
399 :
400 4316 : r = select(bgpipe[0] + 1, &fds, NULL, NULL, &tv);
401 4316 : if (r == 1)
402 : {
403 : char xlogend[64];
404 : uint32 hi,
405 : lo;
406 :
407 80 : MemSet(xlogend, 0, sizeof(xlogend));
408 80 : r = read(bgpipe[0], xlogend, sizeof(xlogend) - 1);
409 80 : if (r < 0)
410 : {
411 0 : pg_log_error("could not read from ready pipe: %m");
412 0 : exit(1);
413 : }
414 :
415 80 : if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
416 : {
417 0 : pg_log_error("could not parse write-ahead log location \"%s\"",
418 : xlogend);
419 0 : exit(1);
420 : }
421 80 : xlogendptr = ((uint64) hi) << 32 | lo;
422 80 : has_xlogendptr = 1;
423 :
424 : /*
425 : * Fall through to check if we've reached the point further
426 : * already.
427 : */
428 : }
429 : else
430 : {
431 : /*
432 : * No data received on the pipe means we don't know the end
433 : * position yet - so just say it's not time to stop yet.
434 : */
435 4236 : return false;
436 : }
437 : #else
438 :
439 : /*
440 : * On win32, has_xlogendptr is set by the main thread, so if it's not
441 : * set here, we just go back and wait until it shows up.
442 : */
443 : return false;
444 : #endif
445 : }
446 :
447 : /*
448 : * At this point we have an end pointer, so compare it to the current
449 : * position to figure out if it's time to stop.
450 : */
451 206 : if (segendpos >= xlogendptr)
452 160 : return true;
453 :
454 : /*
455 : * Have end pointer, but haven't reached it yet - so tell the caller to
456 : * keep streaming.
457 : */
458 46 : return false;
459 : }
460 :
461 : typedef struct
462 : {
463 : PGconn *bgconn;
464 : XLogRecPtr startptr;
465 : char xlog[MAXPGPATH]; /* directory or tarfile depending on mode */
466 : char *sysidentifier;
467 : int timeline;
468 : } logstreamer_param;
469 :
470 : static int
471 82 : LogStreamerMain(logstreamer_param *param)
472 : {
473 : StreamCtl stream;
474 :
475 82 : in_log_streamer = true;
476 :
477 82 : MemSet(&stream, 0, sizeof(stream));
478 82 : stream.startpos = param->startptr;
479 82 : stream.timeline = param->timeline;
480 82 : stream.sysidentifier = param->sysidentifier;
481 82 : stream.stream_stop = reached_end_position;
482 : #ifndef WIN32
483 82 : stream.stop_socket = bgpipe[0];
484 : #else
485 : stream.stop_socket = PGINVALID_SOCKET;
486 : #endif
487 82 : stream.standby_message_timeout = standby_message_timeout;
488 82 : stream.synchronous = false;
489 82 : stream.do_sync = do_sync;
490 82 : stream.mark_done = true;
491 82 : stream.partial_suffix = NULL;
492 82 : stream.replication_slot = replication_slot;
493 :
494 82 : if (format == 'p')
495 74 : stream.walmethod = CreateWalDirectoryMethod(param->xlog, 0, do_sync);
496 : else
497 8 : stream.walmethod = CreateWalTarMethod(param->xlog, compresslevel, do_sync);
498 :
499 82 : if (!ReceiveXlogStream(param->bgconn, &stream))
500 :
501 : /*
502 : * Any errors will already have been reported in the function process,
503 : * but we need to tell the parent that we didn't shutdown in a nice
504 : * way.
505 : */
506 2 : return 1;
507 :
508 80 : if (!stream.walmethod->finish())
509 : {
510 0 : pg_log_error("could not finish writing WAL files: %m");
511 0 : return 1;
512 : }
513 :
514 80 : PQfinish(param->bgconn);
515 :
516 80 : if (format == 'p')
517 72 : FreeWalDirectoryMethod();
518 : else
519 8 : FreeWalTarMethod();
520 80 : pg_free(stream.walmethod);
521 :
522 80 : return 0;
523 : }
524 :
525 : /*
526 : * Initiate background process for receiving xlog during the backup.
527 : * The background stream will use its own database connection so we can
528 : * stream the logfile in parallel with the backups.
529 : */
530 : static void
531 92 : StartLogStreamer(char *startpos, uint32 timeline, char *sysidentifier)
532 : {
533 : logstreamer_param *param;
534 : uint32 hi,
535 : lo;
536 : char statusdir[MAXPGPATH];
537 :
538 92 : param = pg_malloc0(sizeof(logstreamer_param));
539 92 : param->timeline = timeline;
540 92 : param->sysidentifier = sysidentifier;
541 :
542 : /* Convert the starting position */
543 92 : if (sscanf(startpos, "%X/%X", &hi, &lo) != 2)
544 : {
545 0 : pg_log_error("could not parse write-ahead log location \"%s\"",
546 : startpos);
547 0 : exit(1);
548 : }
549 92 : param->startptr = ((uint64) hi) << 32 | lo;
550 : /* Round off to even segment position */
551 92 : param->startptr -= XLogSegmentOffset(param->startptr, WalSegSz);
552 :
553 : #ifndef WIN32
554 : /* Create our background pipe */
555 92 : if (pipe(bgpipe) < 0)
556 : {
557 0 : pg_log_error("could not create pipe for background process: %m");
558 0 : exit(1);
559 : }
560 : #endif
561 :
562 : /* Get a second connection */
563 92 : param->bgconn = GetConnection();
564 92 : if (!param->bgconn)
565 : /* Error message already written in GetConnection() */
566 0 : exit(1);
567 :
568 : /* In post-10 cluster, pg_xlog has been renamed to pg_wal */
569 92 : snprintf(param->xlog, sizeof(param->xlog), "%s/%s",
570 : basedir,
571 92 : PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
572 : "pg_xlog" : "pg_wal");
573 :
574 : /* Temporary replication slots are only supported in 10 and newer */
575 92 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_TEMP_SLOTS)
576 0 : temp_replication_slot = false;
577 :
578 : /*
579 : * Create replication slot if requested
580 : */
581 92 : if (temp_replication_slot && !replication_slot)
582 80 : replication_slot = psprintf("pg_basebackup_%d", (int) PQbackendPID(param->bgconn));
583 92 : if (temp_replication_slot || create_slot)
584 : {
585 84 : if (!CreateReplicationSlot(param->bgconn, replication_slot, NULL,
586 : temp_replication_slot, true, true, false))
587 2 : exit(1);
588 :
589 82 : if (verbose)
590 : {
591 0 : if (temp_replication_slot)
592 0 : pg_log_info("created temporary replication slot \"%s\"",
593 : replication_slot);
594 : else
595 0 : pg_log_info("created replication slot \"%s\"",
596 : replication_slot);
597 : }
598 : }
599 :
600 90 : if (format == 'p')
601 : {
602 : /*
603 : * Create pg_wal/archive_status or pg_xlog/archive_status (and thus
604 : * pg_wal or pg_xlog) depending on the target server so we can write
605 : * to basedir/pg_wal or basedir/pg_xlog as the directory entry in the
606 : * tar file may arrive later.
607 : */
608 80 : snprintf(statusdir, sizeof(statusdir), "%s/%s/archive_status",
609 : basedir,
610 80 : PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
611 : "pg_xlog" : "pg_wal");
612 :
613 80 : if (pg_mkdir_p(statusdir, pg_dir_create_mode) != 0 && errno != EEXIST)
614 : {
615 0 : pg_log_error("could not create directory \"%s\": %m", statusdir);
616 0 : exit(1);
617 : }
618 : }
619 :
620 : /*
621 : * Start a child process and tell it to start streaming. On Unix, this is
622 : * a fork(). On Windows, we create a thread.
623 : */
624 : #ifndef WIN32
625 90 : bgchild = fork();
626 172 : if (bgchild == 0)
627 : {
628 : /* in child process */
629 82 : exit(LogStreamerMain(param));
630 : }
631 90 : else if (bgchild < 0)
632 : {
633 0 : pg_log_error("could not create background process: %m");
634 0 : exit(1);
635 : }
636 :
637 : /*
638 : * Else we are in the parent process and all is well.
639 : */
640 90 : atexit(kill_bgchild_atexit);
641 : #else /* WIN32 */
642 : bgchild = _beginthreadex(NULL, 0, (void *) LogStreamerMain, param, 0, NULL);
643 : if (bgchild == 0)
644 : {
645 : pg_log_error("could not create background thread: %m");
646 : exit(1);
647 : }
648 : #endif
649 90 : }
650 :
651 : /*
652 : * Verify that the given directory exists and is empty. If it does not
653 : * exist, it is created. If it exists but is not empty, an error will
654 : * be given and the process ended.
655 : */
656 : static void
657 106 : verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found)
658 : {
659 106 : switch (pg_check_dir(dirname))
660 : {
661 : case 0:
662 :
663 : /*
664 : * Does not exist, so create
665 : */
666 104 : if (pg_mkdir_p(dirname, pg_dir_create_mode) == -1)
667 : {
668 0 : pg_log_error("could not create directory \"%s\": %m", dirname);
669 0 : exit(1);
670 : }
671 104 : if (created)
672 104 : *created = true;
673 104 : return;
674 : case 1:
675 :
676 : /*
677 : * Exists, empty
678 : */
679 0 : if (found)
680 0 : *found = true;
681 0 : return;
682 : case 2:
683 : case 3:
684 : case 4:
685 :
686 : /*
687 : * Exists, not empty
688 : */
689 2 : pg_log_error("directory \"%s\" exists but is not empty", dirname);
690 2 : exit(1);
691 : case -1:
692 :
693 : /*
694 : * Access problem
695 : */
696 0 : pg_log_error("could not access directory \"%s\": %m", dirname);
697 0 : exit(1);
698 : }
699 : }
700 :
701 :
702 : /*
703 : * Print a progress report based on the global variables. If verbose output
704 : * is enabled, also print the current file name.
705 : *
706 : * Progress report is written at maximum once per second, unless the
707 : * force parameter is set to true.
708 : */
709 : static void
710 120392 : progress_report(int tablespacenum, const char *filename, bool force)
711 : {
712 : int percent;
713 : char totaldone_str[32];
714 : char totalsize_str[32];
715 : pg_time_t now;
716 :
717 120392 : if (!showprogress)
718 240784 : return;
719 :
720 0 : now = time(NULL);
721 0 : if (now == last_progress_report && !force)
722 0 : return; /* Max once per second */
723 :
724 0 : last_progress_report = now;
725 0 : percent = totalsize ? (int) ((totaldone / 1024) * 100 / totalsize) : 0;
726 :
727 : /*
728 : * Avoid overflowing past 100% or the full size. This may make the total
729 : * size number change as we approach the end of the backup (the estimate
730 : * will always be wrong if WAL is included), but that's better than having
731 : * the done column be bigger than the total.
732 : */
733 0 : if (percent > 100)
734 0 : percent = 100;
735 0 : if (totaldone / 1024 > totalsize)
736 0 : totalsize = totaldone / 1024;
737 :
738 : /*
739 : * Separate step to keep platform-dependent format code out of
740 : * translatable strings. And we only test for INT64_FORMAT availability
741 : * in snprintf, not fprintf.
742 : */
743 0 : snprintf(totaldone_str, sizeof(totaldone_str), INT64_FORMAT,
744 : totaldone / 1024);
745 0 : snprintf(totalsize_str, sizeof(totalsize_str), INT64_FORMAT, totalsize);
746 :
747 : #define VERBOSE_FILENAME_LENGTH 35
748 0 : if (verbose)
749 : {
750 0 : if (!filename)
751 :
752 : /*
753 : * No filename given, so clear the status line (used for last
754 : * call)
755 : */
756 0 : fprintf(stderr,
757 0 : ngettext("%*s/%s kB (100%%), %d/%d tablespace %*s",
758 : "%*s/%s kB (100%%), %d/%d tablespaces %*s",
759 : tablespacecount),
760 0 : (int) strlen(totalsize_str),
761 : totaldone_str, totalsize_str,
762 : tablespacenum, tablespacecount,
763 : VERBOSE_FILENAME_LENGTH + 5, "");
764 : else
765 : {
766 0 : bool truncate = (strlen(filename) > VERBOSE_FILENAME_LENGTH);
767 :
768 0 : fprintf(stderr,
769 0 : ngettext("%*s/%s kB (%d%%), %d/%d tablespace (%s%-*.*s)",
770 : "%*s/%s kB (%d%%), %d/%d tablespaces (%s%-*.*s)",
771 : tablespacecount),
772 0 : (int) strlen(totalsize_str),
773 : totaldone_str, totalsize_str, percent,
774 : tablespacenum, tablespacecount,
775 : /* Prefix with "..." if we do leading truncation */
776 : truncate ? "..." : "",
777 : truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
778 : truncate ? VERBOSE_FILENAME_LENGTH - 3 : VERBOSE_FILENAME_LENGTH,
779 : /* Truncate filename at beginning if it's too long */
780 0 : truncate ? filename + strlen(filename) - VERBOSE_FILENAME_LENGTH + 3 : filename);
781 : }
782 : }
783 : else
784 0 : fprintf(stderr,
785 0 : ngettext("%*s/%s kB (%d%%), %d/%d tablespace",
786 : "%*s/%s kB (%d%%), %d/%d tablespaces",
787 : tablespacecount),
788 0 : (int) strlen(totalsize_str),
789 : totaldone_str, totalsize_str, percent,
790 : tablespacenum, tablespacecount);
791 :
792 0 : if (isatty(fileno(stderr)))
793 0 : fprintf(stderr, "\r");
794 : else
795 0 : fprintf(stderr, "\n");
796 : }
797 :
798 : static int32
799 0 : parse_max_rate(char *src)
800 : {
801 : double result;
802 : char *after_num;
803 0 : char *suffix = NULL;
804 :
805 0 : errno = 0;
806 0 : result = strtod(src, &after_num);
807 0 : if (src == after_num)
808 : {
809 0 : pg_log_error("transfer rate \"%s\" is not a valid value", src);
810 0 : exit(1);
811 : }
812 0 : if (errno != 0)
813 : {
814 0 : pg_log_error("invalid transfer rate \"%s\": %m", src);
815 0 : exit(1);
816 : }
817 :
818 0 : if (result <= 0)
819 : {
820 : /*
821 : * Reject obviously wrong values here.
822 : */
823 0 : pg_log_error("transfer rate must be greater than zero");
824 0 : exit(1);
825 : }
826 :
827 : /*
828 : * Evaluate suffix, after skipping over possible whitespace. Lack of
829 : * suffix means kilobytes.
830 : */
831 0 : while (*after_num != '\0' && isspace((unsigned char) *after_num))
832 0 : after_num++;
833 :
834 0 : if (*after_num != '\0')
835 : {
836 0 : suffix = after_num;
837 0 : if (*after_num == 'k')
838 : {
839 : /* kilobyte is the expected unit. */
840 0 : after_num++;
841 : }
842 0 : else if (*after_num == 'M')
843 : {
844 0 : after_num++;
845 0 : result *= 1024.0;
846 : }
847 : }
848 :
849 : /* The rest can only consist of white space. */
850 0 : while (*after_num != '\0' && isspace((unsigned char) *after_num))
851 0 : after_num++;
852 :
853 0 : if (*after_num != '\0')
854 : {
855 0 : pg_log_error("invalid --max-rate unit: \"%s\"", suffix);
856 0 : exit(1);
857 : }
858 :
859 : /* Valid integer? */
860 0 : if ((uint64) result != (uint64) ((uint32) result))
861 : {
862 0 : pg_log_error("transfer rate \"%s\" exceeds integer range", src);
863 0 : exit(1);
864 : }
865 :
866 : /*
867 : * The range is checked on the server side too, but avoid the server
868 : * connection if a nonsensical value was passed.
869 : */
870 0 : if (result < MAX_RATE_LOWER || result > MAX_RATE_UPPER)
871 : {
872 0 : pg_log_error("transfer rate \"%s\" is out of range", src);
873 0 : exit(1);
874 : }
875 :
876 0 : return (int32) result;
877 : }
878 :
879 : /*
880 : * Write a piece of tar data
881 : */
882 : static void
883 17758 : writeTarData(
884 : #ifdef HAVE_LIBZ
885 : gzFile ztarfile,
886 : #endif
887 : FILE *tarfile, char *buf, int r, char *current_file)
888 : {
889 : #ifdef HAVE_LIBZ
890 17758 : if (ztarfile != NULL)
891 : {
892 0 : if (gzwrite(ztarfile, buf, r) != r)
893 : {
894 0 : pg_log_error("could not write to compressed file \"%s\": %s",
895 : current_file, get_gz_error(ztarfile));
896 0 : exit(1);
897 : }
898 : }
899 : else
900 : #endif
901 : {
902 17758 : if (fwrite(buf, r, 1, tarfile) != 1)
903 : {
904 0 : pg_log_error("could not write to file \"%s\": %m", current_file);
905 0 : exit(1);
906 : }
907 : }
908 17758 : }
909 :
910 : #ifdef HAVE_LIBZ
911 : #define WRITE_TAR_DATA(buf, sz) writeTarData(ztarfile, tarfile, buf, sz, filename)
912 : #else
913 : #define WRITE_TAR_DATA(buf, sz) writeTarData(tarfile, buf, sz, filename)
914 : #endif
915 :
916 : /*
917 : * Receive a tar format file from the connection to the server, and write
918 : * the data from this file directly into a tar file. If compression is
919 : * enabled, the data will be compressed while written to the file.
920 : *
921 : * The file will be named base.tar[.gz] if it's for the main data directory
922 : * or <tablespaceoid>.tar[.gz] if it's for another tablespace.
923 : *
924 : * No attempt to inspect or validate the contents of the file is done.
925 : */
926 : static void
927 14 : ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
928 : {
929 : char filename[MAXPGPATH];
930 14 : char *copybuf = NULL;
931 14 : FILE *tarfile = NULL;
932 : char tarhdr[512];
933 14 : bool basetablespace = PQgetisnull(res, rownum, 0);
934 14 : bool in_tarhdr = true;
935 14 : bool skip_file = false;
936 14 : bool is_recovery_guc_supported = true;
937 14 : bool is_postgresql_auto_conf = false;
938 14 : bool found_postgresql_auto_conf = false;
939 14 : int file_padding_len = 0;
940 14 : size_t tarhdrsz = 0;
941 14 : pgoff_t filesz = 0;
942 :
943 : #ifdef HAVE_LIBZ
944 14 : gzFile ztarfile = NULL;
945 : #endif
946 :
947 : /* recovery.conf is integrated into postgresql.conf in 12 and newer */
948 14 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
949 0 : is_recovery_guc_supported = false;
950 :
951 14 : if (basetablespace)
952 : {
953 : /*
954 : * Base tablespaces
955 : */
956 10 : if (strcmp(basedir, "-") == 0)
957 : {
958 : #ifdef WIN32
959 : _setmode(fileno(stdout), _O_BINARY);
960 : #endif
961 :
962 : #ifdef HAVE_LIBZ
963 0 : if (compresslevel != 0)
964 : {
965 0 : ztarfile = gzdopen(dup(fileno(stdout)), "wb");
966 0 : if (gzsetparams(ztarfile, compresslevel,
967 : Z_DEFAULT_STRATEGY) != Z_OK)
968 : {
969 0 : pg_log_error("could not set compression level %d: %s",
970 : compresslevel, get_gz_error(ztarfile));
971 0 : exit(1);
972 : }
973 : }
974 : else
975 : #endif
976 0 : tarfile = stdout;
977 0 : strcpy(filename, "-");
978 : }
979 : else
980 : {
981 : #ifdef HAVE_LIBZ
982 10 : if (compresslevel != 0)
983 : {
984 0 : snprintf(filename, sizeof(filename), "%s/base.tar.gz", basedir);
985 0 : ztarfile = gzopen(filename, "wb");
986 0 : if (gzsetparams(ztarfile, compresslevel,
987 : Z_DEFAULT_STRATEGY) != Z_OK)
988 : {
989 0 : pg_log_error("could not set compression level %d: %s",
990 : compresslevel, get_gz_error(ztarfile));
991 0 : exit(1);
992 : }
993 : }
994 : else
995 : #endif
996 : {
997 10 : snprintf(filename, sizeof(filename), "%s/base.tar", basedir);
998 10 : tarfile = fopen(filename, "wb");
999 : }
1000 : }
1001 : }
1002 : else
1003 : {
1004 : /*
1005 : * Specific tablespace
1006 : */
1007 : #ifdef HAVE_LIBZ
1008 4 : if (compresslevel != 0)
1009 : {
1010 0 : snprintf(filename, sizeof(filename), "%s/%s.tar.gz", basedir,
1011 : PQgetvalue(res, rownum, 0));
1012 0 : ztarfile = gzopen(filename, "wb");
1013 0 : if (gzsetparams(ztarfile, compresslevel,
1014 : Z_DEFAULT_STRATEGY) != Z_OK)
1015 : {
1016 0 : pg_log_error("could not set compression level %d: %s",
1017 : compresslevel, get_gz_error(ztarfile));
1018 0 : exit(1);
1019 : }
1020 : }
1021 : else
1022 : #endif
1023 : {
1024 4 : snprintf(filename, sizeof(filename), "%s/%s.tar", basedir,
1025 : PQgetvalue(res, rownum, 0));
1026 4 : tarfile = fopen(filename, "wb");
1027 : }
1028 : }
1029 :
1030 : #ifdef HAVE_LIBZ
1031 14 : if (compresslevel != 0)
1032 : {
1033 0 : if (!ztarfile)
1034 : {
1035 : /* Compression is in use */
1036 0 : pg_log_error("could not create compressed file \"%s\": %s",
1037 : filename, get_gz_error(ztarfile));
1038 0 : exit(1);
1039 : }
1040 : }
1041 : else
1042 : #endif
1043 : {
1044 : /* Either no zlib support, or zlib support but compresslevel = 0 */
1045 14 : if (!tarfile)
1046 : {
1047 0 : pg_log_error("could not create file \"%s\": %m", filename);
1048 0 : exit(1);
1049 : }
1050 : }
1051 :
1052 : /*
1053 : * Get the COPY data stream
1054 : */
1055 14 : res = PQgetResult(conn);
1056 14 : if (PQresultStatus(res) != PGRES_COPY_OUT)
1057 : {
1058 0 : pg_log_error("could not get COPY data stream: %s",
1059 : PQerrorMessage(conn));
1060 0 : exit(1);
1061 : }
1062 :
1063 : while (1)
1064 17744 : {
1065 : int r;
1066 :
1067 17758 : if (copybuf != NULL)
1068 : {
1069 17744 : PQfreemem(copybuf);
1070 17744 : copybuf = NULL;
1071 : }
1072 :
1073 17758 : r = PQgetCopyData(conn, ©buf, 0);
1074 17758 : if (r == -1)
1075 : {
1076 : /*
1077 : * End of chunk. If requested, and this is the base tablespace,
1078 : * write configuration file into the tarfile. When done, close the
1079 : * file (but not stdout).
1080 : *
1081 : * Also, write two completely empty blocks at the end of the tar
1082 : * file, as required by some tar programs.
1083 : */
1084 : char zerobuf[1024];
1085 :
1086 14 : MemSet(zerobuf, 0, sizeof(zerobuf));
1087 :
1088 14 : if (basetablespace && writerecoveryconf)
1089 : {
1090 : char header[512];
1091 :
1092 : /*
1093 : * If postgresql.auto.conf has not been found in the streamed
1094 : * data, add recovery configuration to postgresql.auto.conf if
1095 : * recovery parameters are GUCs. If the instance connected to
1096 : * is older than 12, create recovery.conf with this data
1097 : * otherwise.
1098 : */
1099 0 : if (!found_postgresql_auto_conf || !is_recovery_guc_supported)
1100 : {
1101 : int padding;
1102 :
1103 0 : tarCreateHeader(header,
1104 : is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf",
1105 : NULL,
1106 0 : recoveryconfcontents->len,
1107 : pg_file_create_mode, 04000, 02000,
1108 : time(NULL));
1109 :
1110 0 : padding = ((recoveryconfcontents->len + 511) & ~511) - recoveryconfcontents->len;
1111 :
1112 0 : WRITE_TAR_DATA(header, sizeof(header));
1113 0 : WRITE_TAR_DATA(recoveryconfcontents->data,
1114 : recoveryconfcontents->len);
1115 0 : if (padding)
1116 0 : WRITE_TAR_DATA(zerobuf, padding);
1117 : }
1118 :
1119 : /*
1120 : * standby.signal is supported only if recovery parameters are
1121 : * GUCs.
1122 : */
1123 0 : if (is_recovery_guc_supported)
1124 : {
1125 0 : tarCreateHeader(header, "standby.signal", NULL,
1126 : 0, /* zero-length file */
1127 : pg_file_create_mode, 04000, 02000,
1128 : time(NULL));
1129 :
1130 0 : WRITE_TAR_DATA(header, sizeof(header));
1131 0 : WRITE_TAR_DATA(zerobuf, 511);
1132 : }
1133 : }
1134 :
1135 : /* 2 * 512 bytes empty data at end of file */
1136 14 : WRITE_TAR_DATA(zerobuf, sizeof(zerobuf));
1137 :
1138 : #ifdef HAVE_LIBZ
1139 14 : if (ztarfile != NULL)
1140 : {
1141 0 : if (gzclose(ztarfile) != 0)
1142 : {
1143 0 : pg_log_error("could not close compressed file \"%s\": %s",
1144 : filename, get_gz_error(ztarfile));
1145 0 : exit(1);
1146 : }
1147 : }
1148 : else
1149 : #endif
1150 : {
1151 14 : if (strcmp(basedir, "-") != 0)
1152 : {
1153 14 : if (fclose(tarfile) != 0)
1154 : {
1155 0 : pg_log_error("could not close file \"%s\": %m",
1156 : filename);
1157 0 : exit(1);
1158 : }
1159 : }
1160 : }
1161 :
1162 14 : break;
1163 : }
1164 17744 : else if (r == -2)
1165 : {
1166 0 : pg_log_error("could not read COPY data: %s",
1167 : PQerrorMessage(conn));
1168 0 : exit(1);
1169 : }
1170 :
1171 17744 : if (!writerecoveryconf || !basetablespace)
1172 : {
1173 : /*
1174 : * When not writing config file, or when not working on the base
1175 : * tablespace, we never have to look for an existing configuration
1176 : * file in the stream.
1177 : */
1178 17744 : WRITE_TAR_DATA(copybuf, r);
1179 : }
1180 : else
1181 : {
1182 : /*
1183 : * Look for a config file in the existing tar stream. If it's
1184 : * there, we must skip it so we can later overwrite it with our
1185 : * own version of the file.
1186 : *
1187 : * To do this, we have to process the individual files inside the
1188 : * TAR stream. The stream consists of a header and zero or more
1189 : * chunks, all 512 bytes long. The stream from the server is
1190 : * broken up into smaller pieces, so we have to track the size of
1191 : * the files to find the next header structure.
1192 : */
1193 0 : int rr = r;
1194 0 : int pos = 0;
1195 :
1196 0 : while (rr > 0)
1197 : {
1198 0 : if (in_tarhdr)
1199 : {
1200 : /*
1201 : * We're currently reading a header structure inside the
1202 : * TAR stream, i.e. the file metadata.
1203 : */
1204 0 : if (tarhdrsz < 512)
1205 : {
1206 : /*
1207 : * Copy the header structure into tarhdr in case the
1208 : * header is not aligned to 512 bytes or it's not
1209 : * returned in whole by the last PQgetCopyData call.
1210 : */
1211 : int hdrleft;
1212 : int bytes2copy;
1213 :
1214 0 : hdrleft = 512 - tarhdrsz;
1215 0 : bytes2copy = (rr > hdrleft ? hdrleft : rr);
1216 :
1217 0 : memcpy(&tarhdr[tarhdrsz], copybuf + pos, bytes2copy);
1218 :
1219 0 : rr -= bytes2copy;
1220 0 : pos += bytes2copy;
1221 0 : tarhdrsz += bytes2copy;
1222 : }
1223 : else
1224 : {
1225 : /*
1226 : * We have the complete header structure in tarhdr,
1227 : * look at the file metadata: we may want append
1228 : * recovery info into postgresql.auto.conf and skip
1229 : * standby.signal file if recovery parameters are
1230 : * integrated as GUCs, and recovery.conf otherwise. In
1231 : * both cases we must calculate tar padding.
1232 : */
1233 0 : if (is_recovery_guc_supported)
1234 : {
1235 0 : skip_file = (strcmp(&tarhdr[0], "standby.signal") == 0);
1236 0 : is_postgresql_auto_conf = (strcmp(&tarhdr[0], "postgresql.auto.conf") == 0);
1237 : }
1238 : else
1239 0 : skip_file = (strcmp(&tarhdr[0], "recovery.conf") == 0);
1240 :
1241 0 : filesz = read_tar_number(&tarhdr[124], 12);
1242 0 : file_padding_len = ((filesz + 511) & ~511) - filesz;
1243 :
1244 0 : if (is_recovery_guc_supported &&
1245 0 : is_postgresql_auto_conf &&
1246 : writerecoveryconf)
1247 0 : {
1248 : /* replace tar header */
1249 : char header[512];
1250 :
1251 0 : tarCreateHeader(header, "postgresql.auto.conf", NULL,
1252 0 : filesz + recoveryconfcontents->len,
1253 : pg_file_create_mode, 04000, 02000,
1254 : time(NULL));
1255 :
1256 0 : WRITE_TAR_DATA(header, sizeof(header));
1257 : }
1258 : else
1259 : {
1260 : /* copy stream with padding */
1261 0 : filesz += file_padding_len;
1262 :
1263 0 : if (!skip_file)
1264 : {
1265 : /*
1266 : * If we're not skipping the file, write the
1267 : * tar header unmodified.
1268 : */
1269 0 : WRITE_TAR_DATA(tarhdr, 512);
1270 : }
1271 : }
1272 :
1273 : /* Next part is the file, not the header */
1274 0 : in_tarhdr = false;
1275 : }
1276 : }
1277 : else
1278 : {
1279 : /*
1280 : * We're processing a file's contents.
1281 : */
1282 0 : if (filesz > 0)
1283 : {
1284 : /*
1285 : * We still have data to read (and possibly write).
1286 : */
1287 : int bytes2write;
1288 :
1289 0 : bytes2write = (filesz > rr ? rr : filesz);
1290 :
1291 0 : if (!skip_file)
1292 0 : WRITE_TAR_DATA(copybuf + pos, bytes2write);
1293 :
1294 0 : rr -= bytes2write;
1295 0 : pos += bytes2write;
1296 0 : filesz -= bytes2write;
1297 : }
1298 0 : else if (is_recovery_guc_supported &&
1299 0 : is_postgresql_auto_conf &&
1300 : writerecoveryconf)
1301 0 : {
1302 : /* append recovery config to postgresql.auto.conf */
1303 : int padding;
1304 : int tailsize;
1305 :
1306 0 : tailsize = (512 - file_padding_len) + recoveryconfcontents->len;
1307 0 : padding = ((tailsize + 511) & ~511) - tailsize;
1308 :
1309 0 : WRITE_TAR_DATA(recoveryconfcontents->data, recoveryconfcontents->len);
1310 :
1311 0 : if (padding)
1312 : {
1313 : char zerobuf[512];
1314 :
1315 0 : MemSet(zerobuf, 0, sizeof(zerobuf));
1316 0 : WRITE_TAR_DATA(zerobuf, padding);
1317 : }
1318 :
1319 : /* skip original file padding */
1320 0 : is_postgresql_auto_conf = false;
1321 0 : skip_file = true;
1322 0 : filesz += file_padding_len;
1323 :
1324 0 : found_postgresql_auto_conf = true;
1325 : }
1326 : else
1327 : {
1328 : /*
1329 : * No more data in the current file, the next piece of
1330 : * data (if any) will be a new file header structure.
1331 : */
1332 0 : in_tarhdr = true;
1333 0 : skip_file = false;
1334 0 : is_postgresql_auto_conf = false;
1335 0 : tarhdrsz = 0;
1336 0 : filesz = 0;
1337 : }
1338 : }
1339 : }
1340 : }
1341 17744 : totaldone += r;
1342 17744 : progress_report(rownum, filename, false);
1343 : } /* while (1) */
1344 14 : progress_report(rownum, filename, true);
1345 :
1346 14 : if (copybuf != NULL)
1347 0 : PQfreemem(copybuf);
1348 :
1349 : /* sync the resulting tar file, errors are not considered fatal */
1350 14 : if (do_sync && strcmp(basedir, "-") != 0)
1351 14 : (void) fsync_fname(filename, false);
1352 14 : }
1353 :
1354 :
1355 : /*
1356 : * Retrieve tablespace path, either relocated or original depending on whether
1357 : * -T was passed or not.
1358 : */
1359 : static const char *
1360 14 : get_tablespace_mapping(const char *dir)
1361 : {
1362 : TablespaceListCell *cell;
1363 : char canon_dir[MAXPGPATH];
1364 :
1365 : /* Canonicalize path for comparison consistency */
1366 14 : strlcpy(canon_dir, dir, sizeof(canon_dir));
1367 14 : canonicalize_path(canon_dir);
1368 :
1369 14 : for (cell = tablespace_dirs.head; cell; cell = cell->next)
1370 12 : if (strcmp(canon_dir, cell->old_dir) == 0)
1371 12 : return cell->new_dir;
1372 :
1373 2 : return dir;
1374 : }
1375 :
1376 :
1377 : /*
1378 : * Receive a tar format stream from the connection to the server, and unpack
1379 : * the contents of it into a directory. Only files, directories and
1380 : * symlinks are supported, no other kinds of special files.
1381 : *
1382 : * If the data is for the main data directory, it will be restored in the
1383 : * specified directory. If it's for another tablespace, it will be restored
1384 : * in the original or mapped directory.
1385 : */
1386 : static void
1387 88 : ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
1388 : {
1389 : char current_path[MAXPGPATH];
1390 : char filename[MAXPGPATH];
1391 : const char *mapped_tblspc_path;
1392 88 : pgoff_t current_len_left = 0;
1393 88 : int current_padding = 0;
1394 : bool basetablespace;
1395 88 : char *copybuf = NULL;
1396 88 : FILE *file = NULL;
1397 :
1398 88 : basetablespace = PQgetisnull(res, rownum, 0);
1399 88 : if (basetablespace)
1400 84 : strlcpy(current_path, basedir, sizeof(current_path));
1401 : else
1402 4 : strlcpy(current_path,
1403 4 : get_tablespace_mapping(PQgetvalue(res, rownum, 1)),
1404 : sizeof(current_path));
1405 :
1406 : /*
1407 : * Get the COPY data
1408 : */
1409 88 : res = PQgetResult(conn);
1410 88 : if (PQresultStatus(res) != PGRES_COPY_OUT)
1411 : {
1412 0 : pg_log_error("could not get COPY data stream: %s",
1413 : PQerrorMessage(conn));
1414 0 : exit(1);
1415 : }
1416 :
1417 : while (1)
1418 189234 : {
1419 : int r;
1420 :
1421 189322 : if (copybuf != NULL)
1422 : {
1423 189234 : PQfreemem(copybuf);
1424 189234 : copybuf = NULL;
1425 : }
1426 :
1427 189322 : r = PQgetCopyData(conn, ©buf, 0);
1428 :
1429 189322 : if (r == -1)
1430 : {
1431 : /*
1432 : * End of chunk
1433 : */
1434 88 : if (file)
1435 0 : fclose(file);
1436 :
1437 88 : break;
1438 : }
1439 189234 : else if (r == -2)
1440 : {
1441 0 : pg_log_error("could not read COPY data: %s",
1442 : PQerrorMessage(conn));
1443 0 : exit(1);
1444 : }
1445 :
1446 189234 : if (file == NULL)
1447 : {
1448 : #ifndef WIN32
1449 : int filemode;
1450 : #endif
1451 :
1452 : /*
1453 : * No current file, so this must be the header for a new file
1454 : */
1455 85764 : if (r != 512)
1456 : {
1457 0 : pg_log_error("invalid tar block header size: %d", r);
1458 0 : exit(1);
1459 : }
1460 85764 : totaldone += 512;
1461 :
1462 85764 : current_len_left = read_tar_number(©buf[124], 12);
1463 :
1464 : #ifndef WIN32
1465 : /* Set permissions on the file */
1466 85764 : filemode = read_tar_number(©buf[100], 8);
1467 : #endif
1468 :
1469 : /*
1470 : * All files are padded up to 512 bytes
1471 : */
1472 85764 : current_padding =
1473 85764 : ((current_len_left + 511) & ~511) - current_len_left;
1474 :
1475 : /*
1476 : * First part of header is zero terminated filename
1477 : */
1478 85764 : snprintf(filename, sizeof(filename), "%s/%s", current_path,
1479 : copybuf);
1480 85764 : if (filename[strlen(filename) - 1] == '/')
1481 : {
1482 : /*
1483 : * Ends in a slash means directory or symlink to directory
1484 : */
1485 2122 : if (copybuf[156] == '5')
1486 : {
1487 : /*
1488 : * Directory
1489 : */
1490 2118 : filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1491 2118 : if (mkdir(filename, pg_dir_create_mode) != 0)
1492 : {
1493 : /*
1494 : * When streaming WAL, pg_wal (or pg_xlog for pre-9.6
1495 : * clusters) will have been created by the wal
1496 : * receiver process. Also, when the WAL directory
1497 : * location was specified, pg_wal (or pg_xlog) has
1498 : * already been created as a symbolic link before
1499 : * starting the actual backup. So just ignore creation
1500 : * failures on related directories.
1501 : */
1502 400 : if (!((pg_str_endswith(filename, "/pg_wal") ||
1503 160 : pg_str_endswith(filename, "/pg_xlog") ||
1504 80 : pg_str_endswith(filename, "/archive_status")) &&
1505 160 : errno == EEXIST))
1506 : {
1507 0 : pg_log_error("could not create directory \"%s\": %m",
1508 : filename);
1509 0 : exit(1);
1510 : }
1511 : }
1512 : #ifndef WIN32
1513 2118 : if (chmod(filename, (mode_t) filemode))
1514 0 : pg_log_error("could not set permissions on directory \"%s\": %m",
1515 : filename);
1516 : #endif
1517 : }
1518 4 : else if (copybuf[156] == '2')
1519 : {
1520 : /*
1521 : * Symbolic link
1522 : *
1523 : * It's most likely a link in pg_tblspc directory, to the
1524 : * location of a tablespace. Apply any tablespace mapping
1525 : * given on the command line (--tablespace-mapping). (We
1526 : * blindly apply the mapping without checking that the
1527 : * link really is inside pg_tblspc. We don't expect there
1528 : * to be other symlinks in a data directory, but if there
1529 : * are, you can call it an undocumented feature that you
1530 : * can map them too.)
1531 : */
1532 4 : filename[strlen(filename) - 1] = '\0'; /* Remove trailing slash */
1533 :
1534 4 : mapped_tblspc_path = get_tablespace_mapping(©buf[157]);
1535 4 : if (symlink(mapped_tblspc_path, filename) != 0)
1536 : {
1537 0 : pg_log_error("could not create symbolic link from \"%s\" to \"%s\": %m",
1538 : filename, mapped_tblspc_path);
1539 0 : exit(1);
1540 : }
1541 : }
1542 : else
1543 : {
1544 0 : pg_log_error("unrecognized link indicator \"%c\"",
1545 : copybuf[156]);
1546 0 : exit(1);
1547 : }
1548 2122 : continue; /* directory or link handled */
1549 : }
1550 :
1551 : /*
1552 : * regular file
1553 : */
1554 83642 : file = fopen(filename, "wb");
1555 83642 : if (!file)
1556 : {
1557 0 : pg_log_error("could not create file \"%s\": %m", filename);
1558 0 : exit(1);
1559 : }
1560 :
1561 : #ifndef WIN32
1562 83642 : if (chmod(filename, (mode_t) filemode))
1563 0 : pg_log_error("could not set permissions on file \"%s\": %m",
1564 : filename);
1565 : #endif
1566 :
1567 83642 : if (current_len_left == 0)
1568 : {
1569 : /*
1570 : * Done with this file, next one will be a new tar header
1571 : */
1572 14976 : fclose(file);
1573 14976 : file = NULL;
1574 14976 : continue;
1575 : }
1576 : } /* new file */
1577 : else
1578 : {
1579 : /*
1580 : * Continuing blocks in existing file
1581 : */
1582 103470 : if (current_len_left == 0 && r == current_padding)
1583 : {
1584 : /*
1585 : * Received the padding block for this file, ignore it and
1586 : * close the file, then move on to the next tar header.
1587 : */
1588 924 : fclose(file);
1589 924 : file = NULL;
1590 924 : totaldone += r;
1591 924 : continue;
1592 : }
1593 :
1594 102546 : if (fwrite(copybuf, r, 1, file) != 1)
1595 : {
1596 0 : pg_log_error("could not write to file \"%s\": %m", filename);
1597 0 : exit(1);
1598 : }
1599 102546 : totaldone += r;
1600 102546 : progress_report(rownum, filename, false);
1601 :
1602 102546 : current_len_left -= r;
1603 102546 : if (current_len_left == 0 && current_padding == 0)
1604 : {
1605 : /*
1606 : * Received the last block, and there is no padding to be
1607 : * expected. Close the file and move on to the next tar
1608 : * header.
1609 : */
1610 67742 : fclose(file);
1611 67742 : file = NULL;
1612 67742 : continue;
1613 : }
1614 : } /* continuing data in existing file */
1615 : } /* loop over all data blocks */
1616 88 : progress_report(rownum, filename, true);
1617 :
1618 88 : if (file != NULL)
1619 : {
1620 0 : pg_log_error("COPY stream ended before last file was finished");
1621 0 : exit(1);
1622 : }
1623 :
1624 88 : if (copybuf != NULL)
1625 0 : PQfreemem(copybuf);
1626 :
1627 88 : if (basetablespace && writerecoveryconf)
1628 4 : WriteRecoveryConf();
1629 :
1630 : /*
1631 : * No data is synced here, everything is done for all tablespaces at the
1632 : * end.
1633 : */
1634 88 : }
1635 :
1636 : /*
1637 : * Escape a string so that it can be used as a value in a key-value pair
1638 : * a configuration file.
1639 : */
1640 : static char *
1641 4 : escape_quotes(const char *src)
1642 : {
1643 4 : char *result = escape_single_quotes_ascii(src);
1644 :
1645 4 : if (!result)
1646 : {
1647 0 : pg_log_error("out of memory");
1648 0 : exit(1);
1649 : }
1650 4 : return result;
1651 : }
1652 :
1653 : /*
1654 : * Create a configuration file in memory using a PQExpBuffer
1655 : */
1656 : static void
1657 4 : GenerateRecoveryConf(PGconn *conn)
1658 : {
1659 : PQconninfoOption *connOptions;
1660 : PQconninfoOption *option;
1661 : PQExpBufferData conninfo_buf;
1662 : char *escaped;
1663 :
1664 4 : recoveryconfcontents = createPQExpBuffer();
1665 4 : if (!recoveryconfcontents)
1666 : {
1667 0 : pg_log_error("out of memory");
1668 0 : exit(1);
1669 : }
1670 :
1671 : /*
1672 : * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by
1673 : * standby.signal to trigger a standby state at recovery.
1674 : */
1675 4 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
1676 0 : appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n");
1677 :
1678 4 : connOptions = PQconninfo(conn);
1679 4 : if (connOptions == NULL)
1680 : {
1681 0 : pg_log_error("out of memory");
1682 0 : exit(1);
1683 : }
1684 :
1685 4 : initPQExpBuffer(&conninfo_buf);
1686 124 : for (option = connOptions; option && option->keyword; option++)
1687 : {
1688 : /* Omit empty settings and those libpqwalreceiver overrides. */
1689 236 : if (strcmp(option->keyword, "replication") == 0 ||
1690 228 : strcmp(option->keyword, "dbname") == 0 ||
1691 220 : strcmp(option->keyword, "fallback_application_name") == 0 ||
1692 152 : (option->val == NULL) ||
1693 88 : (option->val != NULL && option->val[0] == '\0'))
1694 84 : continue;
1695 :
1696 : /* Separate key-value pairs with spaces */
1697 36 : if (conninfo_buf.len != 0)
1698 32 : appendPQExpBufferChar(&conninfo_buf, ' ');
1699 :
1700 : /*
1701 : * Write "keyword=value" pieces, the value string is escaped and/or
1702 : * quoted if necessary.
1703 : */
1704 36 : appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword);
1705 36 : appendConnStrVal(&conninfo_buf, option->val);
1706 : }
1707 :
1708 : /*
1709 : * Escape the connection string, so that it can be put in the config file.
1710 : * Note that this is different from the escaping of individual connection
1711 : * options above!
1712 : */
1713 4 : escaped = escape_quotes(conninfo_buf.data);
1714 4 : appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped);
1715 4 : free(escaped);
1716 :
1717 4 : if (replication_slot)
1718 : {
1719 : /* unescaped: ReplicationSlotValidateName allows [a-z0-9_] only */
1720 2 : appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n",
1721 : replication_slot);
1722 : }
1723 :
1724 8 : if (PQExpBufferBroken(recoveryconfcontents) ||
1725 4 : PQExpBufferDataBroken(conninfo_buf))
1726 : {
1727 0 : pg_log_error("out of memory");
1728 0 : exit(1);
1729 : }
1730 :
1731 4 : termPQExpBuffer(&conninfo_buf);
1732 :
1733 4 : PQconninfoFree(connOptions);
1734 4 : }
1735 :
1736 :
1737 : /*
1738 : * Write the configuration file into the directory specified in basedir,
1739 : * with the contents already collected in memory appended. Then write
1740 : * the signal file into the basedir. If the server does not support
1741 : * recovery parameters as GUCs, the signal file is not necessary, and
1742 : * configuration is written to recovery.conf.
1743 : */
1744 : static void
1745 4 : WriteRecoveryConf(void)
1746 : {
1747 : char filename[MAXPGPATH];
1748 : FILE *cf;
1749 4 : bool is_recovery_guc_supported = true;
1750 :
1751 4 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC)
1752 0 : is_recovery_guc_supported = false;
1753 :
1754 4 : snprintf(filename, MAXPGPATH, "%s/%s", basedir,
1755 : is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf");
1756 :
1757 4 : cf = fopen(filename, is_recovery_guc_supported ? "a" : "w");
1758 4 : if (cf == NULL)
1759 : {
1760 0 : pg_log_error("could not open file \"%s\": %m", filename);
1761 0 : exit(1);
1762 : }
1763 :
1764 4 : if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1)
1765 : {
1766 0 : pg_log_error("could not write to file \"%s\": %m", filename);
1767 0 : exit(1);
1768 : }
1769 :
1770 4 : fclose(cf);
1771 :
1772 4 : if (is_recovery_guc_supported)
1773 : {
1774 4 : snprintf(filename, MAXPGPATH, "%s/%s", basedir, "standby.signal");
1775 4 : cf = fopen(filename, "w");
1776 4 : if (cf == NULL)
1777 : {
1778 0 : pg_log_error("could not create file \"%s\": %m", filename);
1779 0 : exit(1);
1780 : }
1781 :
1782 4 : fclose(cf);
1783 : }
1784 4 : }
1785 :
1786 :
1787 : static void
1788 98 : BaseBackup(void)
1789 : {
1790 : PGresult *res;
1791 : char *sysidentifier;
1792 : TimeLineID latesttli;
1793 : TimeLineID starttli;
1794 : char *basebkp;
1795 : char escaped_label[MAXPGPATH];
1796 98 : char *maxrate_clause = NULL;
1797 : int i;
1798 : char xlogstart[64];
1799 : char xlogend[64];
1800 : int minServerMajor,
1801 : maxServerMajor;
1802 : int serverVersion,
1803 : serverMajor;
1804 :
1805 : Assert(conn != NULL);
1806 :
1807 : /*
1808 : * Check server version. BASE_BACKUP command was introduced in 9.1, so we
1809 : * can't work with servers older than 9.1.
1810 : */
1811 98 : minServerMajor = 901;
1812 98 : maxServerMajor = PG_VERSION_NUM / 100;
1813 98 : serverVersion = PQserverVersion(conn);
1814 98 : serverMajor = serverVersion / 100;
1815 98 : if (serverMajor < minServerMajor || serverMajor > maxServerMajor)
1816 : {
1817 0 : const char *serverver = PQparameterStatus(conn, "server_version");
1818 :
1819 0 : pg_log_error("incompatible server version %s",
1820 : serverver ? serverver : "'unknown'");
1821 0 : exit(1);
1822 : }
1823 :
1824 : /*
1825 : * If WAL streaming was requested, also check that the server is new
1826 : * enough for that.
1827 : */
1828 98 : if (includewal == STREAM_WAL && !CheckServerVersionForStreaming(conn))
1829 : {
1830 : /*
1831 : * Error message already written in CheckServerVersionForStreaming(),
1832 : * but add a hint about using -X none.
1833 : */
1834 0 : pg_log_info("HINT: use -X none or -X fetch to disable log streaming");
1835 0 : exit(1);
1836 : }
1837 :
1838 : /*
1839 : * Build contents of configuration file if requested
1840 : */
1841 98 : if (writerecoveryconf)
1842 4 : GenerateRecoveryConf(conn);
1843 :
1844 : /*
1845 : * Run IDENTIFY_SYSTEM so we can get the timeline
1846 : */
1847 98 : if (!RunIdentifySystem(conn, &sysidentifier, &latesttli, NULL, NULL))
1848 0 : exit(1);
1849 :
1850 : /*
1851 : * Start the actual backup
1852 : */
1853 98 : PQescapeStringConn(conn, escaped_label, label, sizeof(escaped_label), &i);
1854 :
1855 98 : if (maxrate > 0)
1856 0 : maxrate_clause = psprintf("MAX_RATE %u", maxrate);
1857 :
1858 98 : if (verbose)
1859 0 : pg_log_info("initiating base backup, waiting for checkpoint to complete");
1860 :
1861 98 : if (showprogress && !verbose)
1862 : {
1863 0 : fprintf(stderr, "waiting for checkpoint");
1864 0 : if (isatty(fileno(stderr)))
1865 0 : fprintf(stderr, "\r");
1866 : else
1867 0 : fprintf(stderr, "\n");
1868 : }
1869 :
1870 98 : basebkp =
1871 588 : psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s",
1872 : escaped_label,
1873 98 : showprogress ? "PROGRESS" : "",
1874 98 : includewal == FETCH_WAL ? "WAL" : "",
1875 98 : fastcheckpoint ? "FAST" : "",
1876 98 : includewal == NO_WAL ? "" : "NOWAIT",
1877 : maxrate_clause ? maxrate_clause : "",
1878 98 : format == 't' ? "TABLESPACE_MAP" : "",
1879 98 : verify_checksums ? "" : "NOVERIFY_CHECKSUMS");
1880 :
1881 98 : if (PQsendQuery(conn, basebkp) == 0)
1882 : {
1883 0 : pg_log_error("could not send replication command \"%s\": %s",
1884 : "BASE_BACKUP", PQerrorMessage(conn));
1885 0 : exit(1);
1886 : }
1887 :
1888 : /*
1889 : * Get the starting WAL location
1890 : */
1891 98 : res = PQgetResult(conn);
1892 98 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
1893 : {
1894 0 : pg_log_error("could not initiate base backup: %s",
1895 : PQerrorMessage(conn));
1896 0 : exit(1);
1897 : }
1898 98 : if (PQntuples(res) != 1)
1899 : {
1900 0 : pg_log_error("server returned unexpected response to BASE_BACKUP command; got %d rows and %d fields, expected %d rows and %d fields",
1901 : PQntuples(res), PQnfields(res), 1, 2);
1902 0 : exit(1);
1903 : }
1904 :
1905 98 : strlcpy(xlogstart, PQgetvalue(res, 0, 0), sizeof(xlogstart));
1906 :
1907 98 : if (verbose)
1908 0 : pg_log_info("checkpoint completed");
1909 :
1910 : /*
1911 : * 9.3 and later sends the TLI of the starting point. With older servers,
1912 : * assume it's the same as the latest timeline reported by
1913 : * IDENTIFY_SYSTEM.
1914 : */
1915 98 : if (PQnfields(res) >= 2)
1916 98 : starttli = atoi(PQgetvalue(res, 0, 1));
1917 : else
1918 0 : starttli = latesttli;
1919 98 : PQclear(res);
1920 98 : MemSet(xlogend, 0, sizeof(xlogend));
1921 :
1922 98 : if (verbose && includewal != NO_WAL)
1923 0 : pg_log_info("write-ahead log start point: %s on timeline %u",
1924 : xlogstart, starttli);
1925 :
1926 : /*
1927 : * Get the header
1928 : */
1929 98 : res = PQgetResult(conn);
1930 98 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
1931 : {
1932 0 : pg_log_error("could not get backup header: %s",
1933 : PQerrorMessage(conn));
1934 0 : exit(1);
1935 : }
1936 98 : if (PQntuples(res) < 1)
1937 : {
1938 0 : pg_log_error("no data returned from server");
1939 0 : exit(1);
1940 : }
1941 :
1942 : /*
1943 : * Sum up the total size, for progress reporting
1944 : */
1945 98 : totalsize = totaldone = 0;
1946 98 : tablespacecount = PQntuples(res);
1947 202 : for (i = 0; i < PQntuples(res); i++)
1948 : {
1949 106 : totalsize += atol(PQgetvalue(res, i, 2));
1950 :
1951 : /*
1952 : * Verify tablespace directories are empty. Don't bother with the
1953 : * first once since it can be relocated, and it will be checked before
1954 : * we do anything anyway.
1955 : */
1956 106 : if (format == 'p' && !PQgetisnull(res, i, 1))
1957 : {
1958 6 : char *path = unconstify(char *, get_tablespace_mapping(PQgetvalue(res, i, 1)));
1959 :
1960 6 : verify_dir_is_empty_or_create(path, &made_tablespace_dirs, &found_tablespace_dirs);
1961 : }
1962 : }
1963 :
1964 : /*
1965 : * When writing to stdout, require a single tablespace
1966 : */
1967 96 : if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1)
1968 : {
1969 0 : pg_log_error("can only write single tablespace to stdout, database has %d",
1970 : PQntuples(res));
1971 0 : exit(1);
1972 : }
1973 :
1974 : /*
1975 : * If we're streaming WAL, start the streaming session before we start
1976 : * receiving the actual data chunks.
1977 : */
1978 96 : if (includewal == STREAM_WAL)
1979 : {
1980 92 : if (verbose)
1981 0 : pg_log_info("starting background WAL receiver");
1982 92 : StartLogStreamer(xlogstart, starttli, sysidentifier);
1983 : }
1984 :
1985 : /*
1986 : * Start receiving chunks
1987 : */
1988 196 : for (i = 0; i < PQntuples(res); i++)
1989 : {
1990 102 : if (format == 't')
1991 14 : ReceiveTarFile(conn, res, i);
1992 : else
1993 88 : ReceiveAndUnpackTarFile(conn, res, i);
1994 : } /* Loop over all tablespaces */
1995 :
1996 94 : if (showprogress)
1997 : {
1998 0 : progress_report(PQntuples(res), NULL, true);
1999 0 : if (isatty(fileno(stderr)))
2000 0 : fprintf(stderr, "\n"); /* Need to move to next line */
2001 : }
2002 :
2003 94 : PQclear(res);
2004 :
2005 : /*
2006 : * Get the stop position
2007 : */
2008 94 : res = PQgetResult(conn);
2009 94 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
2010 : {
2011 2 : pg_log_error("could not get write-ahead log end position from server: %s",
2012 : PQerrorMessage(conn));
2013 2 : exit(1);
2014 : }
2015 92 : if (PQntuples(res) != 1)
2016 : {
2017 0 : pg_log_error("no write-ahead log end position returned from server");
2018 0 : exit(1);
2019 : }
2020 92 : strlcpy(xlogend, PQgetvalue(res, 0, 0), sizeof(xlogend));
2021 92 : if (verbose && includewal != NO_WAL)
2022 0 : pg_log_info("write-ahead log end point: %s", xlogend);
2023 92 : PQclear(res);
2024 :
2025 92 : res = PQgetResult(conn);
2026 92 : if (PQresultStatus(res) != PGRES_COMMAND_OK)
2027 : {
2028 6 : const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
2029 :
2030 12 : if (sqlstate &&
2031 6 : strcmp(sqlstate, ERRCODE_DATA_CORRUPTED) == 0)
2032 : {
2033 6 : pg_log_error("checksum error occurred");
2034 6 : checksum_failure = true;
2035 : }
2036 : else
2037 : {
2038 0 : pg_log_error("final receive failed: %s",
2039 : PQerrorMessage(conn));
2040 : }
2041 6 : exit(1);
2042 : }
2043 :
2044 86 : if (bgchild > 0)
2045 : {
2046 : #ifndef WIN32
2047 : int status;
2048 : pid_t r;
2049 : #else
2050 : DWORD status;
2051 :
2052 : /*
2053 : * get a pointer sized version of bgchild to avoid warnings about
2054 : * casting to a different size on WIN64.
2055 : */
2056 : intptr_t bgchild_handle = bgchild;
2057 : uint32 hi,
2058 : lo;
2059 : #endif
2060 :
2061 82 : if (verbose)
2062 0 : pg_log_info("waiting for background process to finish streaming ...");
2063 :
2064 : #ifndef WIN32
2065 82 : if (write(bgpipe[1], xlogend, strlen(xlogend)) != strlen(xlogend))
2066 : {
2067 0 : pg_log_info("could not send command to background pipe: %m");
2068 0 : exit(1);
2069 : }
2070 :
2071 : /* Just wait for the background process to exit */
2072 82 : r = waitpid(bgchild, &status, 0);
2073 82 : if (r == (pid_t) -1)
2074 : {
2075 0 : pg_log_error("could not wait for child process: %m");
2076 0 : exit(1);
2077 : }
2078 82 : if (r != bgchild)
2079 : {
2080 0 : pg_log_error("child %d died, expected %d", (int) r, (int) bgchild);
2081 0 : exit(1);
2082 : }
2083 82 : if (status != 0)
2084 : {
2085 2 : pg_log_error("%s", wait_result_to_str(status));
2086 2 : exit(1);
2087 : }
2088 : /* Exited normally, we're happy! */
2089 : #else /* WIN32 */
2090 :
2091 : /*
2092 : * On Windows, since we are in the same process, we can just store the
2093 : * value directly in the variable, and then set the flag that says
2094 : * it's there.
2095 : */
2096 : if (sscanf(xlogend, "%X/%X", &hi, &lo) != 2)
2097 : {
2098 : pg_log_error("could not parse write-ahead log location \"%s\"",
2099 : xlogend);
2100 : exit(1);
2101 : }
2102 : xlogendptr = ((uint64) hi) << 32 | lo;
2103 : InterlockedIncrement(&has_xlogendptr);
2104 :
2105 : /* First wait for the thread to exit */
2106 : if (WaitForSingleObjectEx((HANDLE) bgchild_handle, INFINITE, FALSE) !=
2107 : WAIT_OBJECT_0)
2108 : {
2109 : _dosmaperr(GetLastError());
2110 : pg_log_error("could not wait for child thread: %m");
2111 : exit(1);
2112 : }
2113 : if (GetExitCodeThread((HANDLE) bgchild_handle, &status) == 0)
2114 : {
2115 : _dosmaperr(GetLastError());
2116 : pg_log_error("could not get child thread exit status: %m");
2117 : exit(1);
2118 : }
2119 : if (status != 0)
2120 : {
2121 : pg_log_error("child thread exited with error %u",
2122 : (unsigned int) status);
2123 : exit(1);
2124 : }
2125 : /* Exited normally, we're happy */
2126 : #endif
2127 : }
2128 :
2129 : /* Free the configuration file contents */
2130 84 : destroyPQExpBuffer(recoveryconfcontents);
2131 :
2132 : /*
2133 : * End of copy data. Final result is already checked inside the loop.
2134 : */
2135 84 : PQclear(res);
2136 84 : PQfinish(conn);
2137 84 : conn = NULL;
2138 :
2139 : /*
2140 : * Make data persistent on disk once backup is completed. For tar format
2141 : * once syncing the parent directory is fine, each tar file created per
2142 : * tablespace has been already synced. In plain format, all the data of
2143 : * the base directory is synced, taking into account all the tablespaces.
2144 : * Errors are not considered fatal.
2145 : */
2146 84 : if (do_sync)
2147 : {
2148 34 : if (verbose)
2149 0 : pg_log_info("syncing data to disk ...");
2150 34 : if (format == 't')
2151 : {
2152 8 : if (strcmp(basedir, "-") != 0)
2153 8 : (void) fsync_fname(basedir, true);
2154 : }
2155 : else
2156 : {
2157 26 : (void) fsync_pgdata(basedir, serverVersion);
2158 : }
2159 : }
2160 :
2161 84 : if (verbose)
2162 0 : pg_log_info("base backup completed");
2163 84 : }
2164 :
2165 :
2166 : int
2167 128 : main(int argc, char **argv)
2168 : {
2169 : static struct option long_options[] = {
2170 : {"help", no_argument, NULL, '?'},
2171 : {"version", no_argument, NULL, 'V'},
2172 : {"pgdata", required_argument, NULL, 'D'},
2173 : {"format", required_argument, NULL, 'F'},
2174 : {"checkpoint", required_argument, NULL, 'c'},
2175 : {"create-slot", no_argument, NULL, 'C'},
2176 : {"max-rate", required_argument, NULL, 'r'},
2177 : {"write-recovery-conf", no_argument, NULL, 'R'},
2178 : {"slot", required_argument, NULL, 'S'},
2179 : {"tablespace-mapping", required_argument, NULL, 'T'},
2180 : {"wal-method", required_argument, NULL, 'X'},
2181 : {"gzip", no_argument, NULL, 'z'},
2182 : {"compress", required_argument, NULL, 'Z'},
2183 : {"label", required_argument, NULL, 'l'},
2184 : {"no-clean", no_argument, NULL, 'n'},
2185 : {"no-sync", no_argument, NULL, 'N'},
2186 : {"dbname", required_argument, NULL, 'd'},
2187 : {"host", required_argument, NULL, 'h'},
2188 : {"port", required_argument, NULL, 'p'},
2189 : {"username", required_argument, NULL, 'U'},
2190 : {"no-password", no_argument, NULL, 'w'},
2191 : {"password", no_argument, NULL, 'W'},
2192 : {"status-interval", required_argument, NULL, 's'},
2193 : {"verbose", no_argument, NULL, 'v'},
2194 : {"progress", no_argument, NULL, 'P'},
2195 : {"waldir", required_argument, NULL, 1},
2196 : {"no-slot", no_argument, NULL, 2},
2197 : {"no-verify-checksums", no_argument, NULL, 3},
2198 : {NULL, 0, NULL, 0}
2199 : };
2200 : int c;
2201 :
2202 : int option_index;
2203 :
2204 128 : pg_logging_init(argv[0]);
2205 128 : progname = get_progname(argv[0]);
2206 128 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
2207 :
2208 128 : if (argc > 1)
2209 : {
2210 126 : if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-?") == 0)
2211 : {
2212 2 : usage();
2213 2 : exit(0);
2214 : }
2215 124 : else if (strcmp(argv[1], "-V") == 0
2216 124 : || strcmp(argv[1], "--version") == 0)
2217 : {
2218 2 : puts("pg_basebackup (PostgreSQL) " PG_VERSION);
2219 2 : exit(0);
2220 : }
2221 : }
2222 :
2223 124 : atexit(cleanup_directories_atexit);
2224 :
2225 124 : while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP",
2226 : long_options, &option_index)) != -1)
2227 : {
2228 370 : switch (c)
2229 : {
2230 : case 'C':
2231 8 : create_slot = true;
2232 8 : break;
2233 : case 'D':
2234 120 : basedir = pg_strdup(optarg);
2235 120 : break;
2236 : case 'F':
2237 28 : if (strcmp(optarg, "p") == 0 || strcmp(optarg, "plain") == 0)
2238 18 : format = 'p';
2239 10 : else if (strcmp(optarg, "t") == 0 || strcmp(optarg, "tar") == 0)
2240 10 : format = 't';
2241 : else
2242 : {
2243 0 : pg_log_error("invalid output format \"%s\", must be \"plain\" or \"tar\"",
2244 : optarg);
2245 0 : exit(1);
2246 : }
2247 28 : break;
2248 : case 'r':
2249 0 : maxrate = parse_max_rate(optarg);
2250 0 : break;
2251 : case 'R':
2252 4 : writerecoveryconf = true;
2253 4 : break;
2254 : case 'S':
2255 :
2256 : /*
2257 : * When specifying replication slot name, use a permanent
2258 : * slot.
2259 : */
2260 14 : replication_slot = pg_strdup(optarg);
2261 14 : temp_replication_slot = false;
2262 14 : break;
2263 : case 2:
2264 4 : no_slot = true;
2265 4 : break;
2266 : case 'T':
2267 16 : tablespace_list_append(optarg);
2268 4 : break;
2269 : case 'X':
2270 36 : if (strcmp(optarg, "n") == 0 ||
2271 18 : strcmp(optarg, "none") == 0)
2272 : {
2273 4 : includewal = NO_WAL;
2274 : }
2275 28 : else if (strcmp(optarg, "f") == 0 ||
2276 14 : strcmp(optarg, "fetch") == 0)
2277 : {
2278 2 : includewal = FETCH_WAL;
2279 : }
2280 24 : else if (strcmp(optarg, "s") == 0 ||
2281 12 : strcmp(optarg, "stream") == 0)
2282 : {
2283 12 : includewal = STREAM_WAL;
2284 : }
2285 : else
2286 : {
2287 0 : pg_log_error("invalid wal-method option \"%s\", must be \"fetch\", \"stream\", or \"none\"",
2288 : optarg);
2289 0 : exit(1);
2290 : }
2291 18 : break;
2292 : case 1:
2293 2 : xlog_dir = pg_strdup(optarg);
2294 2 : break;
2295 : case 'l':
2296 0 : label = pg_strdup(optarg);
2297 0 : break;
2298 : case 'n':
2299 2 : noclean = true;
2300 2 : break;
2301 : case 'N':
2302 50 : do_sync = false;
2303 50 : break;
2304 : case 'z':
2305 : #ifdef HAVE_LIBZ
2306 0 : compresslevel = Z_DEFAULT_COMPRESSION;
2307 : #else
2308 : compresslevel = 1; /* will be rejected below */
2309 : #endif
2310 0 : break;
2311 : case 'Z':
2312 0 : compresslevel = atoi(optarg);
2313 0 : if (compresslevel < 0 || compresslevel > 9)
2314 : {
2315 0 : pg_log_error("invalid compression level \"%s\"", optarg);
2316 0 : exit(1);
2317 : }
2318 0 : break;
2319 : case 'c':
2320 0 : if (pg_strcasecmp(optarg, "fast") == 0)
2321 0 : fastcheckpoint = true;
2322 0 : else if (pg_strcasecmp(optarg, "spread") == 0)
2323 0 : fastcheckpoint = false;
2324 : else
2325 : {
2326 0 : pg_log_error("invalid checkpoint argument \"%s\", must be \"fast\" or \"spread\"",
2327 : optarg);
2328 0 : exit(1);
2329 : }
2330 0 : break;
2331 : case 'd':
2332 0 : connection_string = pg_strdup(optarg);
2333 0 : break;
2334 : case 'h':
2335 50 : dbhost = pg_strdup(optarg);
2336 50 : break;
2337 : case 'p':
2338 50 : dbport = pg_strdup(optarg);
2339 50 : break;
2340 : case 'U':
2341 0 : dbuser = pg_strdup(optarg);
2342 0 : break;
2343 : case 'w':
2344 0 : dbgetpassword = -1;
2345 0 : break;
2346 : case 'W':
2347 0 : dbgetpassword = 1;
2348 0 : break;
2349 : case 's':
2350 0 : standby_message_timeout = atoi(optarg) * 1000;
2351 0 : if (standby_message_timeout < 0)
2352 : {
2353 0 : pg_log_error("invalid status interval \"%s\"", optarg);
2354 0 : exit(1);
2355 : }
2356 0 : break;
2357 : case 'v':
2358 0 : verbose++;
2359 0 : break;
2360 : case 'P':
2361 0 : showprogress = true;
2362 0 : break;
2363 : case 3:
2364 2 : verify_checksums = false;
2365 2 : break;
2366 : default:
2367 :
2368 : /*
2369 : * getopt_long already emitted a complaint
2370 : */
2371 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2372 : progname);
2373 2 : exit(1);
2374 : }
2375 : }
2376 :
2377 : /*
2378 : * Any non-option arguments?
2379 : */
2380 110 : if (optind < argc)
2381 : {
2382 0 : pg_log_error("too many command-line arguments (first is \"%s\")",
2383 : argv[optind]);
2384 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2385 : progname);
2386 0 : exit(1);
2387 : }
2388 :
2389 : /*
2390 : * Required arguments
2391 : */
2392 110 : if (basedir == NULL)
2393 : {
2394 2 : pg_log_error("no target directory specified");
2395 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2396 : progname);
2397 2 : exit(1);
2398 : }
2399 :
2400 : /*
2401 : * Mutually exclusive arguments
2402 : */
2403 108 : if (format == 'p' && compresslevel != 0)
2404 : {
2405 0 : pg_log_error("only tar mode backups can be compressed");
2406 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2407 : progname);
2408 0 : exit(1);
2409 : }
2410 :
2411 108 : if (format == 't' && includewal == STREAM_WAL && strcmp(basedir, "-") == 0)
2412 : {
2413 0 : pg_log_error("cannot stream write-ahead logs in tar mode to stdout");
2414 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2415 : progname);
2416 0 : exit(1);
2417 : }
2418 :
2419 108 : if (replication_slot && includewal != STREAM_WAL)
2420 : {
2421 2 : pg_log_error("replication slots can only be used with WAL streaming");
2422 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2423 : progname);
2424 2 : exit(1);
2425 : }
2426 :
2427 106 : if (no_slot)
2428 : {
2429 4 : if (replication_slot)
2430 : {
2431 2 : pg_log_error("--no-slot cannot be used with slot name");
2432 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2433 : progname);
2434 2 : exit(1);
2435 : }
2436 2 : temp_replication_slot = false;
2437 : }
2438 :
2439 104 : if (create_slot)
2440 : {
2441 6 : if (!replication_slot)
2442 : {
2443 2 : pg_log_error("%s needs a slot to be specified using --slot",
2444 : "--create-slot");
2445 2 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2446 : progname);
2447 2 : exit(1);
2448 : }
2449 :
2450 4 : if (no_slot)
2451 : {
2452 0 : pg_log_error("--create-slot and --no-slot are incompatible options");
2453 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2454 : progname);
2455 0 : exit(1);
2456 : }
2457 : }
2458 :
2459 102 : if (xlog_dir)
2460 : {
2461 2 : if (format != 'p')
2462 : {
2463 0 : pg_log_error("WAL directory location can only be specified in plain mode");
2464 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2465 : progname);
2466 0 : exit(1);
2467 : }
2468 :
2469 : /* clean up xlog directory name, check it's absolute */
2470 2 : canonicalize_path(xlog_dir);
2471 2 : if (!is_absolute_path(xlog_dir))
2472 : {
2473 0 : pg_log_error("WAL directory location must be an absolute path");
2474 0 : fprintf(stderr, _("Try \"%s --help\" for more information.\n"),
2475 : progname);
2476 0 : exit(1);
2477 : }
2478 : }
2479 :
2480 : #ifndef HAVE_LIBZ
2481 : if (compresslevel != 0)
2482 : {
2483 : pg_log_error("this build does not support compression");
2484 : exit(1);
2485 : }
2486 : #endif
2487 :
2488 : /* connection in replication mode to server */
2489 102 : conn = GetConnection();
2490 102 : if (!conn)
2491 : {
2492 : /* Error message already written in GetConnection() */
2493 4 : exit(1);
2494 : }
2495 98 : atexit(disconnect_atexit);
2496 :
2497 : /*
2498 : * Set umask so that directories/files are created with the same
2499 : * permissions as directories/files in the source data directory.
2500 : *
2501 : * pg_mode_mask is set to owner-only by default and then updated in
2502 : * GetConnection() where we get the mode from the server-side with
2503 : * RetrieveDataDirCreatePerm() and then call SetDataDirectoryCreatePerm().
2504 : */
2505 98 : umask(pg_mode_mask);
2506 :
2507 : /*
2508 : * Verify that the target directory exists, or create it. For plaintext
2509 : * backups, always require the directory. For tar backups, require it
2510 : * unless we are writing to stdout.
2511 : */
2512 98 : if (format == 'p' || strcmp(basedir, "-") != 0)
2513 98 : verify_dir_is_empty_or_create(basedir, &made_new_pgdata, &found_existing_pgdata);
2514 :
2515 : /* determine remote server's xlog segment size */
2516 98 : if (!RetrieveWalSegSize(conn))
2517 0 : exit(1);
2518 :
2519 : /* Create pg_wal symlink, if required */
2520 98 : if (xlog_dir)
2521 : {
2522 : char *linkloc;
2523 :
2524 2 : verify_dir_is_empty_or_create(xlog_dir, &made_new_xlogdir, &found_existing_xlogdir);
2525 :
2526 : /*
2527 : * Form name of the place where the symlink must go. pg_xlog has been
2528 : * renamed to pg_wal in post-10 clusters.
2529 : */
2530 2 : linkloc = psprintf("%s/%s", basedir,
2531 2 : PQserverVersion(conn) < MINIMUM_VERSION_FOR_PG_WAL ?
2532 : "pg_xlog" : "pg_wal");
2533 :
2534 : #ifdef HAVE_SYMLINK
2535 2 : if (symlink(xlog_dir, linkloc) != 0)
2536 : {
2537 0 : pg_log_error("could not create symbolic link \"%s\": %m", linkloc);
2538 0 : exit(1);
2539 : }
2540 : #else
2541 : pg_log_error("symlinks are not supported on this platform");
2542 : exit(1);
2543 : #endif
2544 2 : free(linkloc);
2545 : }
2546 :
2547 98 : BaseBackup();
2548 :
2549 84 : success = true;
2550 84 : return 0;
2551 : }
|
