Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * streamutil.c - utility functions for pg_basebackup, pg_receivewal and
4 : * pg_recvlogical
5 : *
6 : * Author: Magnus Hagander <magnus@hagander.net>
7 : *
8 : * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
9 : *
10 : * IDENTIFICATION
11 : * src/bin/pg_basebackup/streamutil.c
12 : *-------------------------------------------------------------------------
13 : */
14 :
15 : #include "postgres_fe.h"
16 :
17 : #include <sys/time.h>
18 : #include <unistd.h>
19 :
20 : #include "access/xlog_internal.h"
21 : #include "common/fe_memutils.h"
22 : #include "common/file_perm.h"
23 : #include "common/logging.h"
24 : #include "datatype/timestamp.h"
25 : #include "fe_utils/connect.h"
26 : #include "port/pg_bswap.h"
27 : #include "pqexpbuffer.h"
28 : #include "receivelog.h"
29 : #include "streamutil.h"
30 :
31 : #define ERRCODE_DUPLICATE_OBJECT "42710"
32 :
33 : uint32 WalSegSz;
34 :
35 : static bool RetrieveDataDirCreatePerm(PGconn *conn);
36 :
37 : /* SHOW command for replication connection was introduced in version 10 */
38 : #define MINIMUM_VERSION_FOR_SHOW_CMD 100000
39 :
40 : /*
41 : * Group access is supported from version 11.
42 : */
43 : #define MINIMUM_VERSION_FOR_GROUP_ACCESS 110000
44 :
45 : const char *progname;
46 : char *connection_string = NULL;
47 : char *dbhost = NULL;
48 : char *dbuser = NULL;
49 : char *dbport = NULL;
50 : char *dbname = NULL;
51 : int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */
52 : static bool have_password = false;
53 : static char password[100];
54 : PGconn *conn = NULL;
55 :
56 : /*
57 : * Connect to the server. Returns a valid PGconn pointer if connected,
58 : * or NULL on non-permanent error. On permanent error, the function will
59 : * call exit(1) directly.
60 : */
61 : PGconn *
62 304 : GetConnection(void)
63 : {
64 : PGconn *tmpconn;
65 304 : int argcount = 7; /* dbname, replication, fallback_app_name,
66 : * host, user, port, password */
67 : int i;
68 : const char **keywords;
69 : const char **values;
70 : const char *tmpparam;
71 : bool need_password;
72 304 : PQconninfoOption *conn_opts = NULL;
73 : PQconninfoOption *conn_opt;
74 304 : char *err_msg = NULL;
75 :
76 : /* pg_recvlogical uses dbname only; others use connection_string only. */
77 : Assert(dbname == NULL || connection_string == NULL);
78 :
79 : /*
80 : * Merge the connection info inputs given in form of connection string,
81 : * options and default values (dbname=replication, replication=true, etc.)
82 : * Explicitly discard any dbname value in the connection string;
83 : * otherwise, PQconnectdbParams() would interpret that value as being
84 : * itself a connection string.
85 : */
86 304 : i = 0;
87 304 : if (connection_string)
88 : {
89 0 : conn_opts = PQconninfoParse(connection_string, &err_msg);
90 0 : if (conn_opts == NULL)
91 : {
92 0 : pg_log_error("%s", err_msg);
93 0 : exit(1);
94 : }
95 :
96 0 : for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
97 : {
98 0 : if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
99 0 : strcmp(conn_opt->keyword, "dbname") != 0)
100 0 : argcount++;
101 : }
102 :
103 0 : keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
104 0 : values = pg_malloc0((argcount + 1) * sizeof(*values));
105 :
106 0 : for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
107 : {
108 0 : if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
109 0 : strcmp(conn_opt->keyword, "dbname") != 0)
110 : {
111 0 : keywords[i] = conn_opt->keyword;
112 0 : values[i] = conn_opt->val;
113 0 : i++;
114 : }
115 : }
116 : }
117 : else
118 : {
119 304 : keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
120 304 : values = pg_malloc0((argcount + 1) * sizeof(*values));
121 : }
122 :
123 304 : keywords[i] = "dbname";
124 304 : values[i] = dbname == NULL ? "replication" : dbname;
125 304 : i++;
126 304 : keywords[i] = "replication";
127 304 : values[i] = dbname == NULL ? "true" : "database";
128 304 : i++;
129 304 : keywords[i] = "fallback_application_name";
130 304 : values[i] = progname;
131 304 : i++;
132 :
133 304 : if (dbhost)
134 : {
135 112 : keywords[i] = "host";
136 112 : values[i] = dbhost;
137 112 : i++;
138 : }
139 304 : if (dbuser)
140 : {
141 0 : keywords[i] = "user";
142 0 : values[i] = dbuser;
143 0 : i++;
144 : }
145 304 : if (dbport)
146 : {
147 112 : keywords[i] = "port";
148 112 : values[i] = dbport;
149 112 : i++;
150 : }
151 :
152 : /* If -W was given, force prompt for password, but only the first time */
153 304 : need_password = (dbgetpassword == 1 && !have_password);
154 :
155 : do
156 : {
157 : /* Get a new password if appropriate */
158 304 : if (need_password)
159 : {
160 0 : simple_prompt("Password: ", password, sizeof(password), false);
161 0 : have_password = true;
162 0 : need_password = false;
163 : }
164 :
165 : /* Use (or reuse, on a subsequent connection) password if we have it */
166 304 : if (have_password)
167 : {
168 0 : keywords[i] = "password";
169 0 : values[i] = password;
170 : }
171 : else
172 : {
173 304 : keywords[i] = NULL;
174 304 : values[i] = NULL;
175 : }
176 :
177 304 : tmpconn = PQconnectdbParams(keywords, values, true);
178 :
179 : /*
180 : * If there is too little memory even to allocate the PGconn object
181 : * and PQconnectdbParams returns NULL, we call exit(1) directly.
182 : */
183 304 : if (!tmpconn)
184 : {
185 0 : pg_log_error("could not connect to server");
186 0 : exit(1);
187 : }
188 :
189 : /* If we need a password and -w wasn't given, loop back and get one */
190 308 : if (PQstatus(tmpconn) == CONNECTION_BAD &&
191 4 : PQconnectionNeedsPassword(tmpconn) &&
192 0 : dbgetpassword != -1)
193 : {
194 0 : PQfinish(tmpconn);
195 0 : need_password = true;
196 : }
197 : }
198 304 : while (need_password);
199 :
200 304 : if (PQstatus(tmpconn) != CONNECTION_OK)
201 : {
202 4 : pg_log_error("could not connect to server: %s",
203 : PQerrorMessage(tmpconn));
204 4 : PQfinish(tmpconn);
205 4 : free(values);
206 4 : free(keywords);
207 4 : if (conn_opts)
208 0 : PQconninfoFree(conn_opts);
209 4 : return NULL;
210 : }
211 :
212 : /* Connection ok! */
213 300 : free(values);
214 300 : free(keywords);
215 300 : if (conn_opts)
216 0 : PQconninfoFree(conn_opts);
217 :
218 : /*
219 : * Set always-secure search path, so malicious users can't get control.
220 : * The capacity to run normal SQL queries was added in PostgreSQL 10, so
221 : * the search path cannot be changed (by us or attackers) on earlier
222 : * versions.
223 : */
224 300 : if (dbname != NULL && PQserverVersion(tmpconn) >= 100000)
225 : {
226 : PGresult *res;
227 :
228 10 : res = PQexec(tmpconn, ALWAYS_SECURE_SEARCH_PATH_SQL);
229 10 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
230 : {
231 0 : pg_log_error("could not clear search_path: %s",
232 : PQerrorMessage(tmpconn));
233 0 : PQclear(res);
234 0 : PQfinish(tmpconn);
235 0 : exit(1);
236 : }
237 10 : PQclear(res);
238 : }
239 :
240 : /*
241 : * Ensure we have the same value of integer_datetimes (now always "on") as
242 : * the server we are connecting to.
243 : */
244 300 : tmpparam = PQparameterStatus(tmpconn, "integer_datetimes");
245 300 : if (!tmpparam)
246 : {
247 0 : pg_log_error("could not determine server setting for integer_datetimes");
248 0 : PQfinish(tmpconn);
249 0 : exit(1);
250 : }
251 :
252 300 : if (strcmp(tmpparam, "on") != 0)
253 : {
254 0 : pg_log_error("integer_datetimes compile flag does not match server");
255 0 : PQfinish(tmpconn);
256 0 : exit(1);
257 : }
258 :
259 : /*
260 : * Retrieve the source data directory mode and use it to construct a umask
261 : * for creating directories and files.
262 : */
263 300 : if (!RetrieveDataDirCreatePerm(tmpconn))
264 : {
265 0 : PQfinish(tmpconn);
266 0 : exit(1);
267 : }
268 :
269 300 : return tmpconn;
270 : }
271 :
272 : /*
273 : * From version 10, explicitly set wal segment size using SHOW wal_segment_size
274 : * since ControlFile is not accessible here.
275 : */
276 : bool
277 152 : RetrieveWalSegSize(PGconn *conn)
278 : {
279 : PGresult *res;
280 : char xlog_unit[3];
281 : int xlog_val,
282 152 : multiplier = 1;
283 :
284 : /* check connection existence */
285 : Assert(conn != NULL);
286 :
287 : /* for previous versions set the default xlog seg size */
288 152 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_SHOW_CMD)
289 : {
290 0 : WalSegSz = DEFAULT_XLOG_SEG_SIZE;
291 0 : return true;
292 : }
293 :
294 152 : res = PQexec(conn, "SHOW wal_segment_size");
295 152 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
296 : {
297 0 : pg_log_error("could not send replication command \"%s\": %s",
298 : "SHOW wal_segment_size", PQerrorMessage(conn));
299 :
300 0 : PQclear(res);
301 0 : return false;
302 : }
303 152 : if (PQntuples(res) != 1 || PQnfields(res) < 1)
304 : {
305 0 : pg_log_error("could not fetch WAL segment size: got %d rows and %d fields, expected %d rows and %d or more fields",
306 : PQntuples(res), PQnfields(res), 1, 1);
307 :
308 0 : PQclear(res);
309 0 : return false;
310 : }
311 :
312 : /* fetch xlog value and unit from the result */
313 152 : if (sscanf(PQgetvalue(res, 0, 0), "%d%s", &xlog_val, xlog_unit) != 2)
314 : {
315 0 : pg_log_error("WAL segment size could not be parsed");
316 0 : PQclear(res);
317 0 : return false;
318 : }
319 :
320 152 : PQclear(res);
321 :
322 : /* set the multiplier based on unit to convert xlog_val to bytes */
323 152 : if (strcmp(xlog_unit, "MB") == 0)
324 152 : multiplier = 1024 * 1024;
325 0 : else if (strcmp(xlog_unit, "GB") == 0)
326 0 : multiplier = 1024 * 1024 * 1024;
327 :
328 : /* convert and set WalSegSz */
329 152 : WalSegSz = xlog_val * multiplier;
330 :
331 152 : if (!IsValidWalSegSize(WalSegSz))
332 : {
333 0 : pg_log_error(ngettext("WAL segment size must be a power of two between 1 MB and 1 GB, but the remote server reported a value of %d byte",
334 : "WAL segment size must be a power of two between 1 MB and 1 GB, but the remote server reported a value of %d bytes",
335 : WalSegSz),
336 : WalSegSz);
337 0 : return false;
338 : }
339 :
340 152 : return true;
341 : }
342 :
343 : /*
344 : * RetrieveDataDirCreatePerm
345 : *
346 : * This function is used to determine the privileges on the server's PG data
347 : * directory and, based on that, set what the permissions will be for
348 : * directories and files we create.
349 : *
350 : * PG11 added support for (optionally) group read/execute rights to be set on
351 : * the data directory. Prior to PG11, only the owner was allowed to have rights
352 : * on the data directory.
353 : */
354 : static bool
355 300 : RetrieveDataDirCreatePerm(PGconn *conn)
356 : {
357 : PGresult *res;
358 : int data_directory_mode;
359 :
360 : /* check connection existence */
361 : Assert(conn != NULL);
362 :
363 : /* for previous versions leave the default group access */
364 300 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_GROUP_ACCESS)
365 0 : return true;
366 :
367 300 : res = PQexec(conn, "SHOW data_directory_mode");
368 300 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
369 : {
370 0 : pg_log_error("could not send replication command \"%s\": %s",
371 : "SHOW data_directory_mode", PQerrorMessage(conn));
372 :
373 0 : PQclear(res);
374 0 : return false;
375 : }
376 300 : if (PQntuples(res) != 1 || PQnfields(res) < 1)
377 : {
378 0 : pg_log_error("could not fetch group access flag: got %d rows and %d fields, expected %d rows and %d or more fields",
379 : PQntuples(res), PQnfields(res), 1, 1);
380 :
381 0 : PQclear(res);
382 0 : return false;
383 : }
384 :
385 300 : if (sscanf(PQgetvalue(res, 0, 0), "%o", &data_directory_mode) != 1)
386 : {
387 0 : pg_log_error("group access flag could not be parsed: %s",
388 : PQgetvalue(res, 0, 0));
389 :
390 0 : PQclear(res);
391 0 : return false;
392 : }
393 :
394 300 : SetDataDirectoryCreatePerm(data_directory_mode);
395 :
396 300 : PQclear(res);
397 300 : return true;
398 : }
399 :
400 : /*
401 : * Run IDENTIFY_SYSTEM through a given connection and give back to caller
402 : * some result information if requested:
403 : * - System identifier
404 : * - Current timeline ID
405 : * - Start LSN position
406 : * - Database name (NULL in servers prior to 9.4)
407 : */
408 : bool
409 164 : RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli,
410 : XLogRecPtr *startpos, char **db_name)
411 : {
412 : PGresult *res;
413 : uint32 hi,
414 : lo;
415 :
416 : /* Check connection existence */
417 : Assert(conn != NULL);
418 :
419 164 : res = PQexec(conn, "IDENTIFY_SYSTEM");
420 164 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
421 : {
422 0 : pg_log_error("could not send replication command \"%s\": %s",
423 : "IDENTIFY_SYSTEM", PQerrorMessage(conn));
424 :
425 0 : PQclear(res);
426 0 : return false;
427 : }
428 164 : if (PQntuples(res) != 1 || PQnfields(res) < 3)
429 : {
430 0 : pg_log_error("could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields",
431 : PQntuples(res), PQnfields(res), 1, 3);
432 :
433 0 : PQclear(res);
434 0 : return false;
435 : }
436 :
437 : /* Get system identifier */
438 164 : if (sysid != NULL)
439 146 : *sysid = pg_strdup(PQgetvalue(res, 0, 0));
440 :
441 : /* Get timeline ID to start streaming from */
442 164 : if (starttli != NULL)
443 148 : *starttli = atoi(PQgetvalue(res, 0, 1));
444 :
445 : /* Get LSN start position if necessary */
446 164 : if (startpos != NULL)
447 : {
448 2 : if (sscanf(PQgetvalue(res, 0, 2), "%X/%X", &hi, &lo) != 2)
449 : {
450 0 : pg_log_error("could not parse write-ahead log location \"%s\"",
451 : PQgetvalue(res, 0, 2));
452 :
453 0 : PQclear(res);
454 0 : return false;
455 : }
456 2 : *startpos = ((uint64) hi) << 32 | lo;
457 : }
458 :
459 : /* Get database name, only available in 9.4 and newer versions */
460 164 : if (db_name != NULL)
461 : {
462 16 : *db_name = NULL;
463 16 : if (PQserverVersion(conn) >= 90400)
464 : {
465 16 : if (PQnfields(res) < 4)
466 : {
467 0 : pg_log_error("could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields",
468 : PQntuples(res), PQnfields(res), 1, 4);
469 :
470 0 : PQclear(res);
471 0 : return false;
472 : }
473 16 : if (!PQgetisnull(res, 0, 3))
474 10 : *db_name = pg_strdup(PQgetvalue(res, 0, 3));
475 : }
476 : }
477 :
478 164 : PQclear(res);
479 164 : return true;
480 : }
481 :
482 : /*
483 : * Create a replication slot for the given connection. This function
484 : * returns true in case of success.
485 : */
486 : bool
487 134 : CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin,
488 : bool is_temporary, bool is_physical, bool reserve_wal,
489 : bool slot_exists_ok)
490 : {
491 : PQExpBuffer query;
492 : PGresult *res;
493 :
494 134 : query = createPQExpBuffer();
495 :
496 : Assert((is_physical && plugin == NULL) ||
497 : (!is_physical && plugin != NULL));
498 : Assert(slot_name != NULL);
499 :
500 : /* Build query */
501 134 : appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\"", slot_name);
502 134 : if (is_temporary)
503 126 : appendPQExpBufferStr(query, " TEMPORARY");
504 134 : if (is_physical)
505 : {
506 132 : appendPQExpBufferStr(query, " PHYSICAL");
507 132 : if (reserve_wal)
508 130 : appendPQExpBufferStr(query, " RESERVE_WAL");
509 : }
510 : else
511 : {
512 2 : appendPQExpBuffer(query, " LOGICAL \"%s\"", plugin);
513 2 : if (PQserverVersion(conn) >= 100000)
514 : /* pg_recvlogical doesn't use an exported snapshot, so suppress */
515 2 : appendPQExpBufferStr(query, " NOEXPORT_SNAPSHOT");
516 : }
517 :
518 134 : res = PQexec(conn, query->data);
519 134 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
520 : {
521 2 : const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
522 :
523 2 : if (slot_exists_ok &&
524 0 : sqlstate &&
525 0 : strcmp(sqlstate, ERRCODE_DUPLICATE_OBJECT) == 0)
526 : {
527 0 : destroyPQExpBuffer(query);
528 0 : PQclear(res);
529 0 : return true;
530 : }
531 : else
532 : {
533 2 : pg_log_error("could not send replication command \"%s\": %s",
534 : query->data, PQerrorMessage(conn));
535 :
536 2 : destroyPQExpBuffer(query);
537 2 : PQclear(res);
538 2 : return false;
539 : }
540 : }
541 :
542 132 : if (PQntuples(res) != 1 || PQnfields(res) != 4)
543 : {
544 0 : pg_log_error("could not create replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields",
545 : slot_name,
546 : PQntuples(res), PQnfields(res), 1, 4);
547 :
548 0 : destroyPQExpBuffer(query);
549 0 : PQclear(res);
550 0 : return false;
551 : }
552 :
553 132 : destroyPQExpBuffer(query);
554 132 : PQclear(res);
555 132 : return true;
556 : }
557 :
558 : /*
559 : * Drop a replication slot for the given connection. This function
560 : * returns true in case of success.
561 : */
562 : bool
563 2 : DropReplicationSlot(PGconn *conn, const char *slot_name)
564 : {
565 : PQExpBuffer query;
566 : PGresult *res;
567 :
568 : Assert(slot_name != NULL);
569 :
570 2 : query = createPQExpBuffer();
571 :
572 : /* Build query */
573 2 : appendPQExpBuffer(query, "DROP_REPLICATION_SLOT \"%s\"",
574 : slot_name);
575 2 : res = PQexec(conn, query->data);
576 2 : if (PQresultStatus(res) != PGRES_COMMAND_OK)
577 : {
578 0 : pg_log_error("could not send replication command \"%s\": %s",
579 : query->data, PQerrorMessage(conn));
580 :
581 0 : destroyPQExpBuffer(query);
582 0 : PQclear(res);
583 0 : return false;
584 : }
585 :
586 2 : if (PQntuples(res) != 0 || PQnfields(res) != 0)
587 : {
588 0 : pg_log_error("could not drop replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields",
589 : slot_name,
590 : PQntuples(res), PQnfields(res), 0, 0);
591 :
592 0 : destroyPQExpBuffer(query);
593 0 : PQclear(res);
594 0 : return false;
595 : }
596 :
597 2 : destroyPQExpBuffer(query);
598 2 : PQclear(res);
599 2 : return true;
600 : }
601 :
602 :
603 : /*
604 : * Frontend version of GetCurrentTimestamp(), since we are not linked with
605 : * backend code.
606 : */
607 : TimestampTz
608 2728 : feGetCurrentTimestamp(void)
609 : {
610 : TimestampTz result;
611 : struct timeval tp;
612 :
613 2728 : gettimeofday(&tp, NULL);
614 :
615 2728 : result = (TimestampTz) tp.tv_sec -
616 : ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY);
617 2728 : result = (result * USECS_PER_SEC) + tp.tv_usec;
618 :
619 2728 : return result;
620 : }
621 :
622 : /*
623 : * Frontend version of TimestampDifference(), since we are not linked with
624 : * backend code.
625 : */
626 : void
627 2556 : feTimestampDifference(TimestampTz start_time, TimestampTz stop_time,
628 : long *secs, int *microsecs)
629 : {
630 2556 : TimestampTz diff = stop_time - start_time;
631 :
632 2556 : if (diff <= 0)
633 : {
634 0 : *secs = 0;
635 0 : *microsecs = 0;
636 : }
637 : else
638 : {
639 2556 : *secs = (long) (diff / USECS_PER_SEC);
640 2556 : *microsecs = (int) (diff % USECS_PER_SEC);
641 : }
642 2556 : }
643 :
644 : /*
645 : * Frontend version of TimestampDifferenceExceeds(), since we are not
646 : * linked with backend code.
647 : */
648 : bool
649 2722 : feTimestampDifferenceExceeds(TimestampTz start_time,
650 : TimestampTz stop_time,
651 : int msec)
652 : {
653 2722 : TimestampTz diff = stop_time - start_time;
654 :
655 2722 : return (diff >= msec * INT64CONST(1000));
656 : }
657 :
658 : /*
659 : * Converts an int64 to network byte order.
660 : */
661 : void
662 576 : fe_sendint64(int64 i, char *buf)
663 : {
664 576 : uint64 n64 = pg_hton64(i);
665 :
666 576 : memcpy(buf, &n64, sizeof(n64));
667 576 : }
668 :
669 : /*
670 : * Converts an int64 from network byte order to native format.
671 : */
672 : int64
673 4744 : fe_recvint64(char *buf)
674 : {
675 : uint64 n64;
676 :
677 4744 : memcpy(&n64, buf, sizeof(n64));
678 :
679 4744 : return pg_ntoh64(n64);
680 : }
|
