Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * streamutil.c - utility functions for pg_basebackup, pg_receivewal and
4 : * pg_recvlogical
5 : *
6 : * Author: Magnus Hagander <magnus@hagander.net>
7 : *
8 : * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
9 : *
10 : * IDENTIFICATION
11 : * src/bin/pg_basebackup/streamutil.c
12 : *-------------------------------------------------------------------------
13 : */
14 :
15 : #include "postgres_fe.h"
16 :
17 : #include <sys/time.h>
18 : #include <unistd.h>
19 :
20 : #include "access/xlog_internal.h"
21 : #include "common/connect.h"
22 : #include "common/fe_memutils.h"
23 : #include "common/file_perm.h"
24 : #include "common/logging.h"
25 : #include "common/string.h"
26 : #include "datatype/timestamp.h"
27 : #include "port/pg_bswap.h"
28 : #include "pqexpbuffer.h"
29 : #include "receivelog.h"
30 : #include "streamutil.h"
31 :
32 : #define ERRCODE_DUPLICATE_OBJECT "42710"
33 :
34 : uint32 WalSegSz;
35 :
36 : static bool RetrieveDataDirCreatePerm(PGconn *conn);
37 :
38 : /* SHOW command for replication connection was introduced in version 10 */
39 : #define MINIMUM_VERSION_FOR_SHOW_CMD 100000
40 :
41 : /*
42 : * Group access is supported from version 11.
43 : */
44 : #define MINIMUM_VERSION_FOR_GROUP_ACCESS 110000
45 :
46 : const char *progname;
47 : char *connection_string = NULL;
48 : char *dbhost = NULL;
49 : char *dbuser = NULL;
50 : char *dbport = NULL;
51 : char *dbname = NULL;
52 : int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */
53 : static char *password = NULL;
54 : PGconn *conn = NULL;
55 :
56 : /*
57 : * Connect to the server. Returns a valid PGconn pointer if connected,
58 : * or NULL on non-permanent error. On permanent error, the function will
59 : * call exit(1) directly.
60 : */
61 : PGconn *
62 324 : GetConnection(void)
63 : {
64 : PGconn *tmpconn;
65 324 : int argcount = 7; /* dbname, replication, fallback_app_name,
66 : * host, user, port, password */
67 : int i;
68 : const char **keywords;
69 : const char **values;
70 : const char *tmpparam;
71 : bool need_password;
72 324 : PQconninfoOption *conn_opts = NULL;
73 : PQconninfoOption *conn_opt;
74 324 : char *err_msg = NULL;
75 :
76 : /* pg_recvlogical uses dbname only; others use connection_string only. */
77 : Assert(dbname == NULL || connection_string == NULL);
78 :
79 : /*
80 : * Merge the connection info inputs given in form of connection string,
81 : * options and default values (dbname=replication, replication=true, etc.)
82 : * Explicitly discard any dbname value in the connection string;
83 : * otherwise, PQconnectdbParams() would interpret that value as being
84 : * itself a connection string.
85 : */
86 324 : i = 0;
87 324 : if (connection_string)
88 : {
89 0 : conn_opts = PQconninfoParse(connection_string, &err_msg);
90 0 : if (conn_opts == NULL)
91 : {
92 0 : pg_log_error("%s", err_msg);
93 0 : exit(1);
94 : }
95 :
96 0 : for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
97 : {
98 0 : if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
99 0 : strcmp(conn_opt->keyword, "dbname") != 0)
100 0 : argcount++;
101 : }
102 :
103 0 : keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
104 0 : values = pg_malloc0((argcount + 1) * sizeof(*values));
105 :
106 0 : for (conn_opt = conn_opts; conn_opt->keyword != NULL; conn_opt++)
107 : {
108 0 : if (conn_opt->val != NULL && conn_opt->val[0] != '\0' &&
109 0 : strcmp(conn_opt->keyword, "dbname") != 0)
110 : {
111 0 : keywords[i] = conn_opt->keyword;
112 0 : values[i] = conn_opt->val;
113 0 : i++;
114 : }
115 : }
116 : }
117 : else
118 : {
119 324 : keywords = pg_malloc0((argcount + 1) * sizeof(*keywords));
120 324 : values = pg_malloc0((argcount + 1) * sizeof(*values));
121 : }
122 :
123 324 : keywords[i] = "dbname";
124 324 : values[i] = dbname == NULL ? "replication" : dbname;
125 324 : i++;
126 324 : keywords[i] = "replication";
127 324 : values[i] = dbname == NULL ? "true" : "database";
128 324 : i++;
129 324 : keywords[i] = "fallback_application_name";
130 324 : values[i] = progname;
131 324 : i++;
132 :
133 324 : if (dbhost)
134 : {
135 132 : keywords[i] = "host";
136 132 : values[i] = dbhost;
137 132 : i++;
138 : }
139 324 : if (dbuser)
140 : {
141 0 : keywords[i] = "user";
142 0 : values[i] = dbuser;
143 0 : i++;
144 : }
145 324 : if (dbport)
146 : {
147 132 : keywords[i] = "port";
148 132 : values[i] = dbport;
149 132 : i++;
150 : }
151 :
152 : /* If -W was given, force prompt for password, but only the first time */
153 324 : need_password = (dbgetpassword == 1 && !password);
154 :
155 : do
156 : {
157 : /* Get a new password if appropriate */
158 324 : if (need_password)
159 : {
160 0 : if (password)
161 0 : free(password);
162 0 : password = simple_prompt("Password: ", false);
163 0 : need_password = false;
164 : }
165 :
166 : /* Use (or reuse, on a subsequent connection) password if we have it */
167 324 : if (password)
168 : {
169 0 : keywords[i] = "password";
170 0 : values[i] = password;
171 : }
172 : else
173 : {
174 324 : keywords[i] = NULL;
175 324 : values[i] = NULL;
176 : }
177 :
178 324 : tmpconn = PQconnectdbParams(keywords, values, true);
179 :
180 : /*
181 : * If there is too little memory even to allocate the PGconn object
182 : * and PQconnectdbParams returns NULL, we call exit(1) directly.
183 : */
184 324 : if (!tmpconn)
185 : {
186 0 : pg_log_error("could not connect to server");
187 0 : exit(1);
188 : }
189 :
190 : /* If we need a password and -w wasn't given, loop back and get one */
191 328 : if (PQstatus(tmpconn) == CONNECTION_BAD &&
192 4 : PQconnectionNeedsPassword(tmpconn) &&
193 0 : dbgetpassword != -1)
194 : {
195 0 : PQfinish(tmpconn);
196 0 : need_password = true;
197 : }
198 : }
199 324 : while (need_password);
200 :
201 324 : if (PQstatus(tmpconn) != CONNECTION_OK)
202 : {
203 4 : pg_log_error("%s", PQerrorMessage(tmpconn));
204 4 : PQfinish(tmpconn);
205 4 : free(values);
206 4 : free(keywords);
207 4 : if (conn_opts)
208 0 : PQconninfoFree(conn_opts);
209 4 : return NULL;
210 : }
211 :
212 : /* Connection ok! */
213 320 : free(values);
214 320 : free(keywords);
215 320 : if (conn_opts)
216 0 : PQconninfoFree(conn_opts);
217 :
218 : /*
219 : * Set always-secure search path, so malicious users can't get control.
220 : * The capacity to run normal SQL queries was added in PostgreSQL 10, so
221 : * the search path cannot be changed (by us or attackers) on earlier
222 : * versions.
223 : */
224 320 : if (dbname != NULL && PQserverVersion(tmpconn) >= 100000)
225 : {
226 : PGresult *res;
227 :
228 10 : res = PQexec(tmpconn, ALWAYS_SECURE_SEARCH_PATH_SQL);
229 10 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
230 : {
231 0 : pg_log_error("could not clear search_path: %s",
232 : PQerrorMessage(tmpconn));
233 0 : PQclear(res);
234 0 : PQfinish(tmpconn);
235 0 : exit(1);
236 : }
237 10 : PQclear(res);
238 : }
239 :
240 : /*
241 : * Ensure we have the same value of integer_datetimes (now always "on") as
242 : * the server we are connecting to.
243 : */
244 320 : tmpparam = PQparameterStatus(tmpconn, "integer_datetimes");
245 320 : if (!tmpparam)
246 : {
247 0 : pg_log_error("could not determine server setting for integer_datetimes");
248 0 : PQfinish(tmpconn);
249 0 : exit(1);
250 : }
251 :
252 320 : if (strcmp(tmpparam, "on") != 0)
253 : {
254 0 : pg_log_error("integer_datetimes compile flag does not match server");
255 0 : PQfinish(tmpconn);
256 0 : exit(1);
257 : }
258 :
259 : /*
260 : * Retrieve the source data directory mode and use it to construct a umask
261 : * for creating directories and files.
262 : */
263 320 : if (!RetrieveDataDirCreatePerm(tmpconn))
264 : {
265 0 : PQfinish(tmpconn);
266 0 : exit(1);
267 : }
268 :
269 320 : return tmpconn;
270 : }
271 :
272 : /*
273 : * From version 10, explicitly set wal segment size using SHOW wal_segment_size
274 : * since ControlFile is not accessible here.
275 : */
276 : bool
277 162 : RetrieveWalSegSize(PGconn *conn)
278 : {
279 : PGresult *res;
280 : char xlog_unit[3];
281 : int xlog_val,
282 162 : multiplier = 1;
283 :
284 : /* check connection existence */
285 : Assert(conn != NULL);
286 :
287 : /* for previous versions set the default xlog seg size */
288 162 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_SHOW_CMD)
289 : {
290 0 : WalSegSz = DEFAULT_XLOG_SEG_SIZE;
291 0 : return true;
292 : }
293 :
294 162 : res = PQexec(conn, "SHOW wal_segment_size");
295 162 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
296 : {
297 0 : pg_log_error("could not send replication command \"%s\": %s",
298 : "SHOW wal_segment_size", PQerrorMessage(conn));
299 :
300 0 : PQclear(res);
301 0 : return false;
302 : }
303 162 : if (PQntuples(res) != 1 || PQnfields(res) < 1)
304 : {
305 0 : pg_log_error("could not fetch WAL segment size: got %d rows and %d fields, expected %d rows and %d or more fields",
306 : PQntuples(res), PQnfields(res), 1, 1);
307 :
308 0 : PQclear(res);
309 0 : return false;
310 : }
311 :
312 : /* fetch xlog value and unit from the result */
313 162 : if (sscanf(PQgetvalue(res, 0, 0), "%d%s", &xlog_val, xlog_unit) != 2)
314 : {
315 0 : pg_log_error("WAL segment size could not be parsed");
316 0 : PQclear(res);
317 0 : return false;
318 : }
319 :
320 162 : PQclear(res);
321 :
322 : /* set the multiplier based on unit to convert xlog_val to bytes */
323 162 : if (strcmp(xlog_unit, "MB") == 0)
324 162 : multiplier = 1024 * 1024;
325 0 : else if (strcmp(xlog_unit, "GB") == 0)
326 0 : multiplier = 1024 * 1024 * 1024;
327 :
328 : /* convert and set WalSegSz */
329 162 : WalSegSz = xlog_val * multiplier;
330 :
331 162 : if (!IsValidWalSegSize(WalSegSz))
332 : {
333 0 : pg_log_error(ngettext("WAL segment size must be a power of two between 1 MB and 1 GB, but the remote server reported a value of %d byte",
334 : "WAL segment size must be a power of two between 1 MB and 1 GB, but the remote server reported a value of %d bytes",
335 : WalSegSz),
336 : WalSegSz);
337 0 : return false;
338 : }
339 :
340 162 : return true;
341 : }
342 :
343 : /*
344 : * RetrieveDataDirCreatePerm
345 : *
346 : * This function is used to determine the privileges on the server's PG data
347 : * directory and, based on that, set what the permissions will be for
348 : * directories and files we create.
349 : *
350 : * PG11 added support for (optionally) group read/execute rights to be set on
351 : * the data directory. Prior to PG11, only the owner was allowed to have rights
352 : * on the data directory.
353 : */
354 : static bool
355 320 : RetrieveDataDirCreatePerm(PGconn *conn)
356 : {
357 : PGresult *res;
358 : int data_directory_mode;
359 :
360 : /* check connection existence */
361 : Assert(conn != NULL);
362 :
363 : /* for previous versions leave the default group access */
364 320 : if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_GROUP_ACCESS)
365 0 : return true;
366 :
367 320 : res = PQexec(conn, "SHOW data_directory_mode");
368 320 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
369 : {
370 0 : pg_log_error("could not send replication command \"%s\": %s",
371 : "SHOW data_directory_mode", PQerrorMessage(conn));
372 :
373 0 : PQclear(res);
374 0 : return false;
375 : }
376 320 : if (PQntuples(res) != 1 || PQnfields(res) < 1)
377 : {
378 0 : pg_log_error("could not fetch group access flag: got %d rows and %d fields, expected %d rows and %d or more fields",
379 : PQntuples(res), PQnfields(res), 1, 1);
380 :
381 0 : PQclear(res);
382 0 : return false;
383 : }
384 :
385 320 : if (sscanf(PQgetvalue(res, 0, 0), "%o", &data_directory_mode) != 1)
386 : {
387 0 : pg_log_error("group access flag could not be parsed: %s",
388 : PQgetvalue(res, 0, 0));
389 :
390 0 : PQclear(res);
391 0 : return false;
392 : }
393 :
394 320 : SetDataDirectoryCreatePerm(data_directory_mode);
395 :
396 320 : PQclear(res);
397 320 : return true;
398 : }
399 :
400 : /*
401 : * Run IDENTIFY_SYSTEM through a given connection and give back to caller
402 : * some result information if requested:
403 : * - System identifier
404 : * - Current timeline ID
405 : * - Start LSN position
406 : * - Database name (NULL in servers prior to 9.4)
407 : */
408 : bool
409 174 : RunIdentifySystem(PGconn *conn, char **sysid, TimeLineID *starttli,
410 : XLogRecPtr *startpos, char **db_name)
411 : {
412 : PGresult *res;
413 : uint32 hi,
414 : lo;
415 :
416 : /* Check connection existence */
417 : Assert(conn != NULL);
418 :
419 174 : res = PQexec(conn, "IDENTIFY_SYSTEM");
420 174 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
421 : {
422 0 : pg_log_error("could not send replication command \"%s\": %s",
423 : "IDENTIFY_SYSTEM", PQerrorMessage(conn));
424 :
425 0 : PQclear(res);
426 0 : return false;
427 : }
428 174 : if (PQntuples(res) != 1 || PQnfields(res) < 3)
429 : {
430 0 : pg_log_error("could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields",
431 : PQntuples(res), PQnfields(res), 1, 3);
432 :
433 0 : PQclear(res);
434 0 : return false;
435 : }
436 :
437 : /* Get system identifier */
438 174 : if (sysid != NULL)
439 156 : *sysid = pg_strdup(PQgetvalue(res, 0, 0));
440 :
441 : /* Get timeline ID to start streaming from */
442 174 : if (starttli != NULL)
443 158 : *starttli = atoi(PQgetvalue(res, 0, 1));
444 :
445 : /* Get LSN start position if necessary */
446 174 : if (startpos != NULL)
447 : {
448 2 : if (sscanf(PQgetvalue(res, 0, 2), "%X/%X", &hi, &lo) != 2)
449 : {
450 0 : pg_log_error("could not parse write-ahead log location \"%s\"",
451 : PQgetvalue(res, 0, 2));
452 :
453 0 : PQclear(res);
454 0 : return false;
455 : }
456 2 : *startpos = ((uint64) hi) << 32 | lo;
457 : }
458 :
459 : /* Get database name, only available in 9.4 and newer versions */
460 174 : if (db_name != NULL)
461 : {
462 16 : *db_name = NULL;
463 16 : if (PQserverVersion(conn) >= 90400)
464 : {
465 16 : if (PQnfields(res) < 4)
466 : {
467 0 : pg_log_error("could not identify system: got %d rows and %d fields, expected %d rows and %d or more fields",
468 : PQntuples(res), PQnfields(res), 1, 4);
469 :
470 0 : PQclear(res);
471 0 : return false;
472 : }
473 16 : if (!PQgetisnull(res, 0, 3))
474 10 : *db_name = pg_strdup(PQgetvalue(res, 0, 3));
475 : }
476 : }
477 :
478 174 : PQclear(res);
479 174 : return true;
480 : }
481 :
482 : /*
483 : * Create a replication slot for the given connection. This function
484 : * returns true in case of success.
485 : */
486 : bool
487 144 : CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin,
488 : bool is_temporary, bool is_physical, bool reserve_wal,
489 : bool slot_exists_ok)
490 : {
491 : PQExpBuffer query;
492 : PGresult *res;
493 :
494 144 : query = createPQExpBuffer();
495 :
496 : Assert((is_physical && plugin == NULL) ||
497 : (!is_physical && plugin != NULL));
498 : Assert(slot_name != NULL);
499 :
500 : /* Build query */
501 144 : appendPQExpBuffer(query, "CREATE_REPLICATION_SLOT \"%s\"", slot_name);
502 144 : if (is_temporary)
503 136 : appendPQExpBufferStr(query, " TEMPORARY");
504 144 : if (is_physical)
505 : {
506 142 : appendPQExpBufferStr(query, " PHYSICAL");
507 142 : if (reserve_wal)
508 140 : appendPQExpBufferStr(query, " RESERVE_WAL");
509 : }
510 : else
511 : {
512 2 : appendPQExpBuffer(query, " LOGICAL \"%s\"", plugin);
513 2 : if (PQserverVersion(conn) >= 100000)
514 : /* pg_recvlogical doesn't use an exported snapshot, so suppress */
515 2 : appendPQExpBufferStr(query, " NOEXPORT_SNAPSHOT");
516 : }
517 :
518 144 : res = PQexec(conn, query->data);
519 144 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
520 : {
521 2 : const char *sqlstate = PQresultErrorField(res, PG_DIAG_SQLSTATE);
522 :
523 2 : if (slot_exists_ok &&
524 0 : sqlstate &&
525 0 : strcmp(sqlstate, ERRCODE_DUPLICATE_OBJECT) == 0)
526 : {
527 0 : destroyPQExpBuffer(query);
528 0 : PQclear(res);
529 0 : return true;
530 : }
531 : else
532 : {
533 2 : pg_log_error("could not send replication command \"%s\": %s",
534 : query->data, PQerrorMessage(conn));
535 :
536 2 : destroyPQExpBuffer(query);
537 2 : PQclear(res);
538 2 : return false;
539 : }
540 : }
541 :
542 142 : if (PQntuples(res) != 1 || PQnfields(res) != 4)
543 : {
544 0 : pg_log_error("could not create replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields",
545 : slot_name,
546 : PQntuples(res), PQnfields(res), 1, 4);
547 :
548 0 : destroyPQExpBuffer(query);
549 0 : PQclear(res);
550 0 : return false;
551 : }
552 :
553 142 : destroyPQExpBuffer(query);
554 142 : PQclear(res);
555 142 : return true;
556 : }
557 :
558 : /*
559 : * Drop a replication slot for the given connection. This function
560 : * returns true in case of success.
561 : */
562 : bool
563 2 : DropReplicationSlot(PGconn *conn, const char *slot_name)
564 : {
565 : PQExpBuffer query;
566 : PGresult *res;
567 :
568 : Assert(slot_name != NULL);
569 :
570 2 : query = createPQExpBuffer();
571 :
572 : /* Build query */
573 2 : appendPQExpBuffer(query, "DROP_REPLICATION_SLOT \"%s\"",
574 : slot_name);
575 2 : res = PQexec(conn, query->data);
576 2 : if (PQresultStatus(res) != PGRES_COMMAND_OK)
577 : {
578 0 : pg_log_error("could not send replication command \"%s\": %s",
579 : query->data, PQerrorMessage(conn));
580 :
581 0 : destroyPQExpBuffer(query);
582 0 : PQclear(res);
583 0 : return false;
584 : }
585 :
586 2 : if (PQntuples(res) != 0 || PQnfields(res) != 0)
587 : {
588 0 : pg_log_error("could not drop replication slot \"%s\": got %d rows and %d fields, expected %d rows and %d fields",
589 : slot_name,
590 : PQntuples(res), PQnfields(res), 0, 0);
591 :
592 0 : destroyPQExpBuffer(query);
593 0 : PQclear(res);
594 0 : return false;
595 : }
596 :
597 2 : destroyPQExpBuffer(query);
598 2 : PQclear(res);
599 2 : return true;
600 : }
601 :
602 :
603 : /*
604 : * Frontend version of GetCurrentTimestamp(), since we are not linked with
605 : * backend code.
606 : */
607 : TimestampTz
608 1696 : feGetCurrentTimestamp(void)
609 : {
610 : TimestampTz result;
611 : struct timeval tp;
612 :
613 1696 : gettimeofday(&tp, NULL);
614 :
615 1696 : result = (TimestampTz) tp.tv_sec -
616 : ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY);
617 1696 : result = (result * USECS_PER_SEC) + tp.tv_usec;
618 :
619 1696 : return result;
620 : }
621 :
622 : /*
623 : * Frontend version of TimestampDifference(), since we are not linked with
624 : * backend code.
625 : */
626 : void
627 1540 : feTimestampDifference(TimestampTz start_time, TimestampTz stop_time,
628 : long *secs, int *microsecs)
629 : {
630 1540 : TimestampTz diff = stop_time - start_time;
631 :
632 1540 : if (diff <= 0)
633 : {
634 0 : *secs = 0;
635 0 : *microsecs = 0;
636 : }
637 : else
638 : {
639 1540 : *secs = (long) (diff / USECS_PER_SEC);
640 1540 : *microsecs = (int) (diff % USECS_PER_SEC);
641 : }
642 1540 : }
643 :
644 : /*
645 : * Frontend version of TimestampDifferenceExceeds(), since we are not
646 : * linked with backend code.
647 : */
648 : bool
649 1720 : feTimestampDifferenceExceeds(TimestampTz start_time,
650 : TimestampTz stop_time,
651 : int msec)
652 : {
653 1720 : TimestampTz diff = stop_time - start_time;
654 :
655 1720 : return (diff >= msec * INT64CONST(1000));
656 : }
657 :
658 : /*
659 : * Converts an int64 to network byte order.
660 : */
661 : void
662 616 : fe_sendint64(int64 i, char *buf)
663 : {
664 616 : uint64 n64 = pg_hton64(i);
665 :
666 616 : memcpy(buf, &n64, sizeof(n64));
667 616 : }
668 :
669 : /*
670 : * Converts an int64 from network byte order to native format.
671 : */
672 : int64
673 4850 : fe_recvint64(char *buf)
674 : {
675 : uint64 n64;
676 :
677 4850 : memcpy(&n64, buf, sizeof(n64));
678 :
679 4850 : return pg_ntoh64(n64);
680 : }
|
