Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_amcheck.c
4 : * Detects corruption within database relations.
5 : *
6 : * Copyright (c) 2017-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/bin/pg_amcheck/pg_amcheck.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres_fe.h"
14 :
15 : #include <limits.h>
16 : #include <time.h>
17 :
18 : #include "catalog/pg_am_d.h"
19 : #include "catalog/pg_class_d.h"
20 : #include "catalog/pg_namespace_d.h"
21 : #include "common/logging.h"
22 : #include "common/username.h"
23 : #include "fe_utils/cancel.h"
24 : #include "fe_utils/option_utils.h"
25 : #include "fe_utils/parallel_slot.h"
26 : #include "fe_utils/query_utils.h"
27 : #include "fe_utils/simple_list.h"
28 : #include "fe_utils/string_utils.h"
29 : #include "getopt_long.h" /* pgrminclude ignore */
30 : #include "pgtime.h"
31 : #include "storage/block.h"
32 :
33 : typedef struct PatternInfo
34 : {
35 : const char *pattern; /* Unaltered pattern from the command line */
36 : char *db_regex; /* Database regexp parsed from pattern, or
37 : * NULL */
38 : char *nsp_regex; /* Schema regexp parsed from pattern, or NULL */
39 : char *rel_regex; /* Relation regexp parsed from pattern, or
40 : * NULL */
41 : bool heap_only; /* true if rel_regex should only match heap
42 : * tables */
43 : bool btree_only; /* true if rel_regex should only match btree
44 : * indexes */
45 : bool matched; /* true if the pattern matched in any database */
46 : } PatternInfo;
47 :
48 : typedef struct PatternInfoArray
49 : {
50 : PatternInfo *data;
51 : size_t len;
52 : } PatternInfoArray;
53 :
54 : /* pg_amcheck command line options controlled by user flags */
55 : typedef struct AmcheckOptions
56 : {
57 : bool dbpattern;
58 : bool alldb;
59 : bool echo;
60 : bool verbose;
61 : bool strict_names;
62 : bool show_progress;
63 : int jobs;
64 :
65 : /*
66 : * Whether to install missing extensions, and optionally the name of the
67 : * schema in which to install the extension's objects.
68 : */
69 : bool install_missing;
70 : char *install_schema;
71 :
72 : /* Objects to check or not to check, as lists of PatternInfo structs. */
73 : PatternInfoArray include;
74 : PatternInfoArray exclude;
75 :
76 : /*
77 : * As an optimization, if any pattern in the exclude list applies to heap
78 : * tables, or similarly if any such pattern applies to btree indexes, or
79 : * to schemas, then these will be true, otherwise false. These should
80 : * always agree with what you'd conclude by grep'ing through the exclude
81 : * list.
82 : */
83 : bool excludetbl;
84 : bool excludeidx;
85 : bool excludensp;
86 :
87 : /*
88 : * If any inclusion pattern exists, then we should only be checking
89 : * matching relations rather than all relations, so this is true iff
90 : * include is empty.
91 : */
92 : bool allrel;
93 :
94 : /* heap table checking options */
95 : bool no_toast_expansion;
96 : bool reconcile_toast;
97 : bool on_error_stop;
98 : int64 startblock;
99 : int64 endblock;
100 : const char *skip;
101 :
102 : /* btree index checking options */
103 : bool parent_check;
104 : bool rootdescend;
105 : bool heapallindexed;
106 : bool checkunique;
107 :
108 : /* heap and btree hybrid option */
109 : bool no_btree_expansion;
110 : } AmcheckOptions;
111 :
112 : static AmcheckOptions opts = {
113 : .dbpattern = false,
114 : .alldb = false,
115 : .echo = false,
116 : .verbose = false,
117 : .strict_names = true,
118 : .show_progress = false,
119 : .jobs = 1,
120 : .install_missing = false,
121 : .install_schema = "pg_catalog",
122 : .include = {NULL, 0},
123 : .exclude = {NULL, 0},
124 : .excludetbl = false,
125 : .excludeidx = false,
126 : .excludensp = false,
127 : .allrel = true,
128 : .no_toast_expansion = false,
129 : .reconcile_toast = true,
130 : .on_error_stop = false,
131 : .startblock = -1,
132 : .endblock = -1,
133 : .skip = "none",
134 : .parent_check = false,
135 : .rootdescend = false,
136 : .heapallindexed = false,
137 : .checkunique = false,
138 : .no_btree_expansion = false
139 : };
140 :
141 : static const char *progname = NULL;
142 :
143 : /* Whether all relations have so far passed their corruption checks */
144 : static bool all_checks_pass = true;
145 :
146 : /* Time last progress report was displayed */
147 : static pg_time_t last_progress_report = 0;
148 : static bool progress_since_last_stderr = false;
149 :
150 : typedef struct DatabaseInfo
151 : {
152 : char *datname;
153 : char *amcheck_schema; /* escaped, quoted literal */
154 : bool is_checkunique;
155 : } DatabaseInfo;
156 :
157 : typedef struct RelationInfo
158 : {
159 : const DatabaseInfo *datinfo; /* shared by other relinfos */
160 : Oid reloid;
161 : bool is_heap; /* true if heap, false if btree */
162 : char *nspname;
163 : char *relname;
164 : int relpages;
165 : int blocks_to_check;
166 : char *sql; /* set during query run, pg_free'd after */
167 : } RelationInfo;
168 :
169 : /*
170 : * Query for determining if contrib's amcheck is installed. If so, selects the
171 : * namespace name where amcheck's functions can be found.
172 : */
173 : static const char *const amcheck_sql =
174 : "SELECT n.nspname, x.extversion FROM pg_catalog.pg_extension x"
175 : "\nJOIN pg_catalog.pg_namespace n ON x.extnamespace = n.oid"
176 : "\nWHERE x.extname = 'amcheck'";
177 :
178 : static void prepare_heap_command(PQExpBuffer sql, RelationInfo *rel,
179 : PGconn *conn);
180 : static void prepare_btree_command(PQExpBuffer sql, RelationInfo *rel,
181 : PGconn *conn);
182 : static void run_command(ParallelSlot *slot, const char *sql);
183 : static bool verify_heap_slot_handler(PGresult *res, PGconn *conn,
184 : void *context);
185 : static bool verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context);
186 : static void help(const char *progname);
187 : static void progress_report(uint64 relations_total, uint64 relations_checked,
188 : uint64 relpages_total, uint64 relpages_checked,
189 : const char *datname, bool force, bool finished);
190 :
191 : static void append_database_pattern(PatternInfoArray *pia, const char *pattern,
192 : int encoding);
193 : static void append_schema_pattern(PatternInfoArray *pia, const char *pattern,
194 : int encoding);
195 : static void append_relation_pattern(PatternInfoArray *pia, const char *pattern,
196 : int encoding);
197 : static void append_heap_pattern(PatternInfoArray *pia, const char *pattern,
198 : int encoding);
199 : static void append_btree_pattern(PatternInfoArray *pia, const char *pattern,
200 : int encoding);
201 : static void compile_database_list(PGconn *conn, SimplePtrList *databases,
202 : const char *initial_dbname);
203 : static void compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations,
204 : const DatabaseInfo *dat,
205 : uint64 *pagecount);
206 :
207 : #define log_no_match(...) do { \
208 : if (opts.strict_names) \
209 : pg_log_error(__VA_ARGS__); \
210 : else \
211 : pg_log_warning(__VA_ARGS__); \
212 : } while(0)
213 :
214 : #define FREE_AND_SET_NULL(x) do { \
215 : pg_free(x); \
216 : (x) = NULL; \
217 : } while (0)
218 :
219 : int
220 122 : main(int argc, char *argv[])
221 : {
222 122 : PGconn *conn = NULL;
223 : SimplePtrListCell *cell;
224 122 : SimplePtrList databases = {NULL, NULL};
225 122 : SimplePtrList relations = {NULL, NULL};
226 122 : bool failed = false;
227 : const char *latest_datname;
228 : int parallel_workers;
229 : ParallelSlotArray *sa;
230 : PQExpBufferData sql;
231 122 : uint64 reltotal = 0;
232 122 : uint64 pageschecked = 0;
233 122 : uint64 pagestotal = 0;
234 122 : uint64 relprogress = 0;
235 : int pattern_id;
236 :
237 : static struct option long_options[] = {
238 : /* Connection options */
239 : {"host", required_argument, NULL, 'h'},
240 : {"port", required_argument, NULL, 'p'},
241 : {"username", required_argument, NULL, 'U'},
242 : {"no-password", no_argument, NULL, 'w'},
243 : {"password", no_argument, NULL, 'W'},
244 : {"maintenance-db", required_argument, NULL, 1},
245 :
246 : /* check options */
247 : {"all", no_argument, NULL, 'a'},
248 : {"database", required_argument, NULL, 'd'},
249 : {"exclude-database", required_argument, NULL, 'D'},
250 : {"echo", no_argument, NULL, 'e'},
251 : {"index", required_argument, NULL, 'i'},
252 : {"exclude-index", required_argument, NULL, 'I'},
253 : {"jobs", required_argument, NULL, 'j'},
254 : {"progress", no_argument, NULL, 'P'},
255 : {"relation", required_argument, NULL, 'r'},
256 : {"exclude-relation", required_argument, NULL, 'R'},
257 : {"schema", required_argument, NULL, 's'},
258 : {"exclude-schema", required_argument, NULL, 'S'},
259 : {"table", required_argument, NULL, 't'},
260 : {"exclude-table", required_argument, NULL, 'T'},
261 : {"verbose", no_argument, NULL, 'v'},
262 : {"no-dependent-indexes", no_argument, NULL, 2},
263 : {"no-dependent-toast", no_argument, NULL, 3},
264 : {"exclude-toast-pointers", no_argument, NULL, 4},
265 : {"on-error-stop", no_argument, NULL, 5},
266 : {"skip", required_argument, NULL, 6},
267 : {"startblock", required_argument, NULL, 7},
268 : {"endblock", required_argument, NULL, 8},
269 : {"rootdescend", no_argument, NULL, 9},
270 : {"no-strict-names", no_argument, NULL, 10},
271 : {"heapallindexed", no_argument, NULL, 11},
272 : {"parent-check", no_argument, NULL, 12},
273 : {"install-missing", optional_argument, NULL, 13},
274 : {"checkunique", no_argument, NULL, 14},
275 :
276 : {NULL, 0, NULL, 0}
277 : };
278 :
279 : int optindex;
280 : int c;
281 :
282 122 : const char *db = NULL;
283 122 : const char *maintenance_db = NULL;
284 :
285 122 : const char *host = NULL;
286 122 : const char *port = NULL;
287 122 : const char *username = NULL;
288 122 : enum trivalue prompt_password = TRI_DEFAULT;
289 122 : int encoding = pg_get_encoding_from_locale(NULL, false);
290 : ConnParams cparams;
291 :
292 122 : pg_logging_init(argv[0]);
293 122 : progname = get_progname(argv[0]);
294 122 : set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_amcheck"));
295 :
296 122 : handle_help_version_opts(argc, argv, progname, help);
297 :
298 : /* process command-line options */
299 438 : while ((c = getopt_long(argc, argv, "ad:D:eh:Hi:I:j:p:Pr:R:s:S:t:T:U:vwW",
300 : long_options, &optindex)) != -1)
301 : {
302 : char *endptr;
303 : unsigned long optval;
304 :
305 344 : switch (c)
306 : {
307 8 : case 'a':
308 8 : opts.alldb = true;
309 8 : break;
310 66 : case 'd':
311 66 : opts.dbpattern = true;
312 66 : append_database_pattern(&opts.include, optarg, encoding);
313 62 : break;
314 2 : case 'D':
315 2 : opts.dbpattern = true;
316 2 : append_database_pattern(&opts.exclude, optarg, encoding);
317 0 : break;
318 0 : case 'e':
319 0 : opts.echo = true;
320 0 : break;
321 0 : case 'h':
322 0 : host = pg_strdup(optarg);
323 0 : break;
324 22 : case 'i':
325 22 : opts.allrel = false;
326 22 : append_btree_pattern(&opts.include, optarg, encoding);
327 22 : break;
328 4 : case 'I':
329 4 : opts.excludeidx = true;
330 4 : append_btree_pattern(&opts.exclude, optarg, encoding);
331 4 : break;
332 0 : case 'j':
333 0 : if (!option_parse_int(optarg, "-j/--jobs", 1, INT_MAX,
334 : &opts.jobs))
335 0 : exit(1);
336 0 : break;
337 62 : case 'p':
338 62 : port = pg_strdup(optarg);
339 62 : break;
340 0 : case 'P':
341 0 : opts.show_progress = true;
342 0 : break;
343 14 : case 'r':
344 14 : opts.allrel = false;
345 14 : append_relation_pattern(&opts.include, optarg, encoding);
346 14 : break;
347 0 : case 'R':
348 0 : opts.excludeidx = true;
349 0 : opts.excludetbl = true;
350 0 : append_relation_pattern(&opts.exclude, optarg, encoding);
351 0 : break;
352 42 : case 's':
353 42 : opts.allrel = false;
354 42 : append_schema_pattern(&opts.include, optarg, encoding);
355 38 : break;
356 16 : case 'S':
357 16 : opts.excludensp = true;
358 16 : append_schema_pattern(&opts.exclude, optarg, encoding);
359 14 : break;
360 32 : case 't':
361 32 : opts.allrel = false;
362 32 : append_heap_pattern(&opts.include, optarg, encoding);
363 28 : break;
364 6 : case 'T':
365 6 : opts.excludetbl = true;
366 6 : append_heap_pattern(&opts.exclude, optarg, encoding);
367 4 : break;
368 2 : case 'U':
369 2 : username = pg_strdup(optarg);
370 2 : break;
371 0 : case 'v':
372 0 : opts.verbose = true;
373 0 : pg_logging_increase_verbosity();
374 0 : break;
375 0 : case 'w':
376 0 : prompt_password = TRI_NO;
377 0 : break;
378 0 : case 'W':
379 0 : prompt_password = TRI_YES;
380 0 : break;
381 0 : case 1:
382 0 : maintenance_db = pg_strdup(optarg);
383 0 : break;
384 8 : case 2:
385 8 : opts.no_btree_expansion = true;
386 8 : break;
387 2 : case 3:
388 2 : opts.no_toast_expansion = true;
389 2 : break;
390 2 : case 4:
391 2 : opts.reconcile_toast = false;
392 2 : break;
393 0 : case 5:
394 0 : opts.on_error_stop = true;
395 0 : break;
396 0 : case 6:
397 0 : if (pg_strcasecmp(optarg, "all-visible") == 0)
398 0 : opts.skip = "all-visible";
399 0 : else if (pg_strcasecmp(optarg, "all-frozen") == 0)
400 0 : opts.skip = "all-frozen";
401 0 : else if (pg_strcasecmp(optarg, "none") == 0)
402 0 : opts.skip = "none";
403 : else
404 0 : pg_fatal("invalid argument for option %s", "--skip");
405 0 : break;
406 4 : case 7:
407 4 : errno = 0;
408 4 : optval = strtoul(optarg, &endptr, 10);
409 4 : if (endptr == optarg || *endptr != '\0' || errno != 0)
410 2 : pg_fatal("invalid start block");
411 2 : if (optval > MaxBlockNumber)
412 0 : pg_fatal("start block out of bounds");
413 2 : opts.startblock = optval;
414 2 : break;
415 4 : case 8:
416 4 : errno = 0;
417 4 : optval = strtoul(optarg, &endptr, 10);
418 4 : if (endptr == optarg || *endptr != '\0' || errno != 0)
419 2 : pg_fatal("invalid end block");
420 2 : if (optval > MaxBlockNumber)
421 0 : pg_fatal("end block out of bounds");
422 2 : opts.endblock = optval;
423 2 : break;
424 4 : case 9:
425 4 : opts.rootdescend = true;
426 4 : opts.parent_check = true;
427 4 : break;
428 22 : case 10:
429 22 : opts.strict_names = false;
430 22 : break;
431 4 : case 11:
432 4 : opts.heapallindexed = true;
433 4 : break;
434 4 : case 12:
435 4 : opts.parent_check = true;
436 4 : break;
437 0 : case 13:
438 0 : opts.install_missing = true;
439 0 : if (optarg)
440 0 : opts.install_schema = pg_strdup(optarg);
441 0 : break;
442 12 : case 14:
443 12 : opts.checkunique = true;
444 12 : break;
445 2 : default:
446 : /* getopt_long already emitted a complaint */
447 2 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
448 2 : exit(1);
449 : }
450 : }
451 :
452 94 : if (opts.endblock >= 0 && opts.endblock < opts.startblock)
453 2 : pg_fatal("end block precedes start block");
454 :
455 : /*
456 : * A single non-option arguments specifies a database name or connection
457 : * string.
458 : */
459 92 : if (optind < argc)
460 : {
461 50 : db = argv[optind];
462 50 : optind++;
463 : }
464 :
465 92 : if (optind < argc)
466 : {
467 0 : pg_log_error("too many command-line arguments (first is \"%s\")",
468 : argv[optind]);
469 0 : pg_log_error_hint("Try \"%s --help\" for more information.", progname);
470 0 : exit(1);
471 : }
472 :
473 : /* fill cparams except for dbname, which is set below */
474 92 : cparams.pghost = host;
475 92 : cparams.pgport = port;
476 92 : cparams.pguser = username;
477 92 : cparams.prompt_password = prompt_password;
478 92 : cparams.dbname = NULL;
479 92 : cparams.override_dbname = NULL;
480 :
481 92 : setup_cancel_handler(NULL);
482 :
483 : /* choose the database for our initial connection */
484 92 : if (opts.alldb)
485 : {
486 8 : if (db != NULL)
487 0 : pg_fatal("cannot specify a database name with --all");
488 8 : cparams.dbname = maintenance_db;
489 : }
490 84 : else if (db != NULL)
491 : {
492 50 : if (opts.dbpattern)
493 0 : pg_fatal("cannot specify both a database name and database patterns");
494 50 : cparams.dbname = db;
495 : }
496 :
497 92 : if (opts.alldb || opts.dbpattern)
498 : {
499 42 : conn = connectMaintenanceDatabase(&cparams, progname, opts.echo);
500 42 : compile_database_list(conn, &databases, NULL);
501 : }
502 : else
503 : {
504 50 : if (cparams.dbname == NULL)
505 : {
506 0 : if (getenv("PGDATABASE"))
507 0 : cparams.dbname = getenv("PGDATABASE");
508 0 : else if (getenv("PGUSER"))
509 0 : cparams.dbname = getenv("PGUSER");
510 : else
511 0 : cparams.dbname = get_user_name_or_exit(progname);
512 : }
513 50 : conn = connectDatabase(&cparams, progname, opts.echo, false, true);
514 46 : compile_database_list(conn, &databases, PQdb(conn));
515 : }
516 :
517 74 : if (databases.head == NULL)
518 : {
519 0 : if (conn != NULL)
520 0 : disconnectDatabase(conn);
521 0 : pg_log_warning("no databases to check");
522 0 : exit(0);
523 : }
524 :
525 : /*
526 : * Compile a list of all relations spanning all databases to be checked.
527 : */
528 188 : for (cell = databases.head; cell; cell = cell->next)
529 : {
530 : PGresult *result;
531 : int ntups;
532 114 : const char *amcheck_schema = NULL;
533 114 : DatabaseInfo *dat = (DatabaseInfo *) cell->ptr;
534 :
535 114 : cparams.override_dbname = dat->datname;
536 114 : if (conn == NULL || strcmp(PQdb(conn), dat->datname) != 0)
537 : {
538 58 : if (conn != NULL)
539 50 : disconnectDatabase(conn);
540 58 : conn = connectDatabase(&cparams, progname, opts.echo, false, true);
541 : }
542 :
543 : /*
544 : * Optionally install amcheck if not already installed in this
545 : * database.
546 : */
547 114 : if (opts.install_missing)
548 : {
549 : char *schema;
550 : char *install_sql;
551 :
552 : /*
553 : * Must re-escape the schema name for each database, as the
554 : * escaping rules may change.
555 : */
556 0 : schema = PQescapeIdentifier(conn, opts.install_schema,
557 0 : strlen(opts.install_schema));
558 0 : install_sql = psprintf("CREATE EXTENSION IF NOT EXISTS amcheck WITH SCHEMA %s",
559 : schema);
560 :
561 0 : executeCommand(conn, install_sql, opts.echo);
562 0 : pfree(install_sql);
563 0 : pfree(schema);
564 : }
565 :
566 : /*
567 : * Verify that amcheck is installed for this next database. User
568 : * error could result in a database not having amcheck that should
569 : * have it, but we also could be iterating over multiple databases
570 : * where not all of them have amcheck installed (for example,
571 : * 'template1').
572 : */
573 114 : result = executeQuery(conn, amcheck_sql, opts.echo);
574 114 : if (PQresultStatus(result) != PGRES_TUPLES_OK)
575 : {
576 : /* Querying the catalog failed. */
577 0 : pg_log_error("database \"%s\": %s",
578 : PQdb(conn), PQerrorMessage(conn));
579 0 : pg_log_error_detail("Query was: %s", amcheck_sql);
580 0 : PQclear(result);
581 0 : disconnectDatabase(conn);
582 0 : exit(1);
583 : }
584 114 : ntups = PQntuples(result);
585 114 : if (ntups == 0)
586 : {
587 : /* Querying the catalog succeeded, but amcheck is missing. */
588 22 : pg_log_warning("skipping database \"%s\": amcheck is not installed",
589 : PQdb(conn));
590 22 : disconnectDatabase(conn);
591 22 : conn = NULL;
592 22 : continue;
593 : }
594 92 : amcheck_schema = PQgetvalue(result, 0, 0);
595 92 : if (opts.verbose)
596 0 : pg_log_info("in database \"%s\": using amcheck version \"%s\" in schema \"%s\"",
597 : PQdb(conn), PQgetvalue(result, 0, 1), amcheck_schema);
598 92 : dat->amcheck_schema = PQescapeIdentifier(conn, amcheck_schema,
599 : strlen(amcheck_schema));
600 :
601 : /*
602 : * Check the version of amcheck extension. Skip requested unique
603 : * constraint check with warning if it is not yet supported by
604 : * amcheck.
605 : */
606 92 : if (opts.checkunique == true)
607 : {
608 : /*
609 : * Now amcheck has only major and minor versions in the string but
610 : * we also support revision just in case. Now it is expected to be
611 : * zero.
612 : */
613 16 : int vmaj = 0,
614 16 : vmin = 0,
615 16 : vrev = 0;
616 16 : const char *amcheck_version = PQgetvalue(result, 0, 1);
617 :
618 16 : sscanf(amcheck_version, "%d.%d.%d", &vmaj, &vmin, &vrev);
619 :
620 : /*
621 : * checkunique option is supported in amcheck since version 1.4
622 : */
623 16 : if ((vmaj == 1 && vmin < 4) || vmaj == 0)
624 : {
625 2 : pg_log_warning("option %s is not supported by amcheck version %s",
626 : "--checkunique", amcheck_version);
627 2 : dat->is_checkunique = false;
628 : }
629 : else
630 14 : dat->is_checkunique = true;
631 : }
632 :
633 92 : PQclear(result);
634 :
635 92 : compile_relation_list_one_db(conn, &relations, dat, &pagestotal);
636 : }
637 :
638 : /*
639 : * Check that all inclusion patterns matched at least one schema or
640 : * relation that we can check.
641 : */
642 206 : for (pattern_id = 0; pattern_id < opts.include.len; pattern_id++)
643 : {
644 132 : PatternInfo *pat = &opts.include.data[pattern_id];
645 :
646 132 : if (!pat->matched && (pat->nsp_regex != NULL || pat->rel_regex != NULL))
647 : {
648 46 : failed = opts.strict_names;
649 :
650 46 : if (pat->heap_only)
651 14 : log_no_match("no heap tables to check matching \"%s\"",
652 : pat->pattern);
653 32 : else if (pat->btree_only)
654 10 : log_no_match("no btree indexes to check matching \"%s\"",
655 : pat->pattern);
656 22 : else if (pat->rel_regex == NULL)
657 8 : log_no_match("no relations to check in schemas matching \"%s\"",
658 : pat->pattern);
659 : else
660 14 : log_no_match("no relations to check matching \"%s\"",
661 : pat->pattern);
662 : }
663 : }
664 :
665 74 : if (failed)
666 : {
667 2 : if (conn != NULL)
668 2 : disconnectDatabase(conn);
669 2 : exit(1);
670 : }
671 :
672 : /*
673 : * Set parallel_workers to the lesser of opts.jobs and the number of
674 : * relations.
675 : */
676 72 : parallel_workers = 0;
677 15636 : for (cell = relations.head; cell; cell = cell->next)
678 : {
679 15564 : reltotal++;
680 15564 : if (parallel_workers < opts.jobs)
681 64 : parallel_workers++;
682 : }
683 :
684 72 : if (reltotal == 0)
685 : {
686 8 : if (conn != NULL)
687 0 : disconnectDatabase(conn);
688 8 : pg_fatal("no relations to check");
689 : }
690 64 : progress_report(reltotal, relprogress, pagestotal, pageschecked,
691 : NULL, true, false);
692 :
693 : /*
694 : * Main event loop.
695 : *
696 : * We use server-side parallelism to check up to parallel_workers
697 : * relations in parallel. The list of relations was computed in database
698 : * order, which minimizes the number of connects and disconnects as we
699 : * process the list.
700 : */
701 64 : latest_datname = NULL;
702 64 : sa = ParallelSlotsSetup(parallel_workers, &cparams, progname, opts.echo,
703 : NULL);
704 64 : if (conn != NULL)
705 : {
706 58 : ParallelSlotsAdoptConn(sa, conn);
707 58 : conn = NULL;
708 : }
709 :
710 64 : initPQExpBuffer(&sql);
711 15628 : for (relprogress = 0, cell = relations.head; cell; cell = cell->next)
712 : {
713 : ParallelSlot *free_slot;
714 : RelationInfo *rel;
715 :
716 15564 : rel = (RelationInfo *) cell->ptr;
717 :
718 15564 : if (CancelRequested)
719 : {
720 0 : failed = true;
721 0 : break;
722 : }
723 :
724 : /*
725 : * The list of relations is in database sorted order. If this next
726 : * relation is in a different database than the last one seen, we are
727 : * about to start checking this database. Note that other slots may
728 : * still be working on relations from prior databases.
729 : */
730 15564 : latest_datname = rel->datinfo->datname;
731 :
732 15564 : progress_report(reltotal, relprogress, pagestotal, pageschecked,
733 : latest_datname, false, false);
734 :
735 15564 : relprogress++;
736 15564 : pageschecked += rel->blocks_to_check;
737 :
738 : /*
739 : * Get a parallel slot for the next amcheck command, blocking if
740 : * necessary until one is available, or until a previously issued slot
741 : * command fails, indicating that we should abort checking the
742 : * remaining objects.
743 : */
744 15564 : free_slot = ParallelSlotsGetIdle(sa, rel->datinfo->datname);
745 15564 : if (!free_slot)
746 : {
747 : /*
748 : * Something failed. We don't need to know what it was, because
749 : * the handler should already have emitted the necessary error
750 : * messages.
751 : */
752 0 : failed = true;
753 0 : break;
754 : }
755 :
756 15564 : if (opts.verbose)
757 0 : PQsetErrorVerbosity(free_slot->connection, PQERRORS_VERBOSE);
758 :
759 : /*
760 : * Execute the appropriate amcheck command for this relation using our
761 : * slot's database connection. We do not wait for the command to
762 : * complete, nor do we perform any error checking, as that is done by
763 : * the parallel slots and our handler callback functions.
764 : */
765 15564 : if (rel->is_heap)
766 : {
767 6906 : if (opts.verbose)
768 : {
769 0 : if (opts.show_progress && progress_since_last_stderr)
770 0 : fprintf(stderr, "\n");
771 0 : pg_log_info("checking heap table \"%s.%s.%s\"",
772 : rel->datinfo->datname, rel->nspname, rel->relname);
773 0 : progress_since_last_stderr = false;
774 : }
775 6906 : prepare_heap_command(&sql, rel, free_slot->connection);
776 6906 : rel->sql = pstrdup(sql.data); /* pg_free'd after command */
777 6906 : ParallelSlotSetHandler(free_slot, verify_heap_slot_handler, rel);
778 6906 : run_command(free_slot, rel->sql);
779 : }
780 : else
781 : {
782 8658 : if (opts.verbose)
783 : {
784 0 : if (opts.show_progress && progress_since_last_stderr)
785 0 : fprintf(stderr, "\n");
786 :
787 0 : pg_log_info("checking btree index \"%s.%s.%s\"",
788 : rel->datinfo->datname, rel->nspname, rel->relname);
789 0 : progress_since_last_stderr = false;
790 : }
791 8658 : prepare_btree_command(&sql, rel, free_slot->connection);
792 8658 : rel->sql = pstrdup(sql.data); /* pg_free'd after command */
793 8658 : ParallelSlotSetHandler(free_slot, verify_btree_slot_handler, rel);
794 8658 : run_command(free_slot, rel->sql);
795 : }
796 : }
797 64 : termPQExpBuffer(&sql);
798 :
799 64 : if (!failed)
800 : {
801 :
802 : /*
803 : * Wait for all slots to complete, or for one to indicate that an
804 : * error occurred. Like above, we rely on the handler emitting the
805 : * necessary error messages.
806 : */
807 64 : if (sa && !ParallelSlotsWaitCompletion(sa))
808 0 : failed = true;
809 :
810 64 : progress_report(reltotal, relprogress, pagestotal, pageschecked, NULL, true, true);
811 : }
812 :
813 64 : if (sa)
814 : {
815 64 : ParallelSlotsTerminate(sa);
816 64 : FREE_AND_SET_NULL(sa);
817 : }
818 :
819 64 : if (failed)
820 0 : exit(1);
821 :
822 64 : if (!all_checks_pass)
823 28 : exit(2);
824 : }
825 :
826 : /*
827 : * prepare_heap_command
828 : *
829 : * Creates a SQL command for running amcheck checking on the given heap
830 : * relation. The command is phrased as a SQL query, with column order and
831 : * names matching the expectations of verify_heap_slot_handler, which will
832 : * receive and handle each row returned from the verify_heapam() function.
833 : *
834 : * The constructed SQL command will silently skip temporary tables, as checking
835 : * them would needlessly draw errors from the underlying amcheck function.
836 : *
837 : * sql: buffer into which the heap table checking command will be written
838 : * rel: relation information for the heap table to be checked
839 : * conn: the connection to be used, for string escaping purposes
840 : */
841 : static void
842 6906 : prepare_heap_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
843 : {
844 6906 : resetPQExpBuffer(sql);
845 13812 : appendPQExpBuffer(sql,
846 : "SELECT v.blkno, v.offnum, v.attnum, v.msg "
847 : "FROM pg_catalog.pg_class c, %s.verify_heapam("
848 : "\nrelation := c.oid, on_error_stop := %s, check_toast := %s, skip := '%s'",
849 6906 : rel->datinfo->amcheck_schema,
850 6906 : opts.on_error_stop ? "true" : "false",
851 6906 : opts.reconcile_toast ? "true" : "false",
852 : opts.skip);
853 :
854 6906 : if (opts.startblock >= 0)
855 0 : appendPQExpBuffer(sql, ", startblock := " INT64_FORMAT, opts.startblock);
856 6906 : if (opts.endblock >= 0)
857 0 : appendPQExpBuffer(sql, ", endblock := " INT64_FORMAT, opts.endblock);
858 :
859 6906 : appendPQExpBuffer(sql,
860 : "\n) v WHERE c.oid = %u "
861 : "AND c.relpersistence != " CppAsString2(RELPERSISTENCE_TEMP),
862 : rel->reloid);
863 6906 : }
864 :
865 : /*
866 : * prepare_btree_command
867 : *
868 : * Creates a SQL command for running amcheck checking on the given btree index
869 : * relation. The command does not select any columns, as btree checking
870 : * functions do not return any, but rather return corruption information by
871 : * raising errors, which verify_btree_slot_handler expects.
872 : *
873 : * The constructed SQL command will silently skip temporary indexes, and
874 : * indexes being reindexed concurrently, as checking them would needlessly draw
875 : * errors from the underlying amcheck functions.
876 : *
877 : * sql: buffer into which the heap table checking command will be written
878 : * rel: relation information for the index to be checked
879 : * conn: the connection to be used, for string escaping purposes
880 : */
881 : static void
882 8658 : prepare_btree_command(PQExpBuffer sql, RelationInfo *rel, PGconn *conn)
883 : {
884 8658 : resetPQExpBuffer(sql);
885 :
886 8658 : if (opts.parent_check)
887 288 : appendPQExpBuffer(sql,
888 : "SELECT %s.bt_index_parent_check("
889 : "index := c.oid, heapallindexed := %s, rootdescend := %s "
890 : "%s)"
891 : "\nFROM pg_catalog.pg_class c, pg_catalog.pg_index i "
892 : "WHERE c.oid = %u "
893 : "AND c.oid = i.indexrelid "
894 : "AND c.relpersistence != " CppAsString2(RELPERSISTENCE_TEMP) " "
895 : "AND i.indisready AND i.indisvalid AND i.indislive",
896 96 : rel->datinfo->amcheck_schema,
897 96 : (opts.heapallindexed ? "true" : "false"),
898 96 : (opts.rootdescend ? "true" : "false"),
899 96 : (rel->datinfo->is_checkunique ? ", checkunique := true" : ""),
900 : rel->reloid);
901 : else
902 17124 : appendPQExpBuffer(sql,
903 : "SELECT %s.bt_index_check("
904 : "index := c.oid, heapallindexed := %s "
905 : "%s)"
906 : "\nFROM pg_catalog.pg_class c, pg_catalog.pg_index i "
907 : "WHERE c.oid = %u "
908 : "AND c.oid = i.indexrelid "
909 : "AND c.relpersistence != " CppAsString2(RELPERSISTENCE_TEMP) " "
910 : "AND i.indisready AND i.indisvalid AND i.indislive",
911 8562 : rel->datinfo->amcheck_schema,
912 8562 : (opts.heapallindexed ? "true" : "false"),
913 8562 : (rel->datinfo->is_checkunique ? ", checkunique := true" : ""),
914 : rel->reloid);
915 8658 : }
916 :
917 : /*
918 : * run_command
919 : *
920 : * Sends a command to the server without waiting for the command to complete.
921 : * Logs an error if the command cannot be sent, but otherwise any errors are
922 : * expected to be handled by a ParallelSlotHandler.
923 : *
924 : * If reconnecting to the database is necessary, the cparams argument may be
925 : * modified.
926 : *
927 : * slot: slot with connection to the server we should use for the command
928 : * sql: query to send
929 : */
930 : static void
931 15564 : run_command(ParallelSlot *slot, const char *sql)
932 : {
933 15564 : if (opts.echo)
934 0 : printf("%s\n", sql);
935 :
936 15564 : if (PQsendQuery(slot->connection, sql) == 0)
937 : {
938 0 : pg_log_error("error sending command to database \"%s\": %s",
939 : PQdb(slot->connection),
940 : PQerrorMessage(slot->connection));
941 0 : pg_log_error_detail("Command was: %s", sql);
942 0 : exit(1);
943 : }
944 15564 : }
945 :
946 : /*
947 : * should_processing_continue
948 : *
949 : * Checks a query result returned from a query (presumably issued on a slot's
950 : * connection) to determine if parallel slots should continue issuing further
951 : * commands.
952 : *
953 : * Note: Heap relation corruption is reported by verify_heapam() via the result
954 : * set, rather than an ERROR, but running verify_heapam() on a corrupted heap
955 : * table may still result in an error being returned from the server due to
956 : * missing relation files, bad checksums, etc. The btree corruption checking
957 : * functions always use errors to communicate corruption messages. We can't
958 : * just abort processing because we got a mere ERROR.
959 : *
960 : * res: result from an executed sql query
961 : */
962 : static bool
963 15564 : should_processing_continue(PGresult *res)
964 : {
965 : const char *severity;
966 :
967 15564 : switch (PQresultStatus(res))
968 : {
969 : /* These are expected and ok */
970 15456 : case PGRES_COMMAND_OK:
971 : case PGRES_TUPLES_OK:
972 : case PGRES_NONFATAL_ERROR:
973 15456 : break;
974 :
975 : /* This is expected but requires closer scrutiny */
976 108 : case PGRES_FATAL_ERROR:
977 108 : severity = PQresultErrorField(res, PG_DIAG_SEVERITY_NONLOCALIZED);
978 108 : if (severity == NULL)
979 0 : return false; /* libpq failure, probably lost connection */
980 108 : if (strcmp(severity, "FATAL") == 0)
981 0 : return false;
982 108 : if (strcmp(severity, "PANIC") == 0)
983 0 : return false;
984 108 : break;
985 :
986 : /* These are unexpected */
987 0 : case PGRES_BAD_RESPONSE:
988 : case PGRES_EMPTY_QUERY:
989 : case PGRES_COPY_OUT:
990 : case PGRES_COPY_IN:
991 : case PGRES_COPY_BOTH:
992 : case PGRES_SINGLE_TUPLE:
993 : case PGRES_PIPELINE_SYNC:
994 : case PGRES_PIPELINE_ABORTED:
995 : case PGRES_TUPLES_CHUNK:
996 0 : return false;
997 : }
998 15564 : return true;
999 : }
1000 :
1001 : /*
1002 : * Returns a copy of the argument string with all lines indented four spaces.
1003 : *
1004 : * The caller should pg_free the result when finished with it.
1005 : */
1006 : static char *
1007 108 : indent_lines(const char *str)
1008 : {
1009 : PQExpBufferData buf;
1010 : const char *c;
1011 : char *result;
1012 :
1013 108 : initPQExpBuffer(&buf);
1014 108 : appendPQExpBufferStr(&buf, " ");
1015 8596 : for (c = str; *c; c++)
1016 : {
1017 8488 : appendPQExpBufferChar(&buf, *c);
1018 8488 : if (c[0] == '\n' && c[1] != '\0')
1019 4 : appendPQExpBufferStr(&buf, " ");
1020 : }
1021 108 : result = pstrdup(buf.data);
1022 108 : termPQExpBuffer(&buf);
1023 :
1024 108 : return result;
1025 : }
1026 :
1027 : /*
1028 : * verify_heap_slot_handler
1029 : *
1030 : * ParallelSlotHandler that receives results from a heap table checking command
1031 : * created by prepare_heap_command and outputs the results for the user.
1032 : *
1033 : * res: result from an executed sql query
1034 : * conn: connection on which the sql query was executed
1035 : * context: the sql query being handled, as a cstring
1036 : */
1037 : static bool
1038 6906 : verify_heap_slot_handler(PGresult *res, PGconn *conn, void *context)
1039 : {
1040 6906 : RelationInfo *rel = (RelationInfo *) context;
1041 :
1042 6906 : if (PQresultStatus(res) == PGRES_TUPLES_OK)
1043 : {
1044 : int i;
1045 6866 : int ntups = PQntuples(res);
1046 :
1047 6866 : if (ntups > 0)
1048 18 : all_checks_pass = false;
1049 :
1050 6966 : for (i = 0; i < ntups; i++)
1051 : {
1052 : const char *msg;
1053 :
1054 : /* The message string should never be null, but check */
1055 100 : if (PQgetisnull(res, i, 3))
1056 0 : msg = "NO MESSAGE";
1057 : else
1058 100 : msg = PQgetvalue(res, i, 3);
1059 :
1060 100 : if (!PQgetisnull(res, i, 2))
1061 4 : printf(_("heap table \"%s.%s.%s\", block %s, offset %s, attribute %s:\n"),
1062 : rel->datinfo->datname, rel->nspname, rel->relname,
1063 : PQgetvalue(res, i, 0), /* blkno */
1064 : PQgetvalue(res, i, 1), /* offnum */
1065 : PQgetvalue(res, i, 2)); /* attnum */
1066 :
1067 96 : else if (!PQgetisnull(res, i, 1))
1068 96 : printf(_("heap table \"%s.%s.%s\", block %s, offset %s:\n"),
1069 : rel->datinfo->datname, rel->nspname, rel->relname,
1070 : PQgetvalue(res, i, 0), /* blkno */
1071 : PQgetvalue(res, i, 1)); /* offnum */
1072 :
1073 0 : else if (!PQgetisnull(res, i, 0))
1074 0 : printf(_("heap table \"%s.%s.%s\", block %s:\n"),
1075 : rel->datinfo->datname, rel->nspname, rel->relname,
1076 : PQgetvalue(res, i, 0)); /* blkno */
1077 :
1078 : else
1079 0 : printf(_("heap table \"%s.%s.%s\":\n"),
1080 : rel->datinfo->datname, rel->nspname, rel->relname);
1081 :
1082 100 : printf(" %s\n", msg);
1083 : }
1084 : }
1085 40 : else if (PQresultStatus(res) != PGRES_TUPLES_OK)
1086 : {
1087 40 : char *msg = indent_lines(PQerrorMessage(conn));
1088 :
1089 40 : all_checks_pass = false;
1090 40 : printf(_("heap table \"%s.%s.%s\":\n"),
1091 : rel->datinfo->datname, rel->nspname, rel->relname);
1092 40 : printf("%s", msg);
1093 40 : if (opts.verbose)
1094 0 : printf(_("query was: %s\n"), rel->sql);
1095 40 : FREE_AND_SET_NULL(msg);
1096 : }
1097 :
1098 6906 : FREE_AND_SET_NULL(rel->sql);
1099 6906 : FREE_AND_SET_NULL(rel->nspname);
1100 6906 : FREE_AND_SET_NULL(rel->relname);
1101 :
1102 6906 : return should_processing_continue(res);
1103 : }
1104 :
1105 : /*
1106 : * verify_btree_slot_handler
1107 : *
1108 : * ParallelSlotHandler that receives results from a btree checking command
1109 : * created by prepare_btree_command and outputs them for the user. The results
1110 : * from the btree checking command is assumed to be empty, but when the results
1111 : * are an error code, the useful information about the corruption is expected
1112 : * in the connection's error message.
1113 : *
1114 : * res: result from an executed sql query
1115 : * conn: connection on which the sql query was executed
1116 : * context: unused
1117 : */
1118 : static bool
1119 8658 : verify_btree_slot_handler(PGresult *res, PGconn *conn, void *context)
1120 : {
1121 8658 : RelationInfo *rel = (RelationInfo *) context;
1122 :
1123 8658 : if (PQresultStatus(res) == PGRES_TUPLES_OK)
1124 : {
1125 8590 : int ntups = PQntuples(res);
1126 :
1127 8590 : if (ntups > 1)
1128 : {
1129 : /*
1130 : * We expect the btree checking functions to return one void row
1131 : * each, or zero rows if the check was skipped due to the object
1132 : * being in the wrong state to be checked, so we should output
1133 : * some sort of warning if we get anything more, not because it
1134 : * indicates corruption, but because it suggests a mismatch
1135 : * between amcheck and pg_amcheck versions.
1136 : *
1137 : * In conjunction with --progress, anything written to stderr at
1138 : * this time would present strangely to the user without an extra
1139 : * newline, so we print one. If we were multithreaded, we'd have
1140 : * to avoid splitting this across multiple calls, but we're in an
1141 : * event loop, so it doesn't matter.
1142 : */
1143 0 : if (opts.show_progress && progress_since_last_stderr)
1144 0 : fprintf(stderr, "\n");
1145 0 : pg_log_warning("btree index \"%s.%s.%s\": btree checking function returned unexpected number of rows: %d",
1146 : rel->datinfo->datname, rel->nspname, rel->relname, ntups);
1147 0 : if (opts.verbose)
1148 0 : pg_log_warning_detail("Query was: %s", rel->sql);
1149 0 : pg_log_warning_hint("Are %s's and amcheck's versions compatible?",
1150 : progname);
1151 0 : progress_since_last_stderr = false;
1152 : }
1153 : }
1154 : else
1155 : {
1156 68 : char *msg = indent_lines(PQerrorMessage(conn));
1157 :
1158 68 : all_checks_pass = false;
1159 68 : printf(_("btree index \"%s.%s.%s\":\n"),
1160 : rel->datinfo->datname, rel->nspname, rel->relname);
1161 68 : printf("%s", msg);
1162 68 : if (opts.verbose)
1163 0 : printf(_("query was: %s\n"), rel->sql);
1164 68 : FREE_AND_SET_NULL(msg);
1165 : }
1166 :
1167 8658 : FREE_AND_SET_NULL(rel->sql);
1168 8658 : FREE_AND_SET_NULL(rel->nspname);
1169 8658 : FREE_AND_SET_NULL(rel->relname);
1170 :
1171 8658 : return should_processing_continue(res);
1172 : }
1173 :
1174 : /*
1175 : * help
1176 : *
1177 : * Prints help page for the program
1178 : *
1179 : * progname: the name of the executed program, such as "pg_amcheck"
1180 : */
1181 : static void
1182 2 : help(const char *progname)
1183 : {
1184 2 : printf(_("%s checks objects in a PostgreSQL database for corruption.\n\n"), progname);
1185 2 : printf(_("Usage:\n"));
1186 2 : printf(_(" %s [OPTION]... [DBNAME]\n"), progname);
1187 2 : printf(_("\nTarget options:\n"));
1188 2 : printf(_(" -a, --all check all databases\n"));
1189 2 : printf(_(" -d, --database=PATTERN check matching database(s)\n"));
1190 2 : printf(_(" -D, --exclude-database=PATTERN do NOT check matching database(s)\n"));
1191 2 : printf(_(" -i, --index=PATTERN check matching index(es)\n"));
1192 2 : printf(_(" -I, --exclude-index=PATTERN do NOT check matching index(es)\n"));
1193 2 : printf(_(" -r, --relation=PATTERN check matching relation(s)\n"));
1194 2 : printf(_(" -R, --exclude-relation=PATTERN do NOT check matching relation(s)\n"));
1195 2 : printf(_(" -s, --schema=PATTERN check matching schema(s)\n"));
1196 2 : printf(_(" -S, --exclude-schema=PATTERN do NOT check matching schema(s)\n"));
1197 2 : printf(_(" -t, --table=PATTERN check matching table(s)\n"));
1198 2 : printf(_(" -T, --exclude-table=PATTERN do NOT check matching table(s)\n"));
1199 2 : printf(_(" --no-dependent-indexes do NOT expand list of relations to include indexes\n"));
1200 2 : printf(_(" --no-dependent-toast do NOT expand list of relations to include TOAST tables\n"));
1201 2 : printf(_(" --no-strict-names do NOT require patterns to match objects\n"));
1202 2 : printf(_("\nTable checking options:\n"));
1203 2 : printf(_(" --exclude-toast-pointers do NOT follow relation TOAST pointers\n"));
1204 2 : printf(_(" --on-error-stop stop checking at end of first corrupt page\n"));
1205 2 : printf(_(" --skip=OPTION do NOT check \"all-frozen\" or \"all-visible\" blocks\n"));
1206 2 : printf(_(" --startblock=BLOCK begin checking table(s) at the given block number\n"));
1207 2 : printf(_(" --endblock=BLOCK check table(s) only up to the given block number\n"));
1208 2 : printf(_("\nB-tree index checking options:\n"));
1209 2 : printf(_(" --checkunique check unique constraint if index is unique\n"));
1210 2 : printf(_(" --heapallindexed check that all heap tuples are found within indexes\n"));
1211 2 : printf(_(" --parent-check check index parent/child relationships\n"));
1212 2 : printf(_(" --rootdescend search from root page to refind tuples\n"));
1213 2 : printf(_("\nConnection options:\n"));
1214 2 : printf(_(" -h, --host=HOSTNAME database server host or socket directory\n"));
1215 2 : printf(_(" -p, --port=PORT database server port\n"));
1216 2 : printf(_(" -U, --username=USERNAME user name to connect as\n"));
1217 2 : printf(_(" -w, --no-password never prompt for password\n"));
1218 2 : printf(_(" -W, --password force password prompt\n"));
1219 2 : printf(_(" --maintenance-db=DBNAME alternate maintenance database\n"));
1220 2 : printf(_("\nOther options:\n"));
1221 2 : printf(_(" -e, --echo show the commands being sent to the server\n"));
1222 2 : printf(_(" -j, --jobs=NUM use this many concurrent connections to the server\n"));
1223 2 : printf(_(" -P, --progress show progress information\n"));
1224 2 : printf(_(" -v, --verbose write a lot of output\n"));
1225 2 : printf(_(" -V, --version output version information, then exit\n"));
1226 2 : printf(_(" --install-missing install missing extensions\n"));
1227 2 : printf(_(" -?, --help show this help, then exit\n"));
1228 :
1229 2 : printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
1230 2 : printf(_("%s home page: <%s>\n"), PACKAGE_NAME, PACKAGE_URL);
1231 2 : }
1232 :
1233 : /*
1234 : * Print a progress report based on the global variables.
1235 : *
1236 : * Progress report is written at maximum once per second, unless the force
1237 : * parameter is set to true.
1238 : *
1239 : * If finished is set to true, this is the last progress report. The cursor
1240 : * is moved to the next line.
1241 : */
1242 : static void
1243 15692 : progress_report(uint64 relations_total, uint64 relations_checked,
1244 : uint64 relpages_total, uint64 relpages_checked,
1245 : const char *datname, bool force, bool finished)
1246 : {
1247 15692 : int percent_rel = 0;
1248 15692 : int percent_pages = 0;
1249 : char checked_rel[32];
1250 : char total_rel[32];
1251 : char checked_pages[32];
1252 : char total_pages[32];
1253 : pg_time_t now;
1254 :
1255 15692 : if (!opts.show_progress)
1256 15692 : return;
1257 :
1258 0 : now = time(NULL);
1259 0 : if (now == last_progress_report && !force && !finished)
1260 0 : return; /* Max once per second */
1261 :
1262 0 : last_progress_report = now;
1263 0 : if (relations_total)
1264 0 : percent_rel = (int) (relations_checked * 100 / relations_total);
1265 0 : if (relpages_total)
1266 0 : percent_pages = (int) (relpages_checked * 100 / relpages_total);
1267 :
1268 0 : snprintf(checked_rel, sizeof(checked_rel), UINT64_FORMAT, relations_checked);
1269 0 : snprintf(total_rel, sizeof(total_rel), UINT64_FORMAT, relations_total);
1270 0 : snprintf(checked_pages, sizeof(checked_pages), UINT64_FORMAT, relpages_checked);
1271 0 : snprintf(total_pages, sizeof(total_pages), UINT64_FORMAT, relpages_total);
1272 :
1273 : #define VERBOSE_DATNAME_LENGTH 35
1274 0 : if (opts.verbose)
1275 : {
1276 0 : if (!datname)
1277 :
1278 : /*
1279 : * No datname given, so clear the status line (used for first and
1280 : * last call)
1281 : */
1282 0 : fprintf(stderr,
1283 0 : _("%*s/%s relations (%d%%), %*s/%s pages (%d%%) %*s"),
1284 0 : (int) strlen(total_rel),
1285 : checked_rel, total_rel, percent_rel,
1286 0 : (int) strlen(total_pages),
1287 : checked_pages, total_pages, percent_pages,
1288 : VERBOSE_DATNAME_LENGTH + 2, "");
1289 : else
1290 : {
1291 0 : bool truncate = (strlen(datname) > VERBOSE_DATNAME_LENGTH);
1292 :
1293 0 : fprintf(stderr,
1294 0 : _("%*s/%s relations (%d%%), %*s/%s pages (%d%%) (%s%-*.*s)"),
1295 0 : (int) strlen(total_rel),
1296 : checked_rel, total_rel, percent_rel,
1297 0 : (int) strlen(total_pages),
1298 : checked_pages, total_pages, percent_pages,
1299 : /* Prefix with "..." if we do leading truncation */
1300 : truncate ? "..." : "",
1301 : truncate ? VERBOSE_DATNAME_LENGTH - 3 : VERBOSE_DATNAME_LENGTH,
1302 : truncate ? VERBOSE_DATNAME_LENGTH - 3 : VERBOSE_DATNAME_LENGTH,
1303 : /* Truncate datname at beginning if it's too long */
1304 0 : truncate ? datname + strlen(datname) - VERBOSE_DATNAME_LENGTH + 3 : datname);
1305 : }
1306 : }
1307 : else
1308 0 : fprintf(stderr,
1309 0 : _("%*s/%s relations (%d%%), %*s/%s pages (%d%%)"),
1310 0 : (int) strlen(total_rel),
1311 : checked_rel, total_rel, percent_rel,
1312 0 : (int) strlen(total_pages),
1313 : checked_pages, total_pages, percent_pages);
1314 :
1315 : /*
1316 : * Stay on the same line if reporting to a terminal and we're not done
1317 : * yet.
1318 : */
1319 0 : if (!finished && isatty(fileno(stderr)))
1320 : {
1321 0 : fputc('\r', stderr);
1322 0 : progress_since_last_stderr = true;
1323 : }
1324 : else
1325 0 : fputc('\n', stderr);
1326 : }
1327 :
1328 : /*
1329 : * Extend the pattern info array to hold one additional initialized pattern
1330 : * info entry.
1331 : *
1332 : * Returns a pointer to the new entry.
1333 : */
1334 : static PatternInfo *
1335 204 : extend_pattern_info_array(PatternInfoArray *pia)
1336 : {
1337 : PatternInfo *result;
1338 :
1339 204 : pia->len++;
1340 204 : pia->data = (PatternInfo *) pg_realloc(pia->data, pia->len * sizeof(PatternInfo));
1341 204 : result = &pia->data[pia->len - 1];
1342 204 : memset(result, 0, sizeof(*result));
1343 :
1344 204 : return result;
1345 : }
1346 :
1347 : /*
1348 : * append_database_pattern
1349 : *
1350 : * Adds the given pattern interpreted as a database name pattern.
1351 : *
1352 : * pia: the pattern info array to be appended
1353 : * pattern: the database name pattern
1354 : * encoding: client encoding for parsing the pattern
1355 : */
1356 : static void
1357 68 : append_database_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1358 : {
1359 : PQExpBufferData buf;
1360 : int dotcnt;
1361 68 : PatternInfo *info = extend_pattern_info_array(pia);
1362 :
1363 68 : initPQExpBuffer(&buf);
1364 68 : patternToSQLRegex(encoding, NULL, NULL, &buf, pattern, false, false,
1365 : &dotcnt);
1366 68 : if (dotcnt > 0)
1367 : {
1368 6 : pg_log_error("improper qualified name (too many dotted names): %s", pattern);
1369 6 : exit(2);
1370 : }
1371 62 : info->pattern = pattern;
1372 62 : info->db_regex = pstrdup(buf.data);
1373 :
1374 62 : termPQExpBuffer(&buf);
1375 62 : }
1376 :
1377 : /*
1378 : * append_schema_pattern
1379 : *
1380 : * Adds the given pattern interpreted as a schema name pattern.
1381 : *
1382 : * pia: the pattern info array to be appended
1383 : * pattern: the schema name pattern
1384 : * encoding: client encoding for parsing the pattern
1385 : */
1386 : static void
1387 58 : append_schema_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1388 : {
1389 : PQExpBufferData dbbuf;
1390 : PQExpBufferData nspbuf;
1391 : int dotcnt;
1392 58 : PatternInfo *info = extend_pattern_info_array(pia);
1393 :
1394 58 : initPQExpBuffer(&dbbuf);
1395 58 : initPQExpBuffer(&nspbuf);
1396 :
1397 58 : patternToSQLRegex(encoding, NULL, &dbbuf, &nspbuf, pattern, false, false,
1398 : &dotcnt);
1399 58 : if (dotcnt > 1)
1400 : {
1401 6 : pg_log_error("improper qualified name (too many dotted names): %s", pattern);
1402 6 : exit(2);
1403 : }
1404 52 : info->pattern = pattern;
1405 52 : if (dbbuf.data[0])
1406 : {
1407 0 : opts.dbpattern = true;
1408 0 : info->db_regex = pstrdup(dbbuf.data);
1409 : }
1410 52 : if (nspbuf.data[0])
1411 52 : info->nsp_regex = pstrdup(nspbuf.data);
1412 :
1413 52 : termPQExpBuffer(&dbbuf);
1414 52 : termPQExpBuffer(&nspbuf);
1415 52 : }
1416 :
1417 : /*
1418 : * append_relation_pattern_helper
1419 : *
1420 : * Adds to a list the given pattern interpreted as a relation pattern.
1421 : *
1422 : * pia: the pattern info array to be appended
1423 : * pattern: the relation name pattern
1424 : * encoding: client encoding for parsing the pattern
1425 : * heap_only: whether the pattern should only be matched against heap tables
1426 : * btree_only: whether the pattern should only be matched against btree indexes
1427 : */
1428 : static void
1429 78 : append_relation_pattern_helper(PatternInfoArray *pia, const char *pattern,
1430 : int encoding, bool heap_only, bool btree_only)
1431 : {
1432 : PQExpBufferData dbbuf;
1433 : PQExpBufferData nspbuf;
1434 : PQExpBufferData relbuf;
1435 : int dotcnt;
1436 78 : PatternInfo *info = extend_pattern_info_array(pia);
1437 :
1438 78 : initPQExpBuffer(&dbbuf);
1439 78 : initPQExpBuffer(&nspbuf);
1440 78 : initPQExpBuffer(&relbuf);
1441 :
1442 78 : patternToSQLRegex(encoding, &dbbuf, &nspbuf, &relbuf, pattern, false,
1443 : false, &dotcnt);
1444 78 : if (dotcnt > 2)
1445 : {
1446 6 : pg_log_error("improper relation name (too many dotted names): %s", pattern);
1447 6 : exit(2);
1448 : }
1449 72 : info->pattern = pattern;
1450 72 : if (dbbuf.data[0])
1451 : {
1452 28 : opts.dbpattern = true;
1453 28 : info->db_regex = pstrdup(dbbuf.data);
1454 : }
1455 72 : if (nspbuf.data[0])
1456 40 : info->nsp_regex = pstrdup(nspbuf.data);
1457 72 : if (relbuf.data[0])
1458 72 : info->rel_regex = pstrdup(relbuf.data);
1459 :
1460 72 : termPQExpBuffer(&dbbuf);
1461 72 : termPQExpBuffer(&nspbuf);
1462 72 : termPQExpBuffer(&relbuf);
1463 :
1464 72 : info->heap_only = heap_only;
1465 72 : info->btree_only = btree_only;
1466 72 : }
1467 :
1468 : /*
1469 : * append_relation_pattern
1470 : *
1471 : * Adds the given pattern interpreted as a relation pattern, to be matched
1472 : * against both heap tables and btree indexes.
1473 : *
1474 : * pia: the pattern info array to be appended
1475 : * pattern: the relation name pattern
1476 : * encoding: client encoding for parsing the pattern
1477 : */
1478 : static void
1479 14 : append_relation_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1480 : {
1481 14 : append_relation_pattern_helper(pia, pattern, encoding, false, false);
1482 14 : }
1483 :
1484 : /*
1485 : * append_heap_pattern
1486 : *
1487 : * Adds the given pattern interpreted as a relation pattern, to be matched only
1488 : * against heap tables.
1489 : *
1490 : * pia: the pattern info array to be appended
1491 : * pattern: the relation name pattern
1492 : * encoding: client encoding for parsing the pattern
1493 : */
1494 : static void
1495 38 : append_heap_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1496 : {
1497 38 : append_relation_pattern_helper(pia, pattern, encoding, true, false);
1498 32 : }
1499 :
1500 : /*
1501 : * append_btree_pattern
1502 : *
1503 : * Adds the given pattern interpreted as a relation pattern, to be matched only
1504 : * against btree indexes.
1505 : *
1506 : * pia: the pattern info array to be appended
1507 : * pattern: the relation name pattern
1508 : * encoding: client encoding for parsing the pattern
1509 : */
1510 : static void
1511 26 : append_btree_pattern(PatternInfoArray *pia, const char *pattern, int encoding)
1512 : {
1513 26 : append_relation_pattern_helper(pia, pattern, encoding, false, true);
1514 26 : }
1515 :
1516 : /*
1517 : * append_db_pattern_cte
1518 : *
1519 : * Appends to the buffer the body of a Common Table Expression (CTE) containing
1520 : * the database portions filtered from the list of patterns expressed as two
1521 : * columns:
1522 : *
1523 : * pattern_id: the index of this pattern in pia->data[]
1524 : * rgx: the database regular expression parsed from the pattern
1525 : *
1526 : * Patterns without a database portion are skipped. Patterns with more than
1527 : * just a database portion are optionally skipped, depending on argument
1528 : * 'inclusive'.
1529 : *
1530 : * buf: the buffer to be appended
1531 : * pia: the array of patterns to be inserted into the CTE
1532 : * conn: the database connection
1533 : * inclusive: whether to include patterns with schema and/or relation parts
1534 : *
1535 : * Returns whether any database patterns were appended.
1536 : */
1537 : static bool
1538 130 : append_db_pattern_cte(PQExpBuffer buf, const PatternInfoArray *pia,
1539 : PGconn *conn, bool inclusive)
1540 : {
1541 : int pattern_id;
1542 : const char *comma;
1543 : bool have_values;
1544 :
1545 130 : comma = "";
1546 130 : have_values = false;
1547 302 : for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1548 : {
1549 172 : PatternInfo *info = &pia->data[pattern_id];
1550 :
1551 172 : if (info->db_regex != NULL &&
1552 0 : (inclusive || (info->nsp_regex == NULL && info->rel_regex == NULL)))
1553 : {
1554 90 : if (!have_values)
1555 34 : appendPQExpBufferStr(buf, "\nVALUES");
1556 90 : have_values = true;
1557 90 : appendPQExpBuffer(buf, "%s\n(%d, ", comma, pattern_id);
1558 90 : appendStringLiteralConn(buf, info->db_regex, conn);
1559 90 : appendPQExpBufferChar(buf, ')');
1560 90 : comma = ",";
1561 : }
1562 : }
1563 :
1564 130 : if (!have_values)
1565 96 : appendPQExpBufferStr(buf, "\nSELECT NULL, NULL, NULL WHERE false");
1566 :
1567 130 : return have_values;
1568 : }
1569 :
1570 : /*
1571 : * compile_database_list
1572 : *
1573 : * If any database patterns exist, or if --all was given, compiles a distinct
1574 : * list of databases to check using a SQL query based on the patterns plus the
1575 : * literal initial database name, if given. If no database patterns exist and
1576 : * --all was not given, the query is not necessary, and only the initial
1577 : * database name (if any) is added to the list.
1578 : *
1579 : * conn: connection to the initial database
1580 : * databases: the list onto which databases should be appended
1581 : * initial_dbname: an optional extra database name to include in the list
1582 : */
1583 : static void
1584 88 : compile_database_list(PGconn *conn, SimplePtrList *databases,
1585 : const char *initial_dbname)
1586 : {
1587 : PGresult *res;
1588 : PQExpBufferData sql;
1589 : int ntups;
1590 : int i;
1591 : bool fatal;
1592 :
1593 88 : if (initial_dbname)
1594 : {
1595 46 : DatabaseInfo *dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1596 :
1597 : /* This database is included. Add to list */
1598 46 : if (opts.verbose)
1599 0 : pg_log_info("including database \"%s\"", initial_dbname);
1600 :
1601 46 : dat->datname = pstrdup(initial_dbname);
1602 46 : simple_ptr_list_append(databases, dat);
1603 : }
1604 :
1605 88 : initPQExpBuffer(&sql);
1606 :
1607 : /* Append the include patterns CTE. */
1608 88 : appendPQExpBufferStr(&sql, "WITH include_raw (pattern_id, rgx) AS (");
1609 88 : if (!append_db_pattern_cte(&sql, &opts.include, conn, true) &&
1610 54 : !opts.alldb)
1611 : {
1612 : /*
1613 : * None of the inclusion patterns (if any) contain database portions,
1614 : * so there is no need to query the database to resolve database
1615 : * patterns.
1616 : *
1617 : * Since we're also not operating under --all, we don't need to query
1618 : * the exhaustive list of connectable databases, either.
1619 : */
1620 46 : termPQExpBuffer(&sql);
1621 46 : return;
1622 : }
1623 :
1624 : /* Append the exclude patterns CTE. */
1625 42 : appendPQExpBufferStr(&sql, "),\nexclude_raw (pattern_id, rgx) AS (");
1626 42 : append_db_pattern_cte(&sql, &opts.exclude, conn, false);
1627 42 : appendPQExpBufferStr(&sql, "),");
1628 :
1629 : /*
1630 : * Append the database CTE, which includes whether each database is
1631 : * connectable and also joins against exclude_raw to determine whether
1632 : * each database is excluded.
1633 : */
1634 42 : appendPQExpBufferStr(&sql,
1635 : "\ndatabase (datname) AS ("
1636 : "\nSELECT d.datname "
1637 : "FROM pg_catalog.pg_database d "
1638 : "LEFT OUTER JOIN exclude_raw e "
1639 : "ON d.datname ~ e.rgx "
1640 : "\nWHERE d.datallowconn AND datconnlimit != -2 "
1641 : "AND e.pattern_id IS NULL"
1642 : "),"
1643 :
1644 : /*
1645 : * Append the include_pat CTE, which joins the include_raw CTE against the
1646 : * databases CTE to determine if all the inclusion patterns had matches,
1647 : * and whether each matched pattern had the misfortune of only matching
1648 : * excluded or unconnectable databases.
1649 : */
1650 : "\ninclude_pat (pattern_id, checkable) AS ("
1651 : "\nSELECT i.pattern_id, "
1652 : "COUNT(*) FILTER ("
1653 : "WHERE d IS NOT NULL"
1654 : ") AS checkable"
1655 : "\nFROM include_raw i "
1656 : "LEFT OUTER JOIN database d "
1657 : "ON d.datname ~ i.rgx"
1658 : "\nGROUP BY i.pattern_id"
1659 : "),"
1660 :
1661 : /*
1662 : * Append the filtered_databases CTE, which selects from the database CTE
1663 : * optionally joined against the include_raw CTE to only select databases
1664 : * that match an inclusion pattern. This appears to duplicate what the
1665 : * include_pat CTE already did above, but here we want only databases, and
1666 : * there we wanted patterns.
1667 : */
1668 : "\nfiltered_databases (datname) AS ("
1669 : "\nSELECT DISTINCT d.datname "
1670 : "FROM database d");
1671 42 : if (!opts.alldb)
1672 34 : appendPQExpBufferStr(&sql,
1673 : " INNER JOIN include_raw i "
1674 : "ON d.datname ~ i.rgx");
1675 42 : appendPQExpBufferStr(&sql,
1676 : ")"
1677 :
1678 : /*
1679 : * Select the checkable databases and the unmatched inclusion patterns.
1680 : */
1681 : "\nSELECT pattern_id, datname FROM ("
1682 : "\nSELECT pattern_id, NULL::TEXT AS datname "
1683 : "FROM include_pat "
1684 : "WHERE checkable = 0 "
1685 : "UNION ALL"
1686 : "\nSELECT NULL, datname "
1687 : "FROM filtered_databases"
1688 : ") AS combined_records"
1689 : "\nORDER BY pattern_id NULLS LAST, datname");
1690 :
1691 42 : res = executeQuery(conn, sql.data, opts.echo);
1692 42 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
1693 : {
1694 0 : pg_log_error("query failed: %s", PQerrorMessage(conn));
1695 0 : pg_log_error_detail("Query was: %s", sql.data);
1696 0 : disconnectDatabase(conn);
1697 0 : exit(1);
1698 : }
1699 42 : termPQExpBuffer(&sql);
1700 :
1701 42 : ntups = PQntuples(res);
1702 148 : for (fatal = false, i = 0; i < ntups; i++)
1703 : {
1704 106 : int pattern_id = -1;
1705 106 : const char *datname = NULL;
1706 :
1707 106 : if (!PQgetisnull(res, i, 0))
1708 26 : pattern_id = atoi(PQgetvalue(res, i, 0));
1709 106 : if (!PQgetisnull(res, i, 1))
1710 80 : datname = PQgetvalue(res, i, 1);
1711 :
1712 106 : if (pattern_id >= 0)
1713 : {
1714 : /*
1715 : * Current record pertains to an inclusion pattern that matched no
1716 : * checkable databases.
1717 : */
1718 26 : fatal = opts.strict_names;
1719 26 : if (pattern_id >= opts.include.len)
1720 0 : pg_fatal("internal error: received unexpected database pattern_id %d",
1721 : pattern_id);
1722 26 : log_no_match("no connectable databases to check matching \"%s\"",
1723 : opts.include.data[pattern_id].pattern);
1724 : }
1725 : else
1726 : {
1727 : DatabaseInfo *dat;
1728 :
1729 : /* Current record pertains to a database */
1730 : Assert(datname != NULL);
1731 :
1732 : /* Avoid entering a duplicate entry matching the initial_dbname */
1733 80 : if (initial_dbname != NULL && strcmp(initial_dbname, datname) == 0)
1734 0 : continue;
1735 :
1736 : /* This database is included. Add to list */
1737 80 : if (opts.verbose)
1738 0 : pg_log_info("including database \"%s\"", datname);
1739 :
1740 80 : dat = (DatabaseInfo *) pg_malloc0(sizeof(DatabaseInfo));
1741 80 : dat->datname = pstrdup(datname);
1742 80 : simple_ptr_list_append(databases, dat);
1743 : }
1744 : }
1745 42 : PQclear(res);
1746 :
1747 42 : if (fatal)
1748 : {
1749 14 : if (conn != NULL)
1750 14 : disconnectDatabase(conn);
1751 14 : exit(1);
1752 : }
1753 : }
1754 :
1755 : /*
1756 : * append_rel_pattern_raw_cte
1757 : *
1758 : * Appends to the buffer the body of a Common Table Expression (CTE) containing
1759 : * the given patterns as six columns:
1760 : *
1761 : * pattern_id: the index of this pattern in pia->data[]
1762 : * db_regex: the database regexp parsed from the pattern, or NULL if the
1763 : * pattern had no database part
1764 : * nsp_regex: the namespace regexp parsed from the pattern, or NULL if the
1765 : * pattern had no namespace part
1766 : * rel_regex: the relname regexp parsed from the pattern, or NULL if the
1767 : * pattern had no relname part
1768 : * heap_only: true if the pattern applies only to heap tables (not indexes)
1769 : * btree_only: true if the pattern applies only to btree indexes (not tables)
1770 : *
1771 : * buf: the buffer to be appended
1772 : * patterns: the array of patterns to be inserted into the CTE
1773 : * conn: the database connection
1774 : */
1775 : static void
1776 62 : append_rel_pattern_raw_cte(PQExpBuffer buf, const PatternInfoArray *pia,
1777 : PGconn *conn)
1778 : {
1779 : int pattern_id;
1780 : const char *comma;
1781 : bool have_values;
1782 :
1783 62 : comma = "";
1784 62 : have_values = false;
1785 200 : for (pattern_id = 0; pattern_id < pia->len; pattern_id++)
1786 : {
1787 138 : PatternInfo *info = &pia->data[pattern_id];
1788 :
1789 138 : if (!have_values)
1790 62 : appendPQExpBufferStr(buf, "\nVALUES");
1791 138 : have_values = true;
1792 138 : appendPQExpBuffer(buf, "%s\n(%d::INTEGER, ", comma, pattern_id);
1793 138 : if (info->db_regex == NULL)
1794 106 : appendPQExpBufferStr(buf, "NULL");
1795 : else
1796 32 : appendStringLiteralConn(buf, info->db_regex, conn);
1797 138 : appendPQExpBufferStr(buf, "::TEXT, ");
1798 138 : if (info->nsp_regex == NULL)
1799 46 : appendPQExpBufferStr(buf, "NULL");
1800 : else
1801 92 : appendStringLiteralConn(buf, info->nsp_regex, conn);
1802 138 : appendPQExpBufferStr(buf, "::TEXT, ");
1803 138 : if (info->rel_regex == NULL)
1804 68 : appendPQExpBufferStr(buf, "NULL");
1805 : else
1806 70 : appendStringLiteralConn(buf, info->rel_regex, conn);
1807 138 : if (info->heap_only)
1808 26 : appendPQExpBufferStr(buf, "::TEXT, true::BOOLEAN");
1809 : else
1810 112 : appendPQExpBufferStr(buf, "::TEXT, false::BOOLEAN");
1811 138 : if (info->btree_only)
1812 30 : appendPQExpBufferStr(buf, ", true::BOOLEAN");
1813 : else
1814 108 : appendPQExpBufferStr(buf, ", false::BOOLEAN");
1815 138 : appendPQExpBufferChar(buf, ')');
1816 138 : comma = ",";
1817 : }
1818 :
1819 62 : if (!have_values)
1820 0 : appendPQExpBufferStr(buf,
1821 : "\nSELECT NULL::INTEGER, NULL::TEXT, NULL::TEXT, "
1822 : "NULL::TEXT, NULL::BOOLEAN, NULL::BOOLEAN "
1823 : "WHERE false");
1824 62 : }
1825 :
1826 : /*
1827 : * append_rel_pattern_filtered_cte
1828 : *
1829 : * Appends to the buffer a Common Table Expression (CTE) which selects
1830 : * all patterns from the named raw CTE, filtered by database. All patterns
1831 : * which have no database portion or whose database portion matches our
1832 : * connection's database name are selected, with other patterns excluded.
1833 : *
1834 : * The basic idea here is that if we're connected to database "foo" and we have
1835 : * patterns "foo.bar.baz", "alpha.beta" and "one.two.three", we only want to
1836 : * use the first two while processing relations in this database, as the third
1837 : * one is not relevant.
1838 : *
1839 : * buf: the buffer to be appended
1840 : * raw: the name of the CTE to select from
1841 : * filtered: the name of the CTE to create
1842 : * conn: the database connection
1843 : */
1844 : static void
1845 62 : append_rel_pattern_filtered_cte(PQExpBuffer buf, const char *raw,
1846 : const char *filtered, PGconn *conn)
1847 : {
1848 62 : appendPQExpBuffer(buf,
1849 : "\n%s (pattern_id, nsp_regex, rel_regex, heap_only, btree_only) AS ("
1850 : "\nSELECT pattern_id, nsp_regex, rel_regex, heap_only, btree_only "
1851 : "FROM %s r"
1852 : "\nWHERE (r.db_regex IS NULL "
1853 : "OR ",
1854 : filtered, raw);
1855 62 : appendStringLiteralConn(buf, PQdb(conn), conn);
1856 62 : appendPQExpBufferStr(buf, " ~ r.db_regex)");
1857 62 : appendPQExpBufferStr(buf,
1858 : " AND (r.nsp_regex IS NOT NULL"
1859 : " OR r.rel_regex IS NOT NULL)"
1860 : "),");
1861 62 : }
1862 :
1863 : /*
1864 : * compile_relation_list_one_db
1865 : *
1866 : * Compiles a list of relations to check within the currently connected
1867 : * database based on the user supplied options, sorted by descending size,
1868 : * and appends them to the given list of relations.
1869 : *
1870 : * The cells of the constructed list contain all information about the relation
1871 : * necessary to connect to the database and check the object, including which
1872 : * database to connect to, where contrib/amcheck is installed, and the Oid and
1873 : * type of object (heap table vs. btree index). Rather than duplicating the
1874 : * database details per relation, the relation structs use references to the
1875 : * same database object, provided by the caller.
1876 : *
1877 : * conn: connection to this next database, which should be the same as in 'dat'
1878 : * relations: list onto which the relations information should be appended
1879 : * dat: the database info struct for use by each relation
1880 : * pagecount: gets incremented by the number of blocks to check in all
1881 : * relations added
1882 : */
1883 : static void
1884 92 : compile_relation_list_one_db(PGconn *conn, SimplePtrList *relations,
1885 : const DatabaseInfo *dat,
1886 : uint64 *pagecount)
1887 : {
1888 : PGresult *res;
1889 : PQExpBufferData sql;
1890 : int ntups;
1891 : int i;
1892 :
1893 92 : initPQExpBuffer(&sql);
1894 92 : appendPQExpBufferStr(&sql, "WITH");
1895 :
1896 : /* Append CTEs for the relation inclusion patterns, if any */
1897 92 : if (!opts.allrel)
1898 : {
1899 42 : appendPQExpBufferStr(&sql,
1900 : " include_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1901 42 : append_rel_pattern_raw_cte(&sql, &opts.include, conn);
1902 42 : appendPQExpBufferStr(&sql, "\n),");
1903 42 : append_rel_pattern_filtered_cte(&sql, "include_raw", "include_pat", conn);
1904 : }
1905 :
1906 : /* Append CTEs for the relation exclusion patterns, if any */
1907 92 : if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1908 : {
1909 20 : appendPQExpBufferStr(&sql,
1910 : " exclude_raw (pattern_id, db_regex, nsp_regex, rel_regex, heap_only, btree_only) AS (");
1911 20 : append_rel_pattern_raw_cte(&sql, &opts.exclude, conn);
1912 20 : appendPQExpBufferStr(&sql, "\n),");
1913 20 : append_rel_pattern_filtered_cte(&sql, "exclude_raw", "exclude_pat", conn);
1914 : }
1915 :
1916 : /* Append the relation CTE. */
1917 92 : appendPQExpBufferStr(&sql,
1918 : " relation (pattern_id, oid, nspname, relname, reltoastrelid, relpages, is_heap, is_btree) AS ("
1919 : "\nSELECT DISTINCT ON (c.oid");
1920 92 : if (!opts.allrel)
1921 42 : appendPQExpBufferStr(&sql, ", ip.pattern_id) ip.pattern_id,");
1922 : else
1923 50 : appendPQExpBufferStr(&sql, ") NULL::INTEGER AS pattern_id,");
1924 92 : appendPQExpBuffer(&sql,
1925 : "\nc.oid, n.nspname, c.relname, c.reltoastrelid, c.relpages, "
1926 : "c.relam = %u AS is_heap, "
1927 : "c.relam = %u AS is_btree"
1928 : "\nFROM pg_catalog.pg_class c "
1929 : "INNER JOIN pg_catalog.pg_namespace n "
1930 : "ON c.relnamespace = n.oid",
1931 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
1932 92 : if (!opts.allrel)
1933 42 : appendPQExpBuffer(&sql,
1934 : "\nINNER JOIN include_pat ip"
1935 : "\nON (n.nspname ~ ip.nsp_regex OR ip.nsp_regex IS NULL)"
1936 : "\nAND (c.relname ~ ip.rel_regex OR ip.rel_regex IS NULL)"
1937 : "\nAND (c.relam = %u OR NOT ip.heap_only)"
1938 : "\nAND (c.relam = %u OR NOT ip.btree_only)",
1939 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
1940 92 : if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1941 20 : appendPQExpBuffer(&sql,
1942 : "\nLEFT OUTER JOIN exclude_pat ep"
1943 : "\nON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
1944 : "\nAND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
1945 : "\nAND (c.relam = %u OR NOT ep.heap_only OR ep.rel_regex IS NULL)"
1946 : "\nAND (c.relam = %u OR NOT ep.btree_only OR ep.rel_regex IS NULL)",
1947 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
1948 :
1949 : /*
1950 : * Exclude temporary tables and indexes, which must necessarily belong to
1951 : * other sessions. (We don't create any ourselves.) We must ultimately
1952 : * exclude indexes marked invalid or not ready, but we delay that decision
1953 : * until firing off the amcheck command, as the state of an index may
1954 : * change by then.
1955 : */
1956 92 : appendPQExpBufferStr(&sql, "\nWHERE c.relpersistence != "
1957 : CppAsString2(RELPERSISTENCE_TEMP));
1958 92 : if (opts.excludetbl || opts.excludeidx || opts.excludensp)
1959 20 : appendPQExpBufferStr(&sql, "\nAND ep.pattern_id IS NULL");
1960 :
1961 : /*
1962 : * We need to be careful not to break the --no-dependent-toast and
1963 : * --no-dependent-indexes options. By default, the btree indexes, toast
1964 : * tables, and toast table btree indexes associated with primary heap
1965 : * tables are included, using their own CTEs below. We implement the
1966 : * --exclude-* options by not creating those CTEs, but that's no use if
1967 : * we've already selected the toast and indexes here. On the other hand,
1968 : * we want inclusion patterns that match indexes or toast tables to be
1969 : * honored. So, if inclusion patterns were given, we want to select all
1970 : * tables, toast tables, or indexes that match the patterns. But if no
1971 : * inclusion patterns were given, and we're simply matching all relations,
1972 : * then we only want to match the primary tables here.
1973 : */
1974 92 : if (opts.allrel)
1975 50 : appendPQExpBuffer(&sql,
1976 : " AND c.relam = %u "
1977 : "AND c.relkind IN ("
1978 : CppAsString2(RELKIND_RELATION) ", "
1979 : CppAsString2(RELKIND_SEQUENCE) ", "
1980 : CppAsString2(RELKIND_MATVIEW) ", "
1981 : CppAsString2(RELKIND_TOASTVALUE) ") "
1982 : "AND c.relnamespace != %u",
1983 : HEAP_TABLE_AM_OID, PG_TOAST_NAMESPACE);
1984 : else
1985 42 : appendPQExpBuffer(&sql,
1986 : " AND c.relam IN (%u, %u)"
1987 : "AND c.relkind IN ("
1988 : CppAsString2(RELKIND_RELATION) ", "
1989 : CppAsString2(RELKIND_SEQUENCE) ", "
1990 : CppAsString2(RELKIND_MATVIEW) ", "
1991 : CppAsString2(RELKIND_TOASTVALUE) ", "
1992 : CppAsString2(RELKIND_INDEX) ") "
1993 : "AND ((c.relam = %u AND c.relkind IN ("
1994 : CppAsString2(RELKIND_RELATION) ", "
1995 : CppAsString2(RELKIND_SEQUENCE) ", "
1996 : CppAsString2(RELKIND_MATVIEW) ", "
1997 : CppAsString2(RELKIND_TOASTVALUE) ")) OR "
1998 : "(c.relam = %u AND c.relkind = "
1999 : CppAsString2(RELKIND_INDEX) "))",
2000 : HEAP_TABLE_AM_OID, BTREE_AM_OID,
2001 : HEAP_TABLE_AM_OID, BTREE_AM_OID);
2002 :
2003 92 : appendPQExpBufferStr(&sql,
2004 : "\nORDER BY c.oid)");
2005 :
2006 92 : if (!opts.no_toast_expansion)
2007 : {
2008 : /*
2009 : * Include a CTE for toast tables associated with primary heap tables
2010 : * selected above, filtering by exclusion patterns (if any) that match
2011 : * toast table names.
2012 : */
2013 90 : appendPQExpBufferStr(&sql,
2014 : ", toast (oid, nspname, relname, relpages) AS ("
2015 : "\nSELECT t.oid, 'pg_toast', t.relname, t.relpages"
2016 : "\nFROM pg_catalog.pg_class t "
2017 : "INNER JOIN relation r "
2018 : "ON r.reltoastrelid = t.oid");
2019 90 : if (opts.excludetbl || opts.excludensp)
2020 18 : appendPQExpBufferStr(&sql,
2021 : "\nLEFT OUTER JOIN exclude_pat ep"
2022 : "\nON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL)"
2023 : "\nAND (t.relname ~ ep.rel_regex OR ep.rel_regex IS NULL)"
2024 : "\nAND ep.heap_only"
2025 : "\nWHERE ep.pattern_id IS NULL"
2026 : "\nAND t.relpersistence != " CppAsString2(RELPERSISTENCE_TEMP));
2027 90 : appendPQExpBufferStr(&sql,
2028 : "\n)");
2029 : }
2030 92 : if (!opts.no_btree_expansion)
2031 : {
2032 : /*
2033 : * Include a CTE for btree indexes associated with primary heap tables
2034 : * selected above, filtering by exclusion patterns (if any) that match
2035 : * btree index names.
2036 : */
2037 84 : appendPQExpBufferStr(&sql,
2038 : ", index (oid, nspname, relname, relpages) AS ("
2039 : "\nSELECT c.oid, r.nspname, c.relname, c.relpages "
2040 : "FROM relation r"
2041 : "\nINNER JOIN pg_catalog.pg_index i "
2042 : "ON r.oid = i.indrelid "
2043 : "INNER JOIN pg_catalog.pg_class c "
2044 : "ON i.indexrelid = c.oid "
2045 : "AND c.relpersistence != " CppAsString2(RELPERSISTENCE_TEMP));
2046 84 : if (opts.excludeidx || opts.excludensp)
2047 18 : appendPQExpBufferStr(&sql,
2048 : "\nINNER JOIN pg_catalog.pg_namespace n "
2049 : "ON c.relnamespace = n.oid"
2050 : "\nLEFT OUTER JOIN exclude_pat ep "
2051 : "ON (n.nspname ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
2052 : "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
2053 : "AND ep.btree_only"
2054 : "\nWHERE ep.pattern_id IS NULL");
2055 : else
2056 66 : appendPQExpBufferStr(&sql,
2057 : "\nWHERE true");
2058 84 : appendPQExpBuffer(&sql,
2059 : " AND c.relam = %u "
2060 : "AND c.relkind = " CppAsString2(RELKIND_INDEX),
2061 : BTREE_AM_OID);
2062 84 : if (opts.no_toast_expansion)
2063 2 : appendPQExpBuffer(&sql,
2064 : " AND c.relnamespace != %u",
2065 : PG_TOAST_NAMESPACE);
2066 84 : appendPQExpBufferStr(&sql, "\n)");
2067 : }
2068 :
2069 92 : if (!opts.no_toast_expansion && !opts.no_btree_expansion)
2070 : {
2071 : /*
2072 : * Include a CTE for btree indexes associated with toast tables of
2073 : * primary heap tables selected above, filtering by exclusion patterns
2074 : * (if any) that match the toast index names.
2075 : */
2076 82 : appendPQExpBufferStr(&sql,
2077 : ", toast_index (oid, nspname, relname, relpages) AS ("
2078 : "\nSELECT c.oid, 'pg_toast', c.relname, c.relpages "
2079 : "FROM toast t "
2080 : "INNER JOIN pg_catalog.pg_index i "
2081 : "ON t.oid = i.indrelid"
2082 : "\nINNER JOIN pg_catalog.pg_class c "
2083 : "ON i.indexrelid = c.oid "
2084 : "AND c.relpersistence != " CppAsString2(RELPERSISTENCE_TEMP));
2085 82 : if (opts.excludeidx)
2086 2 : appendPQExpBufferStr(&sql,
2087 : "\nLEFT OUTER JOIN exclude_pat ep "
2088 : "ON ('pg_toast' ~ ep.nsp_regex OR ep.nsp_regex IS NULL) "
2089 : "AND (c.relname ~ ep.rel_regex OR ep.rel_regex IS NULL) "
2090 : "AND ep.btree_only "
2091 : "WHERE ep.pattern_id IS NULL");
2092 : else
2093 80 : appendPQExpBufferStr(&sql,
2094 : "\nWHERE true");
2095 82 : appendPQExpBuffer(&sql,
2096 : " AND c.relam = %u"
2097 : " AND c.relkind = " CppAsString2(RELKIND_INDEX) ")",
2098 : BTREE_AM_OID);
2099 : }
2100 :
2101 : /*
2102 : * Roll-up distinct rows from CTEs.
2103 : *
2104 : * Relations that match more than one pattern may occur more than once in
2105 : * the list, and indexes and toast for primary relations may also have
2106 : * matched in their own right, so we rely on UNION to deduplicate the
2107 : * list.
2108 : */
2109 92 : appendPQExpBufferStr(&sql,
2110 : "\nSELECT pattern_id, is_heap, is_btree, oid, nspname, relname, relpages "
2111 : "FROM (");
2112 92 : appendPQExpBufferStr(&sql,
2113 : /* Inclusion patterns that failed to match */
2114 : "\nSELECT pattern_id, is_heap, is_btree, "
2115 : "NULL::OID AS oid, "
2116 : "NULL::TEXT AS nspname, "
2117 : "NULL::TEXT AS relname, "
2118 : "NULL::INTEGER AS relpages"
2119 : "\nFROM relation "
2120 : "WHERE pattern_id IS NOT NULL "
2121 : "UNION"
2122 : /* Primary relations */
2123 : "\nSELECT NULL::INTEGER AS pattern_id, "
2124 : "is_heap, is_btree, oid, nspname, relname, relpages "
2125 : "FROM relation");
2126 92 : if (!opts.no_toast_expansion)
2127 90 : appendPQExpBufferStr(&sql,
2128 : " UNION"
2129 : /* Toast tables for primary relations */
2130 : "\nSELECT NULL::INTEGER AS pattern_id, TRUE AS is_heap, "
2131 : "FALSE AS is_btree, oid, nspname, relname, relpages "
2132 : "FROM toast");
2133 92 : if (!opts.no_btree_expansion)
2134 84 : appendPQExpBufferStr(&sql,
2135 : " UNION"
2136 : /* Indexes for primary relations */
2137 : "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2138 : "TRUE AS is_btree, oid, nspname, relname, relpages "
2139 : "FROM index");
2140 92 : if (!opts.no_toast_expansion && !opts.no_btree_expansion)
2141 82 : appendPQExpBufferStr(&sql,
2142 : " UNION"
2143 : /* Indexes for toast relations */
2144 : "\nSELECT NULL::INTEGER AS pattern_id, FALSE AS is_heap, "
2145 : "TRUE AS is_btree, oid, nspname, relname, relpages "
2146 : "FROM toast_index");
2147 92 : appendPQExpBufferStr(&sql,
2148 : "\n) AS combined_records "
2149 : "ORDER BY relpages DESC NULLS FIRST, oid");
2150 :
2151 92 : res = executeQuery(conn, sql.data, opts.echo);
2152 92 : if (PQresultStatus(res) != PGRES_TUPLES_OK)
2153 : {
2154 0 : pg_log_error("query failed: %s", PQerrorMessage(conn));
2155 0 : pg_log_error_detail("Query was: %s", sql.data);
2156 0 : disconnectDatabase(conn);
2157 0 : exit(1);
2158 : }
2159 92 : termPQExpBuffer(&sql);
2160 :
2161 92 : ntups = PQntuples(res);
2162 15736 : for (i = 0; i < ntups; i++)
2163 : {
2164 15644 : int pattern_id = -1;
2165 15644 : bool is_heap = false;
2166 15644 : bool is_btree PG_USED_FOR_ASSERTS_ONLY = false;
2167 15644 : Oid oid = InvalidOid;
2168 15644 : const char *nspname = NULL;
2169 15644 : const char *relname = NULL;
2170 15644 : int relpages = 0;
2171 :
2172 15644 : if (!PQgetisnull(res, i, 0))
2173 80 : pattern_id = atoi(PQgetvalue(res, i, 0));
2174 15644 : if (!PQgetisnull(res, i, 1))
2175 15644 : is_heap = (PQgetvalue(res, i, 1)[0] == 't');
2176 15644 : if (!PQgetisnull(res, i, 2))
2177 15644 : is_btree = (PQgetvalue(res, i, 2)[0] == 't');
2178 15644 : if (!PQgetisnull(res, i, 3))
2179 15564 : oid = atooid(PQgetvalue(res, i, 3));
2180 15644 : if (!PQgetisnull(res, i, 4))
2181 15564 : nspname = PQgetvalue(res, i, 4);
2182 15644 : if (!PQgetisnull(res, i, 5))
2183 15564 : relname = PQgetvalue(res, i, 5);
2184 15644 : if (!PQgetisnull(res, i, 6))
2185 15564 : relpages = atoi(PQgetvalue(res, i, 6));
2186 :
2187 15644 : if (pattern_id >= 0)
2188 : {
2189 : /*
2190 : * Current record pertains to an inclusion pattern. Record that
2191 : * it matched.
2192 : */
2193 :
2194 80 : if (pattern_id >= opts.include.len)
2195 0 : pg_fatal("internal error: received unexpected relation pattern_id %d",
2196 : pattern_id);
2197 :
2198 80 : opts.include.data[pattern_id].matched = true;
2199 : }
2200 : else
2201 : {
2202 : /* Current record pertains to a relation */
2203 :
2204 15564 : RelationInfo *rel = (RelationInfo *) pg_malloc0(sizeof(RelationInfo));
2205 :
2206 : Assert(OidIsValid(oid));
2207 : Assert((is_heap && !is_btree) || (is_btree && !is_heap));
2208 :
2209 15564 : rel->datinfo = dat;
2210 15564 : rel->reloid = oid;
2211 15564 : rel->is_heap = is_heap;
2212 15564 : rel->nspname = pstrdup(nspname);
2213 15564 : rel->relname = pstrdup(relname);
2214 15564 : rel->relpages = relpages;
2215 15564 : rel->blocks_to_check = relpages;
2216 15564 : if (is_heap && (opts.startblock >= 0 || opts.endblock >= 0))
2217 : {
2218 : /*
2219 : * We apply --startblock and --endblock to heap tables, but
2220 : * not btree indexes, and for progress purposes we need to
2221 : * track how many blocks we expect to check.
2222 : */
2223 0 : if (opts.endblock >= 0 && rel->blocks_to_check > opts.endblock)
2224 0 : rel->blocks_to_check = opts.endblock + 1;
2225 0 : if (opts.startblock >= 0)
2226 : {
2227 0 : if (rel->blocks_to_check > opts.startblock)
2228 0 : rel->blocks_to_check -= opts.startblock;
2229 : else
2230 0 : rel->blocks_to_check = 0;
2231 : }
2232 : }
2233 15564 : *pagecount += rel->blocks_to_check;
2234 :
2235 15564 : simple_ptr_list_append(relations, rel);
2236 : }
2237 : }
2238 92 : PQclear(res);
2239 92 : }
|
