Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * verify_heapam.c
4 : * Functions to check postgresql heap relations for corruption
5 : *
6 : * Copyright (c) 2016-2024, PostgreSQL Global Development Group
7 : *
8 : * contrib/amcheck/verify_heapam.c
9 : *-------------------------------------------------------------------------
10 : */
11 : #include "postgres.h"
12 :
13 : #include "access/detoast.h"
14 : #include "access/genam.h"
15 : #include "access/heaptoast.h"
16 : #include "access/multixact.h"
17 : #include "access/relation.h"
18 : #include "access/table.h"
19 : #include "access/toast_internals.h"
20 : #include "access/visibilitymap.h"
21 : #include "access/xact.h"
22 : #include "catalog/pg_am.h"
23 : #include "catalog/pg_class.h"
24 : #include "funcapi.h"
25 : #include "miscadmin.h"
26 : #include "storage/bufmgr.h"
27 : #include "storage/procarray.h"
28 : #include "utils/builtins.h"
29 : #include "utils/fmgroids.h"
30 : #include "utils/rel.h"
31 :
32 604 : PG_FUNCTION_INFO_V1(verify_heapam);
33 :
34 : /* The number of columns in tuples returned by verify_heapam */
35 : #define HEAPCHECK_RELATION_COLS 4
36 :
37 : /* The largest valid toast va_rawsize */
38 : #define VARLENA_SIZE_LIMIT 0x3FFFFFFF
39 :
40 : /*
41 : * Despite the name, we use this for reporting problems with both XIDs and
42 : * MXIDs.
43 : */
44 : typedef enum XidBoundsViolation
45 : {
46 : XID_INVALID,
47 : XID_IN_FUTURE,
48 : XID_PRECEDES_CLUSTERMIN,
49 : XID_PRECEDES_RELMIN,
50 : XID_BOUNDS_OK,
51 : } XidBoundsViolation;
52 :
53 : typedef enum XidCommitStatus
54 : {
55 : XID_COMMITTED,
56 : XID_IS_CURRENT_XID,
57 : XID_IN_PROGRESS,
58 : XID_ABORTED,
59 : } XidCommitStatus;
60 :
61 : typedef enum SkipPages
62 : {
63 : SKIP_PAGES_ALL_FROZEN,
64 : SKIP_PAGES_ALL_VISIBLE,
65 : SKIP_PAGES_NONE,
66 : } SkipPages;
67 :
68 : /*
69 : * Struct holding information about a toasted attribute sufficient to both
70 : * check the toasted attribute and, if found to be corrupt, to report where it
71 : * was encountered in the main table.
72 : */
73 : typedef struct ToastedAttribute
74 : {
75 : struct varatt_external toast_pointer;
76 : BlockNumber blkno; /* block in main table */
77 : OffsetNumber offnum; /* offset in main table */
78 : AttrNumber attnum; /* attribute in main table */
79 : } ToastedAttribute;
80 :
81 : /*
82 : * Struct holding the running context information during
83 : * a lifetime of a verify_heapam execution.
84 : */
85 : typedef struct HeapCheckContext
86 : {
87 : /*
88 : * Cached copies of values from TransamVariables and computed values from
89 : * them.
90 : */
91 : FullTransactionId next_fxid; /* TransamVariables->nextXid */
92 : TransactionId next_xid; /* 32-bit version of next_fxid */
93 : TransactionId oldest_xid; /* TransamVariables->oldestXid */
94 : FullTransactionId oldest_fxid; /* 64-bit version of oldest_xid, computed
95 : * relative to next_fxid */
96 : TransactionId safe_xmin; /* this XID and newer ones can't become
97 : * all-visible while we're running */
98 :
99 : /*
100 : * Cached copy of value from MultiXactState
101 : */
102 : MultiXactId next_mxact; /* MultiXactState->nextMXact */
103 : MultiXactId oldest_mxact; /* MultiXactState->oldestMultiXactId */
104 :
105 : /*
106 : * Cached copies of the most recently checked xid and its status.
107 : */
108 : TransactionId cached_xid;
109 : XidCommitStatus cached_status;
110 :
111 : /* Values concerning the heap relation being checked */
112 : Relation rel;
113 : TransactionId relfrozenxid;
114 : FullTransactionId relfrozenfxid;
115 : TransactionId relminmxid;
116 : Relation toast_rel;
117 : Relation *toast_indexes;
118 : Relation valid_toast_index;
119 : int num_toast_indexes;
120 :
121 : /* Values for iterating over pages in the relation */
122 : BlockNumber blkno;
123 : BufferAccessStrategy bstrategy;
124 : Buffer buffer;
125 : Page page;
126 :
127 : /* Values for iterating over tuples within a page */
128 : OffsetNumber offnum;
129 : ItemId itemid;
130 : uint16 lp_len;
131 : uint16 lp_off;
132 : HeapTupleHeader tuphdr;
133 : int natts;
134 :
135 : /* Values for iterating over attributes within the tuple */
136 : uint32 offset; /* offset in tuple data */
137 : AttrNumber attnum;
138 :
139 : /* True if tuple's xmax makes it eligible for pruning */
140 : bool tuple_could_be_pruned;
141 :
142 : /*
143 : * List of ToastedAttribute structs for toasted attributes which are not
144 : * eligible for pruning and should be checked
145 : */
146 : List *toasted_attributes;
147 :
148 : /* Whether verify_heapam has yet encountered any corrupt tuples */
149 : bool is_corrupt;
150 :
151 : /* The descriptor and tuplestore for verify_heapam's result tuples */
152 : TupleDesc tupdesc;
153 : Tuplestorestate *tupstore;
154 : } HeapCheckContext;
155 :
156 : /* Internal implementation */
157 : static void check_tuple(HeapCheckContext *ctx,
158 : bool *xmin_commit_status_ok,
159 : XidCommitStatus *xmin_commit_status);
160 : static void check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
161 : ToastedAttribute *ta, int32 *expected_chunk_seq,
162 : uint32 extsize);
163 :
164 : static bool check_tuple_attribute(HeapCheckContext *ctx);
165 : static void check_toasted_attribute(HeapCheckContext *ctx,
166 : ToastedAttribute *ta);
167 :
168 : static bool check_tuple_header(HeapCheckContext *ctx);
169 : static bool check_tuple_visibility(HeapCheckContext *ctx,
170 : bool *xmin_commit_status_ok,
171 : XidCommitStatus *xmin_commit_status);
172 :
173 : static void report_corruption(HeapCheckContext *ctx, char *msg);
174 : static void report_toast_corruption(HeapCheckContext *ctx,
175 : ToastedAttribute *ta, char *msg);
176 : static FullTransactionId FullTransactionIdFromXidAndCtx(TransactionId xid,
177 : const HeapCheckContext *ctx);
178 : static void update_cached_xid_range(HeapCheckContext *ctx);
179 : static void update_cached_mxid_range(HeapCheckContext *ctx);
180 : static XidBoundsViolation check_mxid_in_range(MultiXactId mxid,
181 : HeapCheckContext *ctx);
182 : static XidBoundsViolation check_mxid_valid_in_rel(MultiXactId mxid,
183 : HeapCheckContext *ctx);
184 : static XidBoundsViolation get_xid_status(TransactionId xid,
185 : HeapCheckContext *ctx,
186 : XidCommitStatus *status);
187 :
188 : /*
189 : * Scan and report corruption in heap pages, optionally reconciling toasted
190 : * attributes with entries in the associated toast table. Intended to be
191 : * called from SQL with the following parameters:
192 : *
193 : * relation:
194 : * The Oid of the heap relation to be checked.
195 : *
196 : * on_error_stop:
197 : * Whether to stop at the end of the first page for which errors are
198 : * detected. Note that multiple rows may be returned.
199 : *
200 : * check_toast:
201 : * Whether to check each toasted attribute against the toast table to
202 : * verify that it can be found there.
203 : *
204 : * skip:
205 : * What kinds of pages in the heap relation should be skipped. Valid
206 : * options are "all-visible", "all-frozen", and "none".
207 : *
208 : * Returns to the SQL caller a set of tuples, each containing the location
209 : * and a description of a corruption found in the heap.
210 : *
211 : * This code goes to some trouble to avoid crashing the server even if the
212 : * table pages are badly corrupted, but it's probably not perfect. If
213 : * check_toast is true, we'll use regular index lookups to try to fetch TOAST
214 : * tuples, which can certainly cause crashes if the right kind of corruption
215 : * exists in the toast table or index. No matter what parameters you pass,
216 : * we can't protect against crashes that might occur trying to look up the
217 : * commit status of transaction IDs (though we avoid trying to do such lookups
218 : * for transaction IDs that can't legally appear in the table).
219 : */
220 : Datum
221 6882 : verify_heapam(PG_FUNCTION_ARGS)
222 : {
223 6882 : ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
224 : HeapCheckContext ctx;
225 6882 : Buffer vmbuffer = InvalidBuffer;
226 : Oid relid;
227 : bool on_error_stop;
228 : bool check_toast;
229 6882 : SkipPages skip_option = SKIP_PAGES_NONE;
230 : BlockNumber first_block;
231 : BlockNumber last_block;
232 : BlockNumber nblocks;
233 : const char *skip;
234 :
235 : /* Check supplied arguments */
236 6882 : if (PG_ARGISNULL(0))
237 0 : ereport(ERROR,
238 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
239 : errmsg("relation cannot be null")));
240 6882 : relid = PG_GETARG_OID(0);
241 :
242 6882 : if (PG_ARGISNULL(1))
243 0 : ereport(ERROR,
244 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
245 : errmsg("on_error_stop cannot be null")));
246 6882 : on_error_stop = PG_GETARG_BOOL(1);
247 :
248 6882 : if (PG_ARGISNULL(2))
249 0 : ereport(ERROR,
250 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
251 : errmsg("check_toast cannot be null")));
252 6882 : check_toast = PG_GETARG_BOOL(2);
253 :
254 6882 : if (PG_ARGISNULL(3))
255 0 : ereport(ERROR,
256 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
257 : errmsg("skip cannot be null")));
258 6882 : skip = text_to_cstring(PG_GETARG_TEXT_PP(3));
259 6882 : if (pg_strcasecmp(skip, "all-visible") == 0)
260 170 : skip_option = SKIP_PAGES_ALL_VISIBLE;
261 6712 : else if (pg_strcasecmp(skip, "all-frozen") == 0)
262 174 : skip_option = SKIP_PAGES_ALL_FROZEN;
263 6538 : else if (pg_strcasecmp(skip, "none") == 0)
264 6536 : skip_option = SKIP_PAGES_NONE;
265 : else
266 2 : ereport(ERROR,
267 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
268 : errmsg("invalid skip option"),
269 : errhint("Valid skip options are \"all-visible\", \"all-frozen\", and \"none\".")));
270 :
271 6880 : memset(&ctx, 0, sizeof(HeapCheckContext));
272 6880 : ctx.cached_xid = InvalidTransactionId;
273 6880 : ctx.toasted_attributes = NIL;
274 :
275 : /*
276 : * Any xmin newer than the xmin of our snapshot can't become all-visible
277 : * while we're running.
278 : */
279 6880 : ctx.safe_xmin = GetTransactionSnapshot()->xmin;
280 :
281 : /*
282 : * If we report corruption when not examining some individual attribute,
283 : * we need attnum to be reported as NULL. Set that up before any
284 : * corruption reporting might happen.
285 : */
286 6880 : ctx.attnum = -1;
287 :
288 : /* Construct the tuplestore and tuple descriptor */
289 6880 : InitMaterializedSRF(fcinfo, 0);
290 6880 : ctx.tupdesc = rsinfo->setDesc;
291 6880 : ctx.tupstore = rsinfo->setResult;
292 :
293 : /* Open relation, check relkind and access method */
294 6880 : ctx.rel = relation_open(relid, AccessShareLock);
295 :
296 : /*
297 : * Check that a relation's relkind and access method are both supported.
298 : */
299 6880 : if (!RELKIND_HAS_TABLE_AM(ctx.rel->rd_rel->relkind) &&
300 392 : ctx.rel->rd_rel->relkind != RELKIND_SEQUENCE)
301 8 : ereport(ERROR,
302 : (errcode(ERRCODE_WRONG_OBJECT_TYPE),
303 : errmsg("cannot check relation \"%s\"",
304 : RelationGetRelationName(ctx.rel)),
305 : errdetail_relkind_not_supported(ctx.rel->rd_rel->relkind)));
306 :
307 : /*
308 : * Sequences always use heap AM, but they don't show that in the catalogs.
309 : * Other relkinds might be using a different AM, so check.
310 : */
311 6872 : if (ctx.rel->rd_rel->relkind != RELKIND_SEQUENCE &&
312 6488 : ctx.rel->rd_rel->relam != HEAP_TABLE_AM_OID)
313 0 : ereport(ERROR,
314 : (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
315 : errmsg("only heap AM is supported")));
316 :
317 : /*
318 : * Early exit for unlogged relations during recovery. These will have no
319 : * relation fork, so there won't be anything to check. We behave as if
320 : * the relation is empty.
321 : */
322 6872 : if (ctx.rel->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED &&
323 0 : RecoveryInProgress())
324 : {
325 0 : ereport(DEBUG1,
326 : (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION),
327 : errmsg("cannot verify unlogged relation \"%s\" during recovery, skipping",
328 : RelationGetRelationName(ctx.rel))));
329 0 : relation_close(ctx.rel, AccessShareLock);
330 0 : PG_RETURN_NULL();
331 : }
332 :
333 : /* Early exit if the relation is empty */
334 6872 : nblocks = RelationGetNumberOfBlocks(ctx.rel);
335 6838 : if (!nblocks)
336 : {
337 3882 : relation_close(ctx.rel, AccessShareLock);
338 3882 : PG_RETURN_NULL();
339 : }
340 :
341 2956 : ctx.bstrategy = GetAccessStrategy(BAS_BULKREAD);
342 2956 : ctx.buffer = InvalidBuffer;
343 2956 : ctx.page = NULL;
344 :
345 : /* Validate block numbers, or handle nulls. */
346 2956 : if (PG_ARGISNULL(4))
347 2708 : first_block = 0;
348 : else
349 : {
350 248 : int64 fb = PG_GETARG_INT64(4);
351 :
352 248 : if (fb < 0 || fb >= nblocks)
353 2 : ereport(ERROR,
354 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
355 : errmsg("starting block number must be between 0 and %u",
356 : nblocks - 1)));
357 246 : first_block = (BlockNumber) fb;
358 : }
359 2954 : if (PG_ARGISNULL(5))
360 2708 : last_block = nblocks - 1;
361 : else
362 : {
363 246 : int64 lb = PG_GETARG_INT64(5);
364 :
365 246 : if (lb < 0 || lb >= nblocks)
366 2 : ereport(ERROR,
367 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
368 : errmsg("ending block number must be between 0 and %u",
369 : nblocks - 1)));
370 244 : last_block = (BlockNumber) lb;
371 : }
372 :
373 : /* Optionally open the toast relation, if any. */
374 2952 : if (ctx.rel->rd_rel->reltoastrelid && check_toast)
375 1392 : {
376 : int offset;
377 :
378 : /* Main relation has associated toast relation */
379 1392 : ctx.toast_rel = table_open(ctx.rel->rd_rel->reltoastrelid,
380 : AccessShareLock);
381 1392 : offset = toast_open_indexes(ctx.toast_rel,
382 : AccessShareLock,
383 : &(ctx.toast_indexes),
384 : &(ctx.num_toast_indexes));
385 1392 : ctx.valid_toast_index = ctx.toast_indexes[offset];
386 : }
387 : else
388 : {
389 : /*
390 : * Main relation has no associated toast relation, or we're
391 : * intentionally skipping it.
392 : */
393 1560 : ctx.toast_rel = NULL;
394 1560 : ctx.toast_indexes = NULL;
395 1560 : ctx.num_toast_indexes = 0;
396 : }
397 :
398 2952 : update_cached_xid_range(&ctx);
399 2952 : update_cached_mxid_range(&ctx);
400 2952 : ctx.relfrozenxid = ctx.rel->rd_rel->relfrozenxid;
401 2952 : ctx.relfrozenfxid = FullTransactionIdFromXidAndCtx(ctx.relfrozenxid, &ctx);
402 2952 : ctx.relminmxid = ctx.rel->rd_rel->relminmxid;
403 :
404 2952 : if (TransactionIdIsNormal(ctx.relfrozenxid))
405 2568 : ctx.oldest_xid = ctx.relfrozenxid;
406 :
407 26756 : for (ctx.blkno = first_block; ctx.blkno <= last_block; ctx.blkno++)
408 : {
409 : OffsetNumber maxoff;
410 : OffsetNumber predecessor[MaxOffsetNumber];
411 : OffsetNumber successor[MaxOffsetNumber];
412 : bool lp_valid[MaxOffsetNumber];
413 : bool xmin_commit_status_ok[MaxOffsetNumber];
414 : XidCommitStatus xmin_commit_status[MaxOffsetNumber];
415 :
416 23810 : CHECK_FOR_INTERRUPTS();
417 :
418 23810 : memset(predecessor, 0, sizeof(OffsetNumber) * MaxOffsetNumber);
419 :
420 : /* Optionally skip over all-frozen or all-visible blocks */
421 23810 : if (skip_option != SKIP_PAGES_NONE)
422 : {
423 : int32 mapbits;
424 :
425 1472 : mapbits = (int32) visibilitymap_get_status(ctx.rel, ctx.blkno,
426 : &vmbuffer);
427 1472 : if (skip_option == SKIP_PAGES_ALL_FROZEN)
428 : {
429 768 : if ((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0)
430 66 : continue;
431 : }
432 :
433 1408 : if (skip_option == SKIP_PAGES_ALL_VISIBLE)
434 : {
435 704 : if ((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0)
436 2 : continue;
437 : }
438 : }
439 :
440 : /* Read and lock the next page. */
441 23744 : ctx.buffer = ReadBufferExtended(ctx.rel, MAIN_FORKNUM, ctx.blkno,
442 : RBM_NORMAL, ctx.bstrategy);
443 23744 : LockBuffer(ctx.buffer, BUFFER_LOCK_SHARE);
444 23744 : ctx.page = BufferGetPage(ctx.buffer);
445 :
446 : /* Perform tuple checks */
447 23744 : maxoff = PageGetMaxOffsetNumber(ctx.page);
448 1130008 : for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
449 1106264 : ctx.offnum = OffsetNumberNext(ctx.offnum))
450 : {
451 : BlockNumber nextblkno;
452 : OffsetNumber nextoffnum;
453 :
454 1106264 : successor[ctx.offnum] = InvalidOffsetNumber;
455 1106264 : lp_valid[ctx.offnum] = false;
456 1106264 : xmin_commit_status_ok[ctx.offnum] = false;
457 1106264 : ctx.itemid = PageGetItemId(ctx.page, ctx.offnum);
458 :
459 : /* Skip over unused/dead line pointers */
460 1106264 : if (!ItemIdIsUsed(ctx.itemid) || ItemIdIsDead(ctx.itemid))
461 18464 : continue;
462 :
463 : /*
464 : * If this line pointer has been redirected, check that it
465 : * redirects to a valid offset within the line pointer array
466 : */
467 1087800 : if (ItemIdIsRedirected(ctx.itemid))
468 : {
469 11490 : OffsetNumber rdoffnum = ItemIdGetRedirect(ctx.itemid);
470 : ItemId rditem;
471 :
472 11490 : if (rdoffnum < FirstOffsetNumber)
473 : {
474 12 : report_corruption(&ctx,
475 : psprintf("line pointer redirection to item at offset %u precedes minimum offset %u",
476 : (unsigned) rdoffnum,
477 : (unsigned) FirstOffsetNumber));
478 12 : continue;
479 : }
480 11478 : if (rdoffnum > maxoff)
481 : {
482 28 : report_corruption(&ctx,
483 : psprintf("line pointer redirection to item at offset %u exceeds maximum offset %u",
484 : (unsigned) rdoffnum,
485 : (unsigned) maxoff));
486 28 : continue;
487 : }
488 :
489 : /*
490 : * Since we've checked that this redirect points to a line
491 : * pointer between FirstOffsetNumber and maxoff, it should now
492 : * be safe to fetch the referenced line pointer. We expect it
493 : * to be LP_NORMAL; if not, that's corruption.
494 : */
495 11450 : rditem = PageGetItemId(ctx.page, rdoffnum);
496 11450 : if (!ItemIdIsUsed(rditem))
497 : {
498 0 : report_corruption(&ctx,
499 : psprintf("redirected line pointer points to an unused item at offset %u",
500 : (unsigned) rdoffnum));
501 0 : continue;
502 : }
503 11450 : else if (ItemIdIsDead(rditem))
504 : {
505 0 : report_corruption(&ctx,
506 : psprintf("redirected line pointer points to a dead item at offset %u",
507 : (unsigned) rdoffnum));
508 0 : continue;
509 : }
510 11450 : else if (ItemIdIsRedirected(rditem))
511 : {
512 2 : report_corruption(&ctx,
513 : psprintf("redirected line pointer points to another redirected line pointer at offset %u",
514 : (unsigned) rdoffnum));
515 2 : continue;
516 : }
517 :
518 : /*
519 : * Record the fact that this line pointer has passed basic
520 : * sanity checking, and also the offset number to which it
521 : * points.
522 : */
523 11448 : lp_valid[ctx.offnum] = true;
524 11448 : successor[ctx.offnum] = rdoffnum;
525 11448 : continue;
526 : }
527 :
528 : /* Sanity-check the line pointer's offset and length values */
529 1076310 : ctx.lp_len = ItemIdGetLength(ctx.itemid);
530 1076310 : ctx.lp_off = ItemIdGetOffset(ctx.itemid);
531 :
532 1076310 : if (ctx.lp_off != MAXALIGN(ctx.lp_off))
533 : {
534 12 : report_corruption(&ctx,
535 : psprintf("line pointer to page offset %u is not maximally aligned",
536 12 : ctx.lp_off));
537 12 : continue;
538 : }
539 1076298 : if (ctx.lp_len < MAXALIGN(SizeofHeapTupleHeader))
540 : {
541 24 : report_corruption(&ctx,
542 : psprintf("line pointer length %u is less than the minimum tuple header size %u",
543 24 : ctx.lp_len,
544 : (unsigned) MAXALIGN(SizeofHeapTupleHeader)));
545 24 : continue;
546 : }
547 1076274 : if (ctx.lp_off + ctx.lp_len > BLCKSZ)
548 : {
549 28 : report_corruption(&ctx,
550 : psprintf("line pointer to page offset %u with length %u ends beyond maximum page offset %u",
551 28 : ctx.lp_off,
552 28 : ctx.lp_len,
553 : (unsigned) BLCKSZ));
554 28 : continue;
555 : }
556 :
557 : /* It should be safe to examine the tuple's header, at least */
558 1076246 : lp_valid[ctx.offnum] = true;
559 1076246 : ctx.tuphdr = (HeapTupleHeader) PageGetItem(ctx.page, ctx.itemid);
560 1076246 : ctx.natts = HeapTupleHeaderGetNatts(ctx.tuphdr);
561 :
562 : /* Ok, ready to check this next tuple */
563 1076246 : check_tuple(&ctx,
564 1076246 : &xmin_commit_status_ok[ctx.offnum],
565 1076246 : &xmin_commit_status[ctx.offnum]);
566 :
567 : /*
568 : * If the CTID field of this tuple seems to point to another tuple
569 : * on the same page, record that tuple as the successor of this
570 : * one.
571 : */
572 1076246 : nextblkno = ItemPointerGetBlockNumber(&(ctx.tuphdr)->t_ctid);
573 1076246 : nextoffnum = ItemPointerGetOffsetNumber(&(ctx.tuphdr)->t_ctid);
574 1076246 : if (nextblkno == ctx.blkno && nextoffnum != ctx.offnum &&
575 350 : nextoffnum >= FirstOffsetNumber && nextoffnum <= maxoff)
576 350 : successor[ctx.offnum] = nextoffnum;
577 : }
578 :
579 : /*
580 : * Update chain validation. Check each line pointer that's got a valid
581 : * successor against that successor.
582 : */
583 23744 : ctx.attnum = -1;
584 1130008 : for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
585 1106264 : ctx.offnum = OffsetNumberNext(ctx.offnum))
586 : {
587 : ItemId curr_lp;
588 : ItemId next_lp;
589 : HeapTupleHeader curr_htup;
590 : HeapTupleHeader next_htup;
591 : TransactionId curr_xmin;
592 : TransactionId curr_xmax;
593 : TransactionId next_xmin;
594 1106264 : OffsetNumber nextoffnum = successor[ctx.offnum];
595 :
596 : /*
597 : * The current line pointer may not have a successor, either
598 : * because it's not valid or because it didn't point to anything.
599 : * In either case, we have to give up.
600 : *
601 : * If the current line pointer does point to something, it's
602 : * possible that the target line pointer isn't valid. We have to
603 : * give up in that case, too.
604 : */
605 1106264 : if (nextoffnum == InvalidOffsetNumber || !lp_valid[nextoffnum])
606 1094466 : continue;
607 :
608 : /* We have two valid line pointers that we can examine. */
609 11798 : curr_lp = PageGetItemId(ctx.page, ctx.offnum);
610 11798 : next_lp = PageGetItemId(ctx.page, nextoffnum);
611 :
612 : /* Handle the cases where the current line pointer is a redirect. */
613 11798 : if (ItemIdIsRedirected(curr_lp))
614 : {
615 : /*
616 : * We should not have set successor[ctx.offnum] to a value
617 : * other than InvalidOffsetNumber unless that line pointer is
618 : * LP_NORMAL.
619 : */
620 : Assert(ItemIdIsNormal(next_lp));
621 :
622 : /* Can only redirect to a HOT tuple. */
623 11448 : next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
624 11448 : if (!HeapTupleHeaderIsHeapOnly(next_htup))
625 : {
626 2 : report_corruption(&ctx,
627 : psprintf("redirected line pointer points to a non-heap-only tuple at offset %u",
628 : (unsigned) nextoffnum));
629 : }
630 :
631 : /* HOT chains should not intersect. */
632 11448 : if (predecessor[nextoffnum] != InvalidOffsetNumber)
633 : {
634 2 : report_corruption(&ctx,
635 : psprintf("redirect line pointer points to offset %u, but offset %u also points there",
636 2 : (unsigned) nextoffnum, (unsigned) predecessor[nextoffnum]));
637 2 : continue;
638 : }
639 :
640 : /*
641 : * This redirect and the tuple to which it points seem to be
642 : * part of an update chain.
643 : */
644 11446 : predecessor[nextoffnum] = ctx.offnum;
645 11446 : continue;
646 : }
647 :
648 : /*
649 : * If the next line pointer is a redirect, or if it's a tuple but
650 : * the XMAX of this tuple doesn't match the XMIN of the next
651 : * tuple, then the two aren't part of the same update chain and
652 : * there is nothing more to do.
653 : */
654 350 : if (ItemIdIsRedirected(next_lp))
655 0 : continue;
656 350 : curr_htup = (HeapTupleHeader) PageGetItem(ctx.page, curr_lp);
657 350 : curr_xmax = HeapTupleHeaderGetUpdateXid(curr_htup);
658 350 : next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
659 350 : next_xmin = HeapTupleHeaderGetXmin(next_htup);
660 350 : if (!TransactionIdIsValid(curr_xmax) ||
661 : !TransactionIdEquals(curr_xmax, next_xmin))
662 8 : continue;
663 :
664 : /* HOT chains should not intersect. */
665 342 : if (predecessor[nextoffnum] != InvalidOffsetNumber)
666 : {
667 2 : report_corruption(&ctx,
668 : psprintf("tuple points to new version at offset %u, but offset %u also points there",
669 2 : (unsigned) nextoffnum, (unsigned) predecessor[nextoffnum]));
670 2 : continue;
671 : }
672 :
673 : /*
674 : * This tuple and the tuple to which it points seem to be part of
675 : * an update chain.
676 : */
677 340 : predecessor[nextoffnum] = ctx.offnum;
678 :
679 : /*
680 : * If the current tuple is marked as HOT-updated, then the next
681 : * tuple should be marked as a heap-only tuple. Conversely, if the
682 : * current tuple isn't marked as HOT-updated, then the next tuple
683 : * shouldn't be marked as a heap-only tuple.
684 : *
685 : * NB: Can't use HeapTupleHeaderIsHotUpdated() as it checks if
686 : * hint bits indicate xmin/xmax aborted.
687 : */
688 340 : if (!(curr_htup->t_infomask2 & HEAP_HOT_UPDATED) &&
689 2 : HeapTupleHeaderIsHeapOnly(next_htup))
690 : {
691 2 : report_corruption(&ctx,
692 : psprintf("non-heap-only update produced a heap-only tuple at offset %u",
693 : (unsigned) nextoffnum));
694 : }
695 340 : if ((curr_htup->t_infomask2 & HEAP_HOT_UPDATED) &&
696 338 : !HeapTupleHeaderIsHeapOnly(next_htup))
697 : {
698 2 : report_corruption(&ctx,
699 : psprintf("heap-only update produced a non-heap only tuple at offset %u",
700 : (unsigned) nextoffnum));
701 : }
702 :
703 : /*
704 : * If the current tuple's xmin is still in progress but the
705 : * successor tuple's xmin is committed, that's corruption.
706 : *
707 : * NB: We recheck the commit status of the current tuple's xmin
708 : * here, because it might have committed after we checked it and
709 : * before we checked the commit status of the successor tuple's
710 : * xmin. This should be safe because the xmin itself can't have
711 : * changed, only its commit status.
712 : */
713 340 : curr_xmin = HeapTupleHeaderGetXmin(curr_htup);
714 340 : if (xmin_commit_status_ok[ctx.offnum] &&
715 340 : xmin_commit_status[ctx.offnum] == XID_IN_PROGRESS &&
716 2 : xmin_commit_status_ok[nextoffnum] &&
717 4 : xmin_commit_status[nextoffnum] == XID_COMMITTED &&
718 2 : TransactionIdIsInProgress(curr_xmin))
719 : {
720 2 : report_corruption(&ctx,
721 : psprintf("tuple with in-progress xmin %u was updated to produce a tuple at offset %u with committed xmin %u",
722 : (unsigned) curr_xmin,
723 2 : (unsigned) ctx.offnum,
724 : (unsigned) next_xmin));
725 : }
726 :
727 : /*
728 : * If the current tuple's xmin is aborted but the successor
729 : * tuple's xmin is in-progress or committed, that's corruption.
730 : */
731 340 : if (xmin_commit_status_ok[ctx.offnum] &&
732 340 : xmin_commit_status[ctx.offnum] == XID_ABORTED &&
733 4 : xmin_commit_status_ok[nextoffnum])
734 : {
735 4 : if (xmin_commit_status[nextoffnum] == XID_IN_PROGRESS)
736 2 : report_corruption(&ctx,
737 : psprintf("tuple with aborted xmin %u was updated to produce a tuple at offset %u with in-progress xmin %u",
738 : (unsigned) curr_xmin,
739 2 : (unsigned) ctx.offnum,
740 : (unsigned) next_xmin));
741 2 : else if (xmin_commit_status[nextoffnum] == XID_COMMITTED)
742 2 : report_corruption(&ctx,
743 : psprintf("tuple with aborted xmin %u was updated to produce a tuple at offset %u with committed xmin %u",
744 : (unsigned) curr_xmin,
745 2 : (unsigned) ctx.offnum,
746 : (unsigned) next_xmin));
747 : }
748 : }
749 :
750 : /*
751 : * An update chain can start either with a non-heap-only tuple or with
752 : * a redirect line pointer, but not with a heap-only tuple.
753 : *
754 : * (This check is in a separate loop because we need the predecessor
755 : * array to be fully populated before we can perform it.)
756 : */
757 23744 : for (ctx.offnum = FirstOffsetNumber;
758 1130008 : ctx.offnum <= maxoff;
759 1106264 : ctx.offnum = OffsetNumberNext(ctx.offnum))
760 : {
761 1106264 : if (xmin_commit_status_ok[ctx.offnum] &&
762 1076228 : (xmin_commit_status[ctx.offnum] == XID_COMMITTED ||
763 14 : xmin_commit_status[ctx.offnum] == XID_IN_PROGRESS) &&
764 1076218 : predecessor[ctx.offnum] == InvalidOffsetNumber)
765 : {
766 : ItemId curr_lp;
767 :
768 1064438 : curr_lp = PageGetItemId(ctx.page, ctx.offnum);
769 1064438 : if (!ItemIdIsRedirected(curr_lp))
770 : {
771 : HeapTupleHeader curr_htup;
772 :
773 : curr_htup = (HeapTupleHeader)
774 1064438 : PageGetItem(ctx.page, curr_lp);
775 1064438 : if (HeapTupleHeaderIsHeapOnly(curr_htup))
776 8 : report_corruption(&ctx,
777 : psprintf("tuple is root of chain but is marked as heap-only tuple"));
778 : }
779 : }
780 : }
781 :
782 : /* clean up */
783 23744 : UnlockReleaseBuffer(ctx.buffer);
784 :
785 : /*
786 : * Check any toast pointers from the page whose lock we just released
787 : */
788 23744 : if (ctx.toasted_attributes != NIL)
789 : {
790 : ListCell *cell;
791 :
792 25310 : foreach(cell, ctx.toasted_attributes)
793 23630 : check_toasted_attribute(&ctx, lfirst(cell));
794 1680 : list_free_deep(ctx.toasted_attributes);
795 1680 : ctx.toasted_attributes = NIL;
796 : }
797 :
798 23738 : if (on_error_stop && ctx.is_corrupt)
799 0 : break;
800 : }
801 :
802 2946 : if (vmbuffer != InvalidBuffer)
803 6 : ReleaseBuffer(vmbuffer);
804 :
805 : /* Close the associated toast table and indexes, if any. */
806 2946 : if (ctx.toast_indexes)
807 1386 : toast_close_indexes(ctx.toast_indexes, ctx.num_toast_indexes,
808 : AccessShareLock);
809 2946 : if (ctx.toast_rel)
810 1386 : table_close(ctx.toast_rel, AccessShareLock);
811 :
812 : /* Close the main relation */
813 2946 : relation_close(ctx.rel, AccessShareLock);
814 :
815 2946 : PG_RETURN_NULL();
816 : }
817 :
818 : /*
819 : * Shared internal implementation for report_corruption and
820 : * report_toast_corruption.
821 : */
822 : static void
823 172 : report_corruption_internal(Tuplestorestate *tupstore, TupleDesc tupdesc,
824 : BlockNumber blkno, OffsetNumber offnum,
825 : AttrNumber attnum, char *msg)
826 : {
827 172 : Datum values[HEAPCHECK_RELATION_COLS] = {0};
828 172 : bool nulls[HEAPCHECK_RELATION_COLS] = {0};
829 : HeapTuple tuple;
830 :
831 172 : values[0] = Int64GetDatum(blkno);
832 172 : values[1] = Int32GetDatum(offnum);
833 172 : values[2] = Int32GetDatum(attnum);
834 172 : nulls[2] = (attnum < 0);
835 172 : values[3] = CStringGetTextDatum(msg);
836 :
837 : /*
838 : * In principle, there is nothing to prevent a scan over a large, highly
839 : * corrupted table from using work_mem worth of memory building up the
840 : * tuplestore. That's ok, but if we also leak the msg argument memory
841 : * until the end of the query, we could exceed work_mem by more than a
842 : * trivial amount. Therefore, free the msg argument each time we are
843 : * called rather than waiting for our current memory context to be freed.
844 : */
845 172 : pfree(msg);
846 :
847 172 : tuple = heap_form_tuple(tupdesc, values, nulls);
848 172 : tuplestore_puttuple(tupstore, tuple);
849 172 : }
850 :
851 : /*
852 : * Record a single corruption found in the main table. The values in ctx should
853 : * indicate the location of the corruption, and the msg argument should contain
854 : * a human-readable description of the corruption.
855 : *
856 : * The msg argument is pfree'd by this function.
857 : */
858 : static void
859 170 : report_corruption(HeapCheckContext *ctx, char *msg)
860 : {
861 170 : report_corruption_internal(ctx->tupstore, ctx->tupdesc, ctx->blkno,
862 170 : ctx->offnum, ctx->attnum, msg);
863 170 : ctx->is_corrupt = true;
864 170 : }
865 :
866 : /*
867 : * Record corruption found in the toast table. The values in ta should
868 : * indicate the location in the main table where the toast pointer was
869 : * encountered, and the msg argument should contain a human-readable
870 : * description of the toast table corruption.
871 : *
872 : * As above, the msg argument is pfree'd by this function.
873 : */
874 : static void
875 2 : report_toast_corruption(HeapCheckContext *ctx, ToastedAttribute *ta,
876 : char *msg)
877 : {
878 2 : report_corruption_internal(ctx->tupstore, ctx->tupdesc, ta->blkno,
879 2 : ta->offnum, ta->attnum, msg);
880 2 : ctx->is_corrupt = true;
881 2 : }
882 :
883 : /*
884 : * Check for tuple header corruption.
885 : *
886 : * Some kinds of corruption make it unsafe to check the tuple attributes, for
887 : * example when the line pointer refers to a range of bytes outside the page.
888 : * In such cases, we return false (not checkable) after recording appropriate
889 : * corruption messages.
890 : *
891 : * Some other kinds of tuple header corruption confuse the question of where
892 : * the tuple attributes begin, or how long the nulls bitmap is, etc., making it
893 : * unreasonable to attempt to check attributes, even if all candidate answers
894 : * to those questions would not result in reading past the end of the line
895 : * pointer or page. In such cases, like above, we record corruption messages
896 : * about the header and then return false.
897 : *
898 : * Other kinds of tuple header corruption do not bear on the question of
899 : * whether the tuple attributes can be checked, so we record corruption
900 : * messages for them but we do not return false merely because we detected
901 : * them.
902 : *
903 : * Returns whether the tuple is sufficiently sensible to undergo visibility and
904 : * attribute checks.
905 : */
906 : static bool
907 1076246 : check_tuple_header(HeapCheckContext *ctx)
908 : {
909 1076246 : HeapTupleHeader tuphdr = ctx->tuphdr;
910 1076246 : uint16 infomask = tuphdr->t_infomask;
911 1076246 : TransactionId curr_xmax = HeapTupleHeaderGetUpdateXid(tuphdr);
912 1076246 : bool result = true;
913 : unsigned expected_hoff;
914 :
915 1076246 : if (ctx->tuphdr->t_hoff > ctx->lp_len)
916 : {
917 2 : report_corruption(ctx,
918 : psprintf("data begins at offset %u beyond the tuple length %u",
919 2 : ctx->tuphdr->t_hoff, ctx->lp_len));
920 2 : result = false;
921 : }
922 :
923 1076246 : if ((ctx->tuphdr->t_infomask & HEAP_XMAX_COMMITTED) &&
924 360 : (ctx->tuphdr->t_infomask & HEAP_XMAX_IS_MULTI))
925 : {
926 4 : report_corruption(ctx,
927 : pstrdup("multixact should not be marked committed"));
928 :
929 : /*
930 : * This condition is clearly wrong, but it's not enough to justify
931 : * skipping further checks, because we don't rely on this to determine
932 : * whether the tuple is visible or to interpret other relevant header
933 : * fields.
934 : */
935 : }
936 :
937 1076246 : if (!TransactionIdIsValid(curr_xmax) &&
938 1073644 : HeapTupleHeaderIsHotUpdated(tuphdr))
939 : {
940 2 : report_corruption(ctx,
941 : psprintf("tuple has been HOT updated, but xmax is 0"));
942 :
943 : /*
944 : * As above, even though this shouldn't happen, it's not sufficient
945 : * justification for skipping further checks, we should still be able
946 : * to perform sensibly.
947 : */
948 : }
949 :
950 1076246 : if (HeapTupleHeaderIsHeapOnly(tuphdr) &&
951 11790 : ((tuphdr->t_infomask & HEAP_UPDATED) == 0))
952 : {
953 2 : report_corruption(ctx,
954 : psprintf("tuple is heap only, but not the result of an update"));
955 :
956 : /* Here again, we can still perform further checks. */
957 : }
958 :
959 1076246 : if (infomask & HEAP_HASNULL)
960 482386 : expected_hoff = MAXALIGN(SizeofHeapTupleHeader + BITMAPLEN(ctx->natts));
961 : else
962 593860 : expected_hoff = MAXALIGN(SizeofHeapTupleHeader);
963 1076246 : if (ctx->tuphdr->t_hoff != expected_hoff)
964 : {
965 10 : if ((infomask & HEAP_HASNULL) && ctx->natts == 1)
966 0 : report_corruption(ctx,
967 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (1 attribute, has nulls)",
968 0 : expected_hoff, ctx->tuphdr->t_hoff));
969 10 : else if ((infomask & HEAP_HASNULL))
970 2 : report_corruption(ctx,
971 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (%u attributes, has nulls)",
972 2 : expected_hoff, ctx->tuphdr->t_hoff, ctx->natts));
973 8 : else if (ctx->natts == 1)
974 0 : report_corruption(ctx,
975 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (1 attribute, no nulls)",
976 0 : expected_hoff, ctx->tuphdr->t_hoff));
977 : else
978 8 : report_corruption(ctx,
979 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (%u attributes, no nulls)",
980 8 : expected_hoff, ctx->tuphdr->t_hoff, ctx->natts));
981 10 : result = false;
982 : }
983 :
984 1076246 : return result;
985 : }
986 :
987 : /*
988 : * Checks tuple visibility so we know which further checks are safe to
989 : * perform.
990 : *
991 : * If a tuple could have been inserted by a transaction that also added a
992 : * column to the table, but which ultimately did not commit, or which has not
993 : * yet committed, then the table's current TupleDesc might differ from the one
994 : * used to construct this tuple, so we must not check it.
995 : *
996 : * As a special case, if our own transaction inserted the tuple, even if we
997 : * added a column to the table, our TupleDesc should match. We could check the
998 : * tuple, but choose not to do so.
999 : *
1000 : * If a tuple has been updated or deleted, we can still read the old tuple for
1001 : * corruption checking purposes, as long as we are careful about concurrent
1002 : * vacuums. The main table tuple itself cannot be vacuumed away because we
1003 : * hold a buffer lock on the page, but if the deleting transaction is older
1004 : * than our transaction snapshot's xmin, then vacuum could remove the toast at
1005 : * any time, so we must not try to follow TOAST pointers.
1006 : *
1007 : * If xmin or xmax values are older than can be checked against clog, or appear
1008 : * to be in the future (possibly due to wrap-around), then we cannot make a
1009 : * determination about the visibility of the tuple, so we skip further checks.
1010 : *
1011 : * Returns true if the tuple itself should be checked, false otherwise. Sets
1012 : * ctx->tuple_could_be_pruned if the tuple -- and thus also any associated
1013 : * TOAST tuples -- are eligible for pruning.
1014 : *
1015 : * Sets *xmin_commit_status_ok to true if the commit status of xmin is known
1016 : * and false otherwise. If it's set to true, then also set *xmin_commit_status
1017 : * to the actual commit status.
1018 : */
1019 : static bool
1020 1076236 : check_tuple_visibility(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
1021 : XidCommitStatus *xmin_commit_status)
1022 : {
1023 : TransactionId xmin;
1024 : TransactionId xvac;
1025 : TransactionId xmax;
1026 : XidCommitStatus xmin_status;
1027 : XidCommitStatus xvac_status;
1028 : XidCommitStatus xmax_status;
1029 1076236 : HeapTupleHeader tuphdr = ctx->tuphdr;
1030 :
1031 1076236 : ctx->tuple_could_be_pruned = true; /* have not yet proven otherwise */
1032 1076236 : *xmin_commit_status_ok = false; /* have not yet proven otherwise */
1033 :
1034 : /* If xmin is normal, it should be within valid range */
1035 1076236 : xmin = HeapTupleHeaderGetXmin(tuphdr);
1036 1076236 : switch (get_xid_status(xmin, ctx, &xmin_status))
1037 : {
1038 0 : case XID_INVALID:
1039 : /* Could be the result of a speculative insertion that aborted. */
1040 0 : return false;
1041 1076228 : case XID_BOUNDS_OK:
1042 1076228 : *xmin_commit_status_ok = true;
1043 1076228 : *xmin_commit_status = xmin_status;
1044 1076228 : break;
1045 2 : case XID_IN_FUTURE:
1046 2 : report_corruption(ctx,
1047 : psprintf("xmin %u equals or exceeds next valid transaction ID %u:%u",
1048 : xmin,
1049 2 : EpochFromFullTransactionId(ctx->next_fxid),
1050 2 : XidFromFullTransactionId(ctx->next_fxid)));
1051 2 : return false;
1052 4 : case XID_PRECEDES_CLUSTERMIN:
1053 4 : report_corruption(ctx,
1054 : psprintf("xmin %u precedes oldest valid transaction ID %u:%u",
1055 : xmin,
1056 4 : EpochFromFullTransactionId(ctx->oldest_fxid),
1057 4 : XidFromFullTransactionId(ctx->oldest_fxid)));
1058 4 : return false;
1059 2 : case XID_PRECEDES_RELMIN:
1060 2 : report_corruption(ctx,
1061 : psprintf("xmin %u precedes relation freeze threshold %u:%u",
1062 : xmin,
1063 2 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1064 2 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1065 2 : return false;
1066 : }
1067 :
1068 : /*
1069 : * Has inserting transaction committed?
1070 : */
1071 1076228 : if (!HeapTupleHeaderXminCommitted(tuphdr))
1072 : {
1073 28920 : if (HeapTupleHeaderXminInvalid(tuphdr))
1074 0 : return false; /* inserter aborted, don't check */
1075 : /* Used by pre-9.0 binary upgrades */
1076 28920 : else if (tuphdr->t_infomask & HEAP_MOVED_OFF)
1077 : {
1078 0 : xvac = HeapTupleHeaderGetXvac(tuphdr);
1079 :
1080 0 : switch (get_xid_status(xvac, ctx, &xvac_status))
1081 : {
1082 0 : case XID_INVALID:
1083 0 : report_corruption(ctx,
1084 : pstrdup("old-style VACUUM FULL transaction ID for moved off tuple is invalid"));
1085 0 : return false;
1086 0 : case XID_IN_FUTURE:
1087 0 : report_corruption(ctx,
1088 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple equals or exceeds next valid transaction ID %u:%u",
1089 : xvac,
1090 0 : EpochFromFullTransactionId(ctx->next_fxid),
1091 0 : XidFromFullTransactionId(ctx->next_fxid)));
1092 0 : return false;
1093 0 : case XID_PRECEDES_RELMIN:
1094 0 : report_corruption(ctx,
1095 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple precedes relation freeze threshold %u:%u",
1096 : xvac,
1097 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1098 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1099 0 : return false;
1100 0 : case XID_PRECEDES_CLUSTERMIN:
1101 0 : report_corruption(ctx,
1102 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple precedes oldest valid transaction ID %u:%u",
1103 : xvac,
1104 0 : EpochFromFullTransactionId(ctx->oldest_fxid),
1105 0 : XidFromFullTransactionId(ctx->oldest_fxid)));
1106 0 : return false;
1107 0 : case XID_BOUNDS_OK:
1108 0 : break;
1109 : }
1110 :
1111 0 : switch (xvac_status)
1112 : {
1113 0 : case XID_IS_CURRENT_XID:
1114 0 : report_corruption(ctx,
1115 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple matches our current transaction ID",
1116 : xvac));
1117 0 : return false;
1118 0 : case XID_IN_PROGRESS:
1119 0 : report_corruption(ctx,
1120 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple appears to be in progress",
1121 : xvac));
1122 0 : return false;
1123 :
1124 0 : case XID_COMMITTED:
1125 :
1126 : /*
1127 : * The tuple is dead, because the xvac transaction moved
1128 : * it off and committed. It's checkable, but also
1129 : * prunable.
1130 : */
1131 0 : return true;
1132 :
1133 0 : case XID_ABORTED:
1134 :
1135 : /*
1136 : * The original xmin must have committed, because the xvac
1137 : * transaction tried to move it later. Since xvac is
1138 : * aborted, whether it's still alive now depends on the
1139 : * status of xmax.
1140 : */
1141 0 : break;
1142 : }
1143 0 : }
1144 : /* Used by pre-9.0 binary upgrades */
1145 28920 : else if (tuphdr->t_infomask & HEAP_MOVED_IN)
1146 : {
1147 0 : xvac = HeapTupleHeaderGetXvac(tuphdr);
1148 :
1149 0 : switch (get_xid_status(xvac, ctx, &xvac_status))
1150 : {
1151 0 : case XID_INVALID:
1152 0 : report_corruption(ctx,
1153 : pstrdup("old-style VACUUM FULL transaction ID for moved in tuple is invalid"));
1154 0 : return false;
1155 0 : case XID_IN_FUTURE:
1156 0 : report_corruption(ctx,
1157 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple equals or exceeds next valid transaction ID %u:%u",
1158 : xvac,
1159 0 : EpochFromFullTransactionId(ctx->next_fxid),
1160 0 : XidFromFullTransactionId(ctx->next_fxid)));
1161 0 : return false;
1162 0 : case XID_PRECEDES_RELMIN:
1163 0 : report_corruption(ctx,
1164 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple precedes relation freeze threshold %u:%u",
1165 : xvac,
1166 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1167 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1168 0 : return false;
1169 0 : case XID_PRECEDES_CLUSTERMIN:
1170 0 : report_corruption(ctx,
1171 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple precedes oldest valid transaction ID %u:%u",
1172 : xvac,
1173 0 : EpochFromFullTransactionId(ctx->oldest_fxid),
1174 0 : XidFromFullTransactionId(ctx->oldest_fxid)));
1175 0 : return false;
1176 0 : case XID_BOUNDS_OK:
1177 0 : break;
1178 : }
1179 :
1180 0 : switch (xvac_status)
1181 : {
1182 0 : case XID_IS_CURRENT_XID:
1183 0 : report_corruption(ctx,
1184 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple matches our current transaction ID",
1185 : xvac));
1186 0 : return false;
1187 0 : case XID_IN_PROGRESS:
1188 0 : report_corruption(ctx,
1189 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple appears to be in progress",
1190 : xvac));
1191 0 : return false;
1192 :
1193 0 : case XID_COMMITTED:
1194 :
1195 : /*
1196 : * The original xmin must have committed, because the xvac
1197 : * transaction moved it later. Whether it's still alive
1198 : * now depends on the status of xmax.
1199 : */
1200 0 : break;
1201 :
1202 0 : case XID_ABORTED:
1203 :
1204 : /*
1205 : * The tuple is dead, because the xvac transaction moved
1206 : * it off and committed. It's checkable, but also
1207 : * prunable.
1208 : */
1209 0 : return true;
1210 : }
1211 0 : }
1212 28920 : else if (xmin_status != XID_COMMITTED)
1213 : {
1214 : /*
1215 : * Inserting transaction is not in progress, and not committed, so
1216 : * it might have changed the TupleDesc in ways we don't know
1217 : * about. Thus, don't try to check the tuple structure.
1218 : *
1219 : * If xmin_status happens to be XID_IS_CURRENT_XID, then in theory
1220 : * any such DDL changes ought to be visible to us, so perhaps we
1221 : * could check anyway in that case. But, for now, let's be
1222 : * conservative and treat this like any other uncommitted insert.
1223 : */
1224 14 : return false;
1225 : }
1226 : }
1227 :
1228 : /*
1229 : * Okay, the inserter committed, so it was good at some point. Now what
1230 : * about the deleting transaction?
1231 : */
1232 :
1233 1076214 : if (tuphdr->t_infomask & HEAP_XMAX_IS_MULTI)
1234 : {
1235 : /*
1236 : * xmax is a multixact, so sanity-check the MXID. Note that we do this
1237 : * prior to checking for HEAP_XMAX_INVALID or
1238 : * HEAP_XMAX_IS_LOCKED_ONLY. This might therefore complain about
1239 : * things that wouldn't actually be a problem during a normal scan,
1240 : * but eventually we're going to have to freeze, and that process will
1241 : * ignore hint bits.
1242 : *
1243 : * Even if the MXID is out of range, we still know that the original
1244 : * insert committed, so we can check the tuple itself. However, we
1245 : * can't rule out the possibility that this tuple is dead, so don't
1246 : * clear ctx->tuple_could_be_pruned. Possibly we should go ahead and
1247 : * clear that flag anyway if HEAP_XMAX_INVALID is set or if
1248 : * HEAP_XMAX_IS_LOCKED_ONLY is true, but for now we err on the side of
1249 : * avoiding possibly-bogus complaints about missing TOAST entries.
1250 : */
1251 116 : xmax = HeapTupleHeaderGetRawXmax(tuphdr);
1252 116 : switch (check_mxid_valid_in_rel(xmax, ctx))
1253 : {
1254 0 : case XID_INVALID:
1255 0 : report_corruption(ctx,
1256 : pstrdup("multitransaction ID is invalid"));
1257 0 : return true;
1258 2 : case XID_PRECEDES_RELMIN:
1259 2 : report_corruption(ctx,
1260 : psprintf("multitransaction ID %u precedes relation minimum multitransaction ID threshold %u",
1261 : xmax, ctx->relminmxid));
1262 2 : return true;
1263 0 : case XID_PRECEDES_CLUSTERMIN:
1264 0 : report_corruption(ctx,
1265 : psprintf("multitransaction ID %u precedes oldest valid multitransaction ID threshold %u",
1266 : xmax, ctx->oldest_mxact));
1267 0 : return true;
1268 2 : case XID_IN_FUTURE:
1269 2 : report_corruption(ctx,
1270 : psprintf("multitransaction ID %u equals or exceeds next valid multitransaction ID %u",
1271 : xmax,
1272 : ctx->next_mxact));
1273 2 : return true;
1274 112 : case XID_BOUNDS_OK:
1275 112 : break;
1276 : }
1277 1076098 : }
1278 :
1279 1076210 : if (tuphdr->t_infomask & HEAP_XMAX_INVALID)
1280 : {
1281 : /*
1282 : * This tuple is live. A concurrently running transaction could
1283 : * delete it before we get around to checking the toast, but any such
1284 : * running transaction is surely not less than our safe_xmin, so the
1285 : * toast cannot be vacuumed out from under us.
1286 : */
1287 1073618 : ctx->tuple_could_be_pruned = false;
1288 1073618 : return true;
1289 : }
1290 :
1291 2592 : if (HEAP_XMAX_IS_LOCKED_ONLY(tuphdr->t_infomask))
1292 : {
1293 : /*
1294 : * "Deleting" xact really only locked it, so the tuple is live in any
1295 : * case. As above, a concurrently running transaction could delete
1296 : * it, but it cannot be vacuumed out from under us.
1297 : */
1298 56 : ctx->tuple_could_be_pruned = false;
1299 56 : return true;
1300 : }
1301 :
1302 2536 : if (tuphdr->t_infomask & HEAP_XMAX_IS_MULTI)
1303 : {
1304 : /*
1305 : * We already checked above that this multixact is within limits for
1306 : * this table. Now check the update xid from this multixact.
1307 : */
1308 56 : xmax = HeapTupleGetUpdateXid(tuphdr);
1309 56 : switch (get_xid_status(xmax, ctx, &xmax_status))
1310 : {
1311 0 : case XID_INVALID:
1312 : /* not LOCKED_ONLY, so it has to have an xmax */
1313 0 : report_corruption(ctx,
1314 : pstrdup("update xid is invalid"));
1315 0 : return true;
1316 0 : case XID_IN_FUTURE:
1317 0 : report_corruption(ctx,
1318 : psprintf("update xid %u equals or exceeds next valid transaction ID %u:%u",
1319 : xmax,
1320 0 : EpochFromFullTransactionId(ctx->next_fxid),
1321 0 : XidFromFullTransactionId(ctx->next_fxid)));
1322 0 : return true;
1323 0 : case XID_PRECEDES_RELMIN:
1324 0 : report_corruption(ctx,
1325 : psprintf("update xid %u precedes relation freeze threshold %u:%u",
1326 : xmax,
1327 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1328 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1329 0 : return true;
1330 0 : case XID_PRECEDES_CLUSTERMIN:
1331 0 : report_corruption(ctx,
1332 : psprintf("update xid %u precedes oldest valid transaction ID %u:%u",
1333 : xmax,
1334 0 : EpochFromFullTransactionId(ctx->oldest_fxid),
1335 0 : XidFromFullTransactionId(ctx->oldest_fxid)));
1336 0 : return true;
1337 56 : case XID_BOUNDS_OK:
1338 56 : break;
1339 : }
1340 :
1341 56 : switch (xmax_status)
1342 : {
1343 0 : case XID_IS_CURRENT_XID:
1344 : case XID_IN_PROGRESS:
1345 :
1346 : /*
1347 : * The delete is in progress, so it cannot be visible to our
1348 : * snapshot.
1349 : */
1350 0 : ctx->tuple_could_be_pruned = false;
1351 0 : break;
1352 56 : case XID_COMMITTED:
1353 :
1354 : /*
1355 : * The delete committed. Whether the toast can be vacuumed
1356 : * away depends on how old the deleting transaction is.
1357 : */
1358 56 : ctx->tuple_could_be_pruned = TransactionIdPrecedes(xmax,
1359 : ctx->safe_xmin);
1360 56 : break;
1361 0 : case XID_ABORTED:
1362 :
1363 : /*
1364 : * The delete aborted or crashed. The tuple is still live.
1365 : */
1366 0 : ctx->tuple_could_be_pruned = false;
1367 0 : break;
1368 : }
1369 :
1370 : /* Tuple itself is checkable even if it's dead. */
1371 56 : return true;
1372 : }
1373 :
1374 : /* xmax is an XID, not a MXID. Sanity check it. */
1375 2480 : xmax = HeapTupleHeaderGetRawXmax(tuphdr);
1376 2480 : switch (get_xid_status(xmax, ctx, &xmax_status))
1377 : {
1378 2 : case XID_INVALID:
1379 2 : ctx->tuple_could_be_pruned = false;
1380 2 : return true;
1381 0 : case XID_IN_FUTURE:
1382 0 : report_corruption(ctx,
1383 : psprintf("xmax %u equals or exceeds next valid transaction ID %u:%u",
1384 : xmax,
1385 0 : EpochFromFullTransactionId(ctx->next_fxid),
1386 0 : XidFromFullTransactionId(ctx->next_fxid)));
1387 0 : return false; /* corrupt */
1388 0 : case XID_PRECEDES_RELMIN:
1389 0 : report_corruption(ctx,
1390 : psprintf("xmax %u precedes relation freeze threshold %u:%u",
1391 : xmax,
1392 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1393 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1394 0 : return false; /* corrupt */
1395 2 : case XID_PRECEDES_CLUSTERMIN:
1396 2 : report_corruption(ctx,
1397 : psprintf("xmax %u precedes oldest valid transaction ID %u:%u",
1398 : xmax,
1399 2 : EpochFromFullTransactionId(ctx->oldest_fxid),
1400 2 : XidFromFullTransactionId(ctx->oldest_fxid)));
1401 2 : return false; /* corrupt */
1402 2476 : case XID_BOUNDS_OK:
1403 2476 : break;
1404 : }
1405 :
1406 : /*
1407 : * Whether the toast can be vacuumed away depends on how old the deleting
1408 : * transaction is.
1409 : */
1410 2476 : switch (xmax_status)
1411 : {
1412 0 : case XID_IS_CURRENT_XID:
1413 : case XID_IN_PROGRESS:
1414 :
1415 : /*
1416 : * The delete is in progress, so it cannot be visible to our
1417 : * snapshot.
1418 : */
1419 0 : ctx->tuple_could_be_pruned = false;
1420 0 : break;
1421 :
1422 2470 : case XID_COMMITTED:
1423 :
1424 : /*
1425 : * The delete committed. Whether the toast can be vacuumed away
1426 : * depends on how old the deleting transaction is.
1427 : */
1428 2470 : ctx->tuple_could_be_pruned = TransactionIdPrecedes(xmax,
1429 : ctx->safe_xmin);
1430 2470 : break;
1431 :
1432 6 : case XID_ABORTED:
1433 :
1434 : /*
1435 : * The delete aborted or crashed. The tuple is still live.
1436 : */
1437 6 : ctx->tuple_could_be_pruned = false;
1438 6 : break;
1439 : }
1440 :
1441 : /* Tuple itself is checkable even if it's dead. */
1442 2476 : return true;
1443 : }
1444 :
1445 :
1446 : /*
1447 : * Check the current toast tuple against the state tracked in ctx, recording
1448 : * any corruption found in ctx->tupstore.
1449 : *
1450 : * This is not equivalent to running verify_heapam on the toast table itself,
1451 : * and is not hardened against corruption of the toast table. Rather, when
1452 : * validating a toasted attribute in the main table, the sequence of toast
1453 : * tuples that store the toasted value are retrieved and checked in order, with
1454 : * each toast tuple being checked against where we are in the sequence, as well
1455 : * as each toast tuple having its varlena structure sanity checked.
1456 : *
1457 : * On entry, *expected_chunk_seq should be the chunk_seq value that we expect
1458 : * to find in toasttup. On exit, it will be updated to the value the next call
1459 : * to this function should expect to see.
1460 : */
1461 : static void
1462 83182 : check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
1463 : ToastedAttribute *ta, int32 *expected_chunk_seq,
1464 : uint32 extsize)
1465 : {
1466 : int32 chunk_seq;
1467 83182 : int32 last_chunk_seq = (extsize - 1) / TOAST_MAX_CHUNK_SIZE;
1468 : Pointer chunk;
1469 : bool isnull;
1470 : int32 chunksize;
1471 : int32 expected_size;
1472 :
1473 : /* Sanity-check the sequence number. */
1474 83182 : chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2,
1475 83182 : ctx->toast_rel->rd_att, &isnull));
1476 83182 : if (isnull)
1477 : {
1478 0 : report_toast_corruption(ctx, ta,
1479 : psprintf("toast value %u has toast chunk with null sequence number",
1480 : ta->toast_pointer.va_valueid));
1481 0 : return;
1482 : }
1483 83182 : if (chunk_seq != *expected_chunk_seq)
1484 : {
1485 : /* Either the TOAST index is corrupt, or we don't have all chunks. */
1486 0 : report_toast_corruption(ctx, ta,
1487 : psprintf("toast value %u index scan returned chunk %d when expecting chunk %d",
1488 : ta->toast_pointer.va_valueid,
1489 : chunk_seq, *expected_chunk_seq));
1490 : }
1491 83182 : *expected_chunk_seq = chunk_seq + 1;
1492 :
1493 : /* Sanity-check the chunk data. */
1494 83182 : chunk = DatumGetPointer(fastgetattr(toasttup, 3,
1495 83182 : ctx->toast_rel->rd_att, &isnull));
1496 83182 : if (isnull)
1497 : {
1498 0 : report_toast_corruption(ctx, ta,
1499 : psprintf("toast value %u chunk %d has null data",
1500 : ta->toast_pointer.va_valueid,
1501 : chunk_seq));
1502 0 : return;
1503 : }
1504 83182 : if (!VARATT_IS_EXTENDED(chunk))
1505 83182 : chunksize = VARSIZE(chunk) - VARHDRSZ;
1506 0 : else if (VARATT_IS_SHORT(chunk))
1507 : {
1508 : /*
1509 : * could happen due to heap_form_tuple doing its thing
1510 : */
1511 0 : chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT;
1512 : }
1513 : else
1514 : {
1515 : /* should never happen */
1516 0 : uint32 header = ((varattrib_4b *) chunk)->va_4byte.va_header;
1517 :
1518 0 : report_toast_corruption(ctx, ta,
1519 : psprintf("toast value %u chunk %d has invalid varlena header %0x",
1520 : ta->toast_pointer.va_valueid,
1521 : chunk_seq, header));
1522 0 : return;
1523 : }
1524 :
1525 : /*
1526 : * Some checks on the data we've found
1527 : */
1528 83182 : if (chunk_seq > last_chunk_seq)
1529 : {
1530 0 : report_toast_corruption(ctx, ta,
1531 : psprintf("toast value %u chunk %d follows last expected chunk %d",
1532 : ta->toast_pointer.va_valueid,
1533 : chunk_seq, last_chunk_seq));
1534 0 : return;
1535 : }
1536 :
1537 83182 : expected_size = chunk_seq < last_chunk_seq ? TOAST_MAX_CHUNK_SIZE
1538 23622 : : extsize - (last_chunk_seq * TOAST_MAX_CHUNK_SIZE);
1539 :
1540 83182 : if (chunksize != expected_size)
1541 0 : report_toast_corruption(ctx, ta,
1542 : psprintf("toast value %u chunk %d has size %u, but expected size %u",
1543 : ta->toast_pointer.va_valueid,
1544 : chunk_seq, chunksize, expected_size));
1545 : }
1546 :
1547 : /*
1548 : * Check the current attribute as tracked in ctx, recording any corruption
1549 : * found in ctx->tupstore.
1550 : *
1551 : * This function follows the logic performed by heap_deform_tuple(), and in the
1552 : * case of a toasted value, optionally stores the toast pointer so later it can
1553 : * be checked following the logic of detoast_external_attr(), checking for any
1554 : * conditions that would result in either of those functions Asserting or
1555 : * crashing the backend. The checks performed by Asserts present in those two
1556 : * functions are also performed here and in check_toasted_attribute. In cases
1557 : * where those two functions are a bit cavalier in their assumptions about data
1558 : * being correct, we perform additional checks not present in either of those
1559 : * two functions. Where some condition is checked in both of those functions,
1560 : * we perform it here twice, as we parallel the logical flow of those two
1561 : * functions. The presence of duplicate checks seems a reasonable price to pay
1562 : * for keeping this code tightly coupled with the code it protects.
1563 : *
1564 : * Returns true if the tuple attribute is sane enough for processing to
1565 : * continue on to the next attribute, false otherwise.
1566 : */
1567 : static bool
1568 15603264 : check_tuple_attribute(HeapCheckContext *ctx)
1569 : {
1570 : Datum attdatum;
1571 : struct varlena *attr;
1572 : char *tp; /* pointer to the tuple data */
1573 : uint16 infomask;
1574 : Form_pg_attribute thisatt;
1575 : struct varatt_external toast_pointer;
1576 :
1577 15603264 : infomask = ctx->tuphdr->t_infomask;
1578 15603264 : thisatt = TupleDescAttr(RelationGetDescr(ctx->rel), ctx->attnum);
1579 :
1580 15603264 : tp = (char *) ctx->tuphdr + ctx->tuphdr->t_hoff;
1581 :
1582 15603264 : if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len)
1583 : {
1584 0 : report_corruption(ctx,
1585 : psprintf("attribute with length %u starts at offset %u beyond total tuple length %u",
1586 0 : thisatt->attlen,
1587 0 : ctx->tuphdr->t_hoff + ctx->offset,
1588 0 : ctx->lp_len));
1589 0 : return false;
1590 : }
1591 :
1592 : /* Skip null values */
1593 15603264 : if (infomask & HEAP_HASNULL && att_isnull(ctx->attnum, ctx->tuphdr->t_bits))
1594 2646184 : return true;
1595 :
1596 : /* Skip non-varlena values, but update offset first */
1597 12957080 : if (thisatt->attlen != -1)
1598 : {
1599 11898526 : ctx->offset = att_align_nominal(ctx->offset, thisatt->attalign);
1600 11898526 : ctx->offset = att_addlength_pointer(ctx->offset, thisatt->attlen,
1601 : tp + ctx->offset);
1602 11898526 : if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len)
1603 : {
1604 0 : report_corruption(ctx,
1605 : psprintf("attribute with length %u ends at offset %u beyond total tuple length %u",
1606 0 : thisatt->attlen,
1607 0 : ctx->tuphdr->t_hoff + ctx->offset,
1608 0 : ctx->lp_len));
1609 0 : return false;
1610 : }
1611 11898526 : return true;
1612 : }
1613 :
1614 : /* Ok, we're looking at a varlena attribute. */
1615 1058554 : ctx->offset = att_align_pointer(ctx->offset, thisatt->attalign, -1,
1616 : tp + ctx->offset);
1617 :
1618 : /* Get the (possibly corrupt) varlena datum */
1619 1058554 : attdatum = fetchatt(thisatt, tp + ctx->offset);
1620 :
1621 : /*
1622 : * We have the datum, but we cannot decode it carelessly, as it may still
1623 : * be corrupt.
1624 : */
1625 :
1626 : /*
1627 : * Check that VARTAG_SIZE won't hit an Assert on a corrupt va_tag before
1628 : * risking a call into att_addlength_pointer
1629 : */
1630 1058554 : if (VARATT_IS_EXTERNAL(tp + ctx->offset))
1631 : {
1632 52504 : uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
1633 :
1634 52504 : if (va_tag != VARTAG_ONDISK)
1635 : {
1636 0 : report_corruption(ctx,
1637 : psprintf("toasted attribute has unexpected TOAST tag %u",
1638 : va_tag));
1639 : /* We can't know where the next attribute begins */
1640 0 : return false;
1641 : }
1642 : }
1643 :
1644 : /* Ok, should be safe now */
1645 1058554 : ctx->offset = att_addlength_pointer(ctx->offset, thisatt->attlen,
1646 : tp + ctx->offset);
1647 :
1648 1058554 : if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len)
1649 : {
1650 2 : report_corruption(ctx,
1651 : psprintf("attribute with length %u ends at offset %u beyond total tuple length %u",
1652 2 : thisatt->attlen,
1653 2 : ctx->tuphdr->t_hoff + ctx->offset,
1654 2 : ctx->lp_len));
1655 :
1656 2 : return false;
1657 : }
1658 :
1659 : /*
1660 : * heap_deform_tuple would be done with this attribute at this point,
1661 : * having stored it in values[], and would continue to the next attribute.
1662 : * We go further, because we need to check if the toast datum is corrupt.
1663 : */
1664 :
1665 1058552 : attr = (struct varlena *) DatumGetPointer(attdatum);
1666 :
1667 : /*
1668 : * Now we follow the logic of detoast_external_attr(), with the same
1669 : * caveats about being paranoid about corruption.
1670 : */
1671 :
1672 : /* Skip values that are not external */
1673 1058552 : if (!VARATT_IS_EXTERNAL(attr))
1674 1006048 : return true;
1675 :
1676 : /* It is external, and we're looking at a page on disk */
1677 :
1678 : /*
1679 : * Must copy attr into toast_pointer for alignment considerations
1680 : */
1681 52504 : VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
1682 :
1683 : /* Toasted attributes too large to be untoasted should never be stored */
1684 52504 : if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT)
1685 0 : report_corruption(ctx,
1686 : psprintf("toast value %u rawsize %d exceeds limit %d",
1687 : toast_pointer.va_valueid,
1688 : toast_pointer.va_rawsize,
1689 : VARLENA_SIZE_LIMIT));
1690 :
1691 52504 : if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
1692 : {
1693 : ToastCompressionId cmid;
1694 4048 : bool valid = false;
1695 :
1696 : /* Compressed attributes should have a valid compression method */
1697 4048 : cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
1698 4048 : switch (cmid)
1699 : {
1700 : /* List of all valid compression method IDs */
1701 4048 : case TOAST_PGLZ_COMPRESSION_ID:
1702 : case TOAST_LZ4_COMPRESSION_ID:
1703 4048 : valid = true;
1704 4048 : break;
1705 :
1706 : /* Recognized but invalid compression method ID */
1707 0 : case TOAST_INVALID_COMPRESSION_ID:
1708 0 : break;
1709 :
1710 : /* Intentionally no default here */
1711 : }
1712 4048 : if (!valid)
1713 0 : report_corruption(ctx,
1714 : psprintf("toast value %u has invalid compression method id %d",
1715 : toast_pointer.va_valueid, cmid));
1716 : }
1717 :
1718 : /* The tuple header better claim to contain toasted values */
1719 52504 : if (!(infomask & HEAP_HASEXTERNAL))
1720 : {
1721 0 : report_corruption(ctx,
1722 : psprintf("toast value %u is external but tuple header flag HEAP_HASEXTERNAL not set",
1723 : toast_pointer.va_valueid));
1724 0 : return true;
1725 : }
1726 :
1727 : /* The relation better have a toast table */
1728 52504 : if (!ctx->rel->rd_rel->reltoastrelid)
1729 : {
1730 0 : report_corruption(ctx,
1731 : psprintf("toast value %u is external but relation has no toast relation",
1732 : toast_pointer.va_valueid));
1733 0 : return true;
1734 : }
1735 :
1736 : /* If we were told to skip toast checking, then we're done. */
1737 52504 : if (ctx->toast_rel == NULL)
1738 28858 : return true;
1739 :
1740 : /*
1741 : * If this tuple is eligible to be pruned, we cannot check the toast.
1742 : * Otherwise, we push a copy of the toast tuple so we can check it after
1743 : * releasing the main table buffer lock.
1744 : */
1745 23646 : if (!ctx->tuple_could_be_pruned)
1746 : {
1747 : ToastedAttribute *ta;
1748 :
1749 23642 : ta = (ToastedAttribute *) palloc0(sizeof(ToastedAttribute));
1750 :
1751 23642 : VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
1752 23642 : ta->blkno = ctx->blkno;
1753 23642 : ta->offnum = ctx->offnum;
1754 23642 : ta->attnum = ctx->attnum;
1755 23642 : ctx->toasted_attributes = lappend(ctx->toasted_attributes, ta);
1756 : }
1757 :
1758 23646 : return true;
1759 : }
1760 :
1761 : /*
1762 : * For each attribute collected in ctx->toasted_attributes, look up the value
1763 : * in the toast table and perform checks on it. This function should only be
1764 : * called on toast pointers which cannot be vacuumed away during our
1765 : * processing.
1766 : */
1767 : static void
1768 23630 : check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
1769 : {
1770 : SnapshotData SnapshotToast;
1771 : ScanKeyData toastkey;
1772 : SysScanDesc toastscan;
1773 : bool found_toasttup;
1774 : HeapTuple toasttup;
1775 : uint32 extsize;
1776 23630 : int32 expected_chunk_seq = 0;
1777 : int32 last_chunk_seq;
1778 :
1779 23630 : extsize = VARATT_EXTERNAL_GET_EXTSIZE(ta->toast_pointer);
1780 23630 : last_chunk_seq = (extsize - 1) / TOAST_MAX_CHUNK_SIZE;
1781 :
1782 : /*
1783 : * Setup a scan key to find chunks in toast table with matching va_valueid
1784 : */
1785 23630 : ScanKeyInit(&toastkey,
1786 : (AttrNumber) 1,
1787 : BTEqualStrategyNumber, F_OIDEQ,
1788 : ObjectIdGetDatum(ta->toast_pointer.va_valueid));
1789 :
1790 : /*
1791 : * Check if any chunks for this toasted object exist in the toast table,
1792 : * accessible via the index.
1793 : */
1794 23630 : init_toast_snapshot(&SnapshotToast);
1795 23630 : toastscan = systable_beginscan_ordered(ctx->toast_rel,
1796 : ctx->valid_toast_index,
1797 : &SnapshotToast, 1,
1798 : &toastkey);
1799 23630 : found_toasttup = false;
1800 130436 : while ((toasttup =
1801 106812 : systable_getnext_ordered(toastscan,
1802 : ForwardScanDirection)) != NULL)
1803 : {
1804 83182 : found_toasttup = true;
1805 83182 : check_toast_tuple(toasttup, ctx, ta, &expected_chunk_seq, extsize);
1806 : }
1807 23624 : systable_endscan_ordered(toastscan);
1808 :
1809 23624 : if (!found_toasttup)
1810 2 : report_toast_corruption(ctx, ta,
1811 : psprintf("toast value %u not found in toast table",
1812 : ta->toast_pointer.va_valueid));
1813 23622 : else if (expected_chunk_seq <= last_chunk_seq)
1814 0 : report_toast_corruption(ctx, ta,
1815 : psprintf("toast value %u was expected to end at chunk %d, but ended while expecting chunk %d",
1816 : ta->toast_pointer.va_valueid,
1817 : last_chunk_seq, expected_chunk_seq));
1818 23624 : }
1819 :
1820 : /*
1821 : * Check the current tuple as tracked in ctx, recording any corruption found in
1822 : * ctx->tupstore.
1823 : *
1824 : * We return some information about the status of xmin to aid in validating
1825 : * update chains.
1826 : */
1827 : static void
1828 1076246 : check_tuple(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
1829 : XidCommitStatus *xmin_commit_status)
1830 : {
1831 : /*
1832 : * Check various forms of tuple header corruption, and if the header is
1833 : * too corrupt, do not continue with other checks.
1834 : */
1835 1076246 : if (!check_tuple_header(ctx))
1836 10 : return;
1837 :
1838 : /*
1839 : * Check tuple visibility. If the inserting transaction aborted, we
1840 : * cannot assume our relation description matches the tuple structure, and
1841 : * therefore cannot check it.
1842 : */
1843 1076236 : if (!check_tuple_visibility(ctx, xmin_commit_status_ok,
1844 : xmin_commit_status))
1845 24 : return;
1846 :
1847 : /*
1848 : * The tuple is visible, so it must be compatible with the current version
1849 : * of the relation descriptor. It might have fewer columns than are
1850 : * present in the relation descriptor, but it cannot have more.
1851 : */
1852 1076212 : if (RelationGetDescr(ctx->rel)->natts < ctx->natts)
1853 : {
1854 4 : report_corruption(ctx,
1855 : psprintf("number of attributes %u exceeds maximum expected for table %u",
1856 : ctx->natts,
1857 4 : RelationGetDescr(ctx->rel)->natts));
1858 4 : return;
1859 : }
1860 :
1861 : /*
1862 : * Check each attribute unless we hit corruption that confuses what to do
1863 : * next, at which point we abort further attribute checks for this tuple.
1864 : * Note that we don't abort for all types of corruption, only for those
1865 : * types where we don't know how to continue. We also don't abort the
1866 : * checking of toasted attributes collected from the tuple prior to
1867 : * aborting. Those will still be checked later along with other toasted
1868 : * attributes collected from the page.
1869 : */
1870 1076208 : ctx->offset = 0;
1871 16679470 : for (ctx->attnum = 0; ctx->attnum < ctx->natts; ctx->attnum++)
1872 15603264 : if (!check_tuple_attribute(ctx))
1873 2 : break; /* cannot continue */
1874 :
1875 : /* revert attnum to -1 until we again examine individual attributes */
1876 1076208 : ctx->attnum = -1;
1877 : }
1878 :
1879 : /*
1880 : * Convert a TransactionId into a FullTransactionId using our cached values of
1881 : * the valid transaction ID range. It is the caller's responsibility to have
1882 : * already updated the cached values, if necessary.
1883 : */
1884 : static FullTransactionId
1885 150526 : FullTransactionIdFromXidAndCtx(TransactionId xid, const HeapCheckContext *ctx)
1886 : {
1887 : uint64 nextfxid_i;
1888 : int32 diff;
1889 : FullTransactionId fxid;
1890 :
1891 : Assert(TransactionIdIsNormal(ctx->next_xid));
1892 : Assert(FullTransactionIdIsNormal(ctx->next_fxid));
1893 : Assert(XidFromFullTransactionId(ctx->next_fxid) == ctx->next_xid);
1894 :
1895 150526 : if (!TransactionIdIsNormal(xid))
1896 384 : return FullTransactionIdFromEpochAndXid(0, xid);
1897 :
1898 150142 : nextfxid_i = U64FromFullTransactionId(ctx->next_fxid);
1899 :
1900 : /* compute the 32bit modulo difference */
1901 150142 : diff = (int32) (ctx->next_xid - xid);
1902 :
1903 : /*
1904 : * In cases of corruption we might see a 32bit xid that is before epoch 0.
1905 : * We can't represent that as a 64bit xid, due to 64bit xids being
1906 : * unsigned integers, without the modulo arithmetic of 32bit xid. There's
1907 : * no really nice way to deal with that, but it works ok enough to use
1908 : * FirstNormalFullTransactionId in that case, as a freshly initdb'd
1909 : * cluster already has a newer horizon.
1910 : */
1911 150142 : if (diff > 0 && (nextfxid_i - FirstNormalTransactionId) < (int64) diff)
1912 : {
1913 : Assert(EpochFromFullTransactionId(ctx->next_fxid) == 0);
1914 8 : fxid = FirstNormalFullTransactionId;
1915 : }
1916 : else
1917 150134 : fxid = FullTransactionIdFromU64(nextfxid_i - diff);
1918 :
1919 : Assert(FullTransactionIdIsNormal(fxid));
1920 150142 : return fxid;
1921 : }
1922 :
1923 : /*
1924 : * Update our cached range of valid transaction IDs.
1925 : */
1926 : static void
1927 2960 : update_cached_xid_range(HeapCheckContext *ctx)
1928 : {
1929 : /* Make cached copies */
1930 2960 : LWLockAcquire(XidGenLock, LW_SHARED);
1931 2960 : ctx->next_fxid = TransamVariables->nextXid;
1932 2960 : ctx->oldest_xid = TransamVariables->oldestXid;
1933 2960 : LWLockRelease(XidGenLock);
1934 :
1935 : /* And compute alternate versions of the same */
1936 2960 : ctx->next_xid = XidFromFullTransactionId(ctx->next_fxid);
1937 2960 : ctx->oldest_fxid = FullTransactionIdFromXidAndCtx(ctx->oldest_xid, ctx);
1938 2960 : }
1939 :
1940 : /*
1941 : * Update our cached range of valid multitransaction IDs.
1942 : */
1943 : static void
1944 2956 : update_cached_mxid_range(HeapCheckContext *ctx)
1945 : {
1946 2956 : ReadMultiXactIdRange(&ctx->oldest_mxact, &ctx->next_mxact);
1947 2956 : }
1948 :
1949 : /*
1950 : * Return whether the given FullTransactionId is within our cached valid
1951 : * transaction ID range.
1952 : */
1953 : static inline bool
1954 126336 : fxid_in_cached_range(FullTransactionId fxid, const HeapCheckContext *ctx)
1955 : {
1956 252666 : return (FullTransactionIdPrecedesOrEquals(ctx->oldest_fxid, fxid) &&
1957 126330 : FullTransactionIdPrecedes(fxid, ctx->next_fxid));
1958 : }
1959 :
1960 : /*
1961 : * Checks whether a multitransaction ID is in the cached valid range, returning
1962 : * the nature of the range violation, if any.
1963 : */
1964 : static XidBoundsViolation
1965 120 : check_mxid_in_range(MultiXactId mxid, HeapCheckContext *ctx)
1966 : {
1967 120 : if (!TransactionIdIsValid(mxid))
1968 0 : return XID_INVALID;
1969 120 : if (MultiXactIdPrecedes(mxid, ctx->relminmxid))
1970 4 : return XID_PRECEDES_RELMIN;
1971 116 : if (MultiXactIdPrecedes(mxid, ctx->oldest_mxact))
1972 0 : return XID_PRECEDES_CLUSTERMIN;
1973 116 : if (MultiXactIdPrecedesOrEquals(ctx->next_mxact, mxid))
1974 4 : return XID_IN_FUTURE;
1975 112 : return XID_BOUNDS_OK;
1976 : }
1977 :
1978 : /*
1979 : * Checks whether the given mxid is valid to appear in the heap being checked,
1980 : * returning the nature of the range violation, if any.
1981 : *
1982 : * This function attempts to return quickly by caching the known valid mxid
1983 : * range in ctx. Callers should already have performed the initial setup of
1984 : * the cache prior to the first call to this function.
1985 : */
1986 : static XidBoundsViolation
1987 116 : check_mxid_valid_in_rel(MultiXactId mxid, HeapCheckContext *ctx)
1988 : {
1989 : XidBoundsViolation result;
1990 :
1991 116 : result = check_mxid_in_range(mxid, ctx);
1992 116 : if (result == XID_BOUNDS_OK)
1993 112 : return XID_BOUNDS_OK;
1994 :
1995 : /* The range may have advanced. Recheck. */
1996 4 : update_cached_mxid_range(ctx);
1997 4 : return check_mxid_in_range(mxid, ctx);
1998 : }
1999 :
2000 : /*
2001 : * Checks whether the given transaction ID is (or was recently) valid to appear
2002 : * in the heap being checked, or whether it is too old or too new to appear in
2003 : * the relation, returning information about the nature of the bounds violation.
2004 : *
2005 : * We cache the range of valid transaction IDs. If xid is in that range, we
2006 : * conclude that it is valid, even though concurrent changes to the table might
2007 : * invalidate it under certain corrupt conditions. (For example, if the table
2008 : * contains corrupt all-frozen bits, a concurrent vacuum might skip the page(s)
2009 : * containing the xid and then truncate clog and advance the relfrozenxid
2010 : * beyond xid.) Reporting the xid as valid under such conditions seems
2011 : * acceptable, since if we had checked it earlier in our scan it would have
2012 : * truly been valid at that time.
2013 : *
2014 : * If the status argument is not NULL, and if and only if the transaction ID
2015 : * appears to be valid in this relation, the status argument will be set with
2016 : * the commit status of the transaction ID.
2017 : */
2018 : static XidBoundsViolation
2019 1078772 : get_xid_status(TransactionId xid, HeapCheckContext *ctx,
2020 : XidCommitStatus *status)
2021 : {
2022 : FullTransactionId fxid;
2023 : FullTransactionId clog_horizon;
2024 :
2025 : /* Quick check for special xids */
2026 1078772 : if (!TransactionIdIsValid(xid))
2027 2 : return XID_INVALID;
2028 1078770 : else if (xid == BootstrapTransactionId || xid == FrozenTransactionId)
2029 : {
2030 952434 : if (status != NULL)
2031 952434 : *status = XID_COMMITTED;
2032 952434 : return XID_BOUNDS_OK;
2033 : }
2034 :
2035 : /* Check if the xid is within bounds */
2036 126336 : fxid = FullTransactionIdFromXidAndCtx(xid, ctx);
2037 126336 : if (!fxid_in_cached_range(fxid, ctx))
2038 : {
2039 : /*
2040 : * We may have been checking against stale values. Update the cached
2041 : * range to be sure, and since we relied on the cached range when we
2042 : * performed the full xid conversion, reconvert.
2043 : */
2044 8 : update_cached_xid_range(ctx);
2045 8 : fxid = FullTransactionIdFromXidAndCtx(xid, ctx);
2046 : }
2047 :
2048 126336 : if (FullTransactionIdPrecedesOrEquals(ctx->next_fxid, fxid))
2049 2 : return XID_IN_FUTURE;
2050 126334 : if (FullTransactionIdPrecedes(fxid, ctx->oldest_fxid))
2051 6 : return XID_PRECEDES_CLUSTERMIN;
2052 126328 : if (FullTransactionIdPrecedes(fxid, ctx->relfrozenfxid))
2053 2 : return XID_PRECEDES_RELMIN;
2054 :
2055 : /* Early return if the caller does not request clog checking */
2056 126326 : if (status == NULL)
2057 0 : return XID_BOUNDS_OK;
2058 :
2059 : /* Early return if we just checked this xid in a prior call */
2060 126326 : if (xid == ctx->cached_xid)
2061 : {
2062 108056 : *status = ctx->cached_status;
2063 108056 : return XID_BOUNDS_OK;
2064 : }
2065 :
2066 18270 : *status = XID_COMMITTED;
2067 18270 : LWLockAcquire(XactTruncationLock, LW_SHARED);
2068 : clog_horizon =
2069 18270 : FullTransactionIdFromXidAndCtx(TransamVariables->oldestClogXid,
2070 : ctx);
2071 18270 : if (FullTransactionIdPrecedesOrEquals(clog_horizon, fxid))
2072 : {
2073 18270 : if (TransactionIdIsCurrentTransactionId(xid))
2074 0 : *status = XID_IS_CURRENT_XID;
2075 18270 : else if (TransactionIdIsInProgress(xid))
2076 4 : *status = XID_IN_PROGRESS;
2077 18266 : else if (TransactionIdDidCommit(xid))
2078 18252 : *status = XID_COMMITTED;
2079 : else
2080 14 : *status = XID_ABORTED;
2081 : }
2082 18270 : LWLockRelease(XactTruncationLock);
2083 18270 : ctx->cached_xid = xid;
2084 18270 : ctx->cached_status = *status;
2085 18270 : return XID_BOUNDS_OK;
2086 : }
|
