Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * verify_heapam.c
4 : * Functions to check postgresql heap relations for corruption
5 : *
6 : * Copyright (c) 2016-2025, PostgreSQL Global Development Group
7 : *
8 : * contrib/amcheck/verify_heapam.c
9 : *-------------------------------------------------------------------------
10 : */
11 : #include "postgres.h"
12 :
13 : #include "access/detoast.h"
14 : #include "access/genam.h"
15 : #include "access/heaptoast.h"
16 : #include "access/multixact.h"
17 : #include "access/relation.h"
18 : #include "access/table.h"
19 : #include "access/toast_internals.h"
20 : #include "access/visibilitymap.h"
21 : #include "access/xact.h"
22 : #include "catalog/pg_am.h"
23 : #include "catalog/pg_class.h"
24 : #include "funcapi.h"
25 : #include "miscadmin.h"
26 : #include "storage/bufmgr.h"
27 : #include "storage/procarray.h"
28 : #include "storage/read_stream.h"
29 : #include "utils/builtins.h"
30 : #include "utils/fmgroids.h"
31 : #include "utils/rel.h"
32 :
33 604 : PG_FUNCTION_INFO_V1(verify_heapam);
34 :
35 : /* The number of columns in tuples returned by verify_heapam */
36 : #define HEAPCHECK_RELATION_COLS 4
37 :
38 : /* The largest valid toast va_rawsize */
39 : #define VARLENA_SIZE_LIMIT 0x3FFFFFFF
40 :
41 : /*
42 : * Despite the name, we use this for reporting problems with both XIDs and
43 : * MXIDs.
44 : */
45 : typedef enum XidBoundsViolation
46 : {
47 : XID_INVALID,
48 : XID_IN_FUTURE,
49 : XID_PRECEDES_CLUSTERMIN,
50 : XID_PRECEDES_RELMIN,
51 : XID_BOUNDS_OK,
52 : } XidBoundsViolation;
53 :
54 : typedef enum XidCommitStatus
55 : {
56 : XID_COMMITTED,
57 : XID_IS_CURRENT_XID,
58 : XID_IN_PROGRESS,
59 : XID_ABORTED,
60 : } XidCommitStatus;
61 :
62 : typedef enum SkipPages
63 : {
64 : SKIP_PAGES_ALL_FROZEN,
65 : SKIP_PAGES_ALL_VISIBLE,
66 : SKIP_PAGES_NONE,
67 : } SkipPages;
68 :
69 : /*
70 : * Struct holding information about a toasted attribute sufficient to both
71 : * check the toasted attribute and, if found to be corrupt, to report where it
72 : * was encountered in the main table.
73 : */
74 : typedef struct ToastedAttribute
75 : {
76 : struct varatt_external toast_pointer;
77 : BlockNumber blkno; /* block in main table */
78 : OffsetNumber offnum; /* offset in main table */
79 : AttrNumber attnum; /* attribute in main table */
80 : } ToastedAttribute;
81 :
82 : /*
83 : * Struct holding the running context information during
84 : * a lifetime of a verify_heapam execution.
85 : */
86 : typedef struct HeapCheckContext
87 : {
88 : /*
89 : * Cached copies of values from TransamVariables and computed values from
90 : * them.
91 : */
92 : FullTransactionId next_fxid; /* TransamVariables->nextXid */
93 : TransactionId next_xid; /* 32-bit version of next_fxid */
94 : TransactionId oldest_xid; /* TransamVariables->oldestXid */
95 : FullTransactionId oldest_fxid; /* 64-bit version of oldest_xid, computed
96 : * relative to next_fxid */
97 : TransactionId safe_xmin; /* this XID and newer ones can't become
98 : * all-visible while we're running */
99 :
100 : /*
101 : * Cached copy of value from MultiXactState
102 : */
103 : MultiXactId next_mxact; /* MultiXactState->nextMXact */
104 : MultiXactId oldest_mxact; /* MultiXactState->oldestMultiXactId */
105 :
106 : /*
107 : * Cached copies of the most recently checked xid and its status.
108 : */
109 : TransactionId cached_xid;
110 : XidCommitStatus cached_status;
111 :
112 : /* Values concerning the heap relation being checked */
113 : Relation rel;
114 : TransactionId relfrozenxid;
115 : FullTransactionId relfrozenfxid;
116 : TransactionId relminmxid;
117 : Relation toast_rel;
118 : Relation *toast_indexes;
119 : Relation valid_toast_index;
120 : int num_toast_indexes;
121 :
122 : /*
123 : * Values for iterating over pages in the relation. `blkno` is the most
124 : * recent block in the buffer yielded by the read stream API.
125 : */
126 : BlockNumber blkno;
127 : BufferAccessStrategy bstrategy;
128 : Buffer buffer;
129 : Page page;
130 :
131 : /* Values for iterating over tuples within a page */
132 : OffsetNumber offnum;
133 : ItemId itemid;
134 : uint16 lp_len;
135 : uint16 lp_off;
136 : HeapTupleHeader tuphdr;
137 : int natts;
138 :
139 : /* Values for iterating over attributes within the tuple */
140 : uint32 offset; /* offset in tuple data */
141 : AttrNumber attnum;
142 :
143 : /* True if tuple's xmax makes it eligible for pruning */
144 : bool tuple_could_be_pruned;
145 :
146 : /*
147 : * List of ToastedAttribute structs for toasted attributes which are not
148 : * eligible for pruning and should be checked
149 : */
150 : List *toasted_attributes;
151 :
152 : /* Whether verify_heapam has yet encountered any corrupt tuples */
153 : bool is_corrupt;
154 :
155 : /* The descriptor and tuplestore for verify_heapam's result tuples */
156 : TupleDesc tupdesc;
157 : Tuplestorestate *tupstore;
158 : } HeapCheckContext;
159 :
160 : /*
161 : * The per-relation data provided to the read stream API for heap amcheck to
162 : * use in its callback for the SKIP_PAGES_ALL_FROZEN and
163 : * SKIP_PAGES_ALL_VISIBLE options.
164 : */
165 : typedef struct HeapCheckReadStreamData
166 : {
167 : /*
168 : * `range` is used by all SkipPages options. SKIP_PAGES_NONE uses the
169 : * default read stream callback, block_range_read_stream_cb(), which takes
170 : * a BlockRangeReadStreamPrivate as its callback_private_data. `range`
171 : * keeps track of the current block number across
172 : * read_stream_next_buffer() invocations.
173 : */
174 : BlockRangeReadStreamPrivate range;
175 : SkipPages skip_option;
176 : Relation rel;
177 : Buffer *vmbuffer;
178 : } HeapCheckReadStreamData;
179 :
180 :
181 : /* Internal implementation */
182 : static BlockNumber heapcheck_read_stream_next_unskippable(ReadStream *stream,
183 : void *callback_private_data,
184 : void *per_buffer_data);
185 :
186 : static void check_tuple(HeapCheckContext *ctx,
187 : bool *xmin_commit_status_ok,
188 : XidCommitStatus *xmin_commit_status);
189 : static void check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
190 : ToastedAttribute *ta, int32 *expected_chunk_seq,
191 : uint32 extsize);
192 :
193 : static bool check_tuple_attribute(HeapCheckContext *ctx);
194 : static void check_toasted_attribute(HeapCheckContext *ctx,
195 : ToastedAttribute *ta);
196 :
197 : static bool check_tuple_header(HeapCheckContext *ctx);
198 : static bool check_tuple_visibility(HeapCheckContext *ctx,
199 : bool *xmin_commit_status_ok,
200 : XidCommitStatus *xmin_commit_status);
201 :
202 : static void report_corruption(HeapCheckContext *ctx, char *msg);
203 : static void report_toast_corruption(HeapCheckContext *ctx,
204 : ToastedAttribute *ta, char *msg);
205 : static FullTransactionId FullTransactionIdFromXidAndCtx(TransactionId xid,
206 : const HeapCheckContext *ctx);
207 : static void update_cached_xid_range(HeapCheckContext *ctx);
208 : static void update_cached_mxid_range(HeapCheckContext *ctx);
209 : static XidBoundsViolation check_mxid_in_range(MultiXactId mxid,
210 : HeapCheckContext *ctx);
211 : static XidBoundsViolation check_mxid_valid_in_rel(MultiXactId mxid,
212 : HeapCheckContext *ctx);
213 : static XidBoundsViolation get_xid_status(TransactionId xid,
214 : HeapCheckContext *ctx,
215 : XidCommitStatus *status);
216 :
217 : /*
218 : * Scan and report corruption in heap pages, optionally reconciling toasted
219 : * attributes with entries in the associated toast table. Intended to be
220 : * called from SQL with the following parameters:
221 : *
222 : * relation:
223 : * The Oid of the heap relation to be checked.
224 : *
225 : * on_error_stop:
226 : * Whether to stop at the end of the first page for which errors are
227 : * detected. Note that multiple rows may be returned.
228 : *
229 : * check_toast:
230 : * Whether to check each toasted attribute against the toast table to
231 : * verify that it can be found there.
232 : *
233 : * skip:
234 : * What kinds of pages in the heap relation should be skipped. Valid
235 : * options are "all-visible", "all-frozen", and "none".
236 : *
237 : * Returns to the SQL caller a set of tuples, each containing the location
238 : * and a description of a corruption found in the heap.
239 : *
240 : * This code goes to some trouble to avoid crashing the server even if the
241 : * table pages are badly corrupted, but it's probably not perfect. If
242 : * check_toast is true, we'll use regular index lookups to try to fetch TOAST
243 : * tuples, which can certainly cause crashes if the right kind of corruption
244 : * exists in the toast table or index. No matter what parameters you pass,
245 : * we can't protect against crashes that might occur trying to look up the
246 : * commit status of transaction IDs (though we avoid trying to do such lookups
247 : * for transaction IDs that can't legally appear in the table).
248 : */
249 : Datum
250 6882 : verify_heapam(PG_FUNCTION_ARGS)
251 : {
252 6882 : ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
253 : HeapCheckContext ctx;
254 6882 : Buffer vmbuffer = InvalidBuffer;
255 : Oid relid;
256 : bool on_error_stop;
257 : bool check_toast;
258 6882 : SkipPages skip_option = SKIP_PAGES_NONE;
259 : BlockNumber first_block;
260 : BlockNumber last_block;
261 : BlockNumber nblocks;
262 : const char *skip;
263 : ReadStream *stream;
264 : int stream_flags;
265 : ReadStreamBlockNumberCB stream_cb;
266 : void *stream_data;
267 : HeapCheckReadStreamData stream_skip_data;
268 :
269 : /* Check supplied arguments */
270 6882 : if (PG_ARGISNULL(0))
271 0 : ereport(ERROR,
272 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
273 : errmsg("relation cannot be null")));
274 6882 : relid = PG_GETARG_OID(0);
275 :
276 6882 : if (PG_ARGISNULL(1))
277 0 : ereport(ERROR,
278 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
279 : errmsg("on_error_stop cannot be null")));
280 6882 : on_error_stop = PG_GETARG_BOOL(1);
281 :
282 6882 : if (PG_ARGISNULL(2))
283 0 : ereport(ERROR,
284 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
285 : errmsg("check_toast cannot be null")));
286 6882 : check_toast = PG_GETARG_BOOL(2);
287 :
288 6882 : if (PG_ARGISNULL(3))
289 0 : ereport(ERROR,
290 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
291 : errmsg("skip cannot be null")));
292 6882 : skip = text_to_cstring(PG_GETARG_TEXT_PP(3));
293 6882 : if (pg_strcasecmp(skip, "all-visible") == 0)
294 170 : skip_option = SKIP_PAGES_ALL_VISIBLE;
295 6712 : else if (pg_strcasecmp(skip, "all-frozen") == 0)
296 174 : skip_option = SKIP_PAGES_ALL_FROZEN;
297 6538 : else if (pg_strcasecmp(skip, "none") == 0)
298 6536 : skip_option = SKIP_PAGES_NONE;
299 : else
300 2 : ereport(ERROR,
301 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
302 : errmsg("invalid skip option"),
303 : errhint("Valid skip options are \"all-visible\", \"all-frozen\", and \"none\".")));
304 :
305 6880 : memset(&ctx, 0, sizeof(HeapCheckContext));
306 6880 : ctx.cached_xid = InvalidTransactionId;
307 6880 : ctx.toasted_attributes = NIL;
308 :
309 : /*
310 : * Any xmin newer than the xmin of our snapshot can't become all-visible
311 : * while we're running.
312 : */
313 6880 : ctx.safe_xmin = GetTransactionSnapshot()->xmin;
314 :
315 : /*
316 : * If we report corruption when not examining some individual attribute,
317 : * we need attnum to be reported as NULL. Set that up before any
318 : * corruption reporting might happen.
319 : */
320 6880 : ctx.attnum = -1;
321 :
322 : /* Construct the tuplestore and tuple descriptor */
323 6880 : InitMaterializedSRF(fcinfo, 0);
324 6880 : ctx.tupdesc = rsinfo->setDesc;
325 6880 : ctx.tupstore = rsinfo->setResult;
326 :
327 : /* Open relation, check relkind and access method */
328 6880 : ctx.rel = relation_open(relid, AccessShareLock);
329 :
330 : /*
331 : * Check that a relation's relkind and access method are both supported.
332 : */
333 6880 : if (!RELKIND_HAS_TABLE_AM(ctx.rel->rd_rel->relkind) &&
334 392 : ctx.rel->rd_rel->relkind != RELKIND_SEQUENCE)
335 8 : ereport(ERROR,
336 : (errcode(ERRCODE_WRONG_OBJECT_TYPE),
337 : errmsg("cannot check relation \"%s\"",
338 : RelationGetRelationName(ctx.rel)),
339 : errdetail_relkind_not_supported(ctx.rel->rd_rel->relkind)));
340 :
341 : /*
342 : * Sequences always use heap AM, but they don't show that in the catalogs.
343 : * Other relkinds might be using a different AM, so check.
344 : */
345 6872 : if (ctx.rel->rd_rel->relkind != RELKIND_SEQUENCE &&
346 6488 : ctx.rel->rd_rel->relam != HEAP_TABLE_AM_OID)
347 0 : ereport(ERROR,
348 : (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
349 : errmsg("only heap AM is supported")));
350 :
351 : /*
352 : * Early exit for unlogged relations during recovery. These will have no
353 : * relation fork, so there won't be anything to check. We behave as if
354 : * the relation is empty.
355 : */
356 6872 : if (ctx.rel->rd_rel->relpersistence == RELPERSISTENCE_UNLOGGED &&
357 0 : RecoveryInProgress())
358 : {
359 0 : ereport(DEBUG1,
360 : (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION),
361 : errmsg("cannot verify unlogged relation \"%s\" during recovery, skipping",
362 : RelationGetRelationName(ctx.rel))));
363 0 : relation_close(ctx.rel, AccessShareLock);
364 0 : PG_RETURN_NULL();
365 : }
366 :
367 : /* Early exit if the relation is empty */
368 6872 : nblocks = RelationGetNumberOfBlocks(ctx.rel);
369 6838 : if (!nblocks)
370 : {
371 3882 : relation_close(ctx.rel, AccessShareLock);
372 3882 : PG_RETURN_NULL();
373 : }
374 :
375 2956 : ctx.bstrategy = GetAccessStrategy(BAS_BULKREAD);
376 2956 : ctx.buffer = InvalidBuffer;
377 2956 : ctx.page = NULL;
378 :
379 : /* Validate block numbers, or handle nulls. */
380 2956 : if (PG_ARGISNULL(4))
381 2708 : first_block = 0;
382 : else
383 : {
384 248 : int64 fb = PG_GETARG_INT64(4);
385 :
386 248 : if (fb < 0 || fb >= nblocks)
387 2 : ereport(ERROR,
388 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
389 : errmsg("starting block number must be between 0 and %u",
390 : nblocks - 1)));
391 246 : first_block = (BlockNumber) fb;
392 : }
393 2954 : if (PG_ARGISNULL(5))
394 2708 : last_block = nblocks - 1;
395 : else
396 : {
397 246 : int64 lb = PG_GETARG_INT64(5);
398 :
399 246 : if (lb < 0 || lb >= nblocks)
400 2 : ereport(ERROR,
401 : (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
402 : errmsg("ending block number must be between 0 and %u",
403 : nblocks - 1)));
404 244 : last_block = (BlockNumber) lb;
405 : }
406 :
407 : /* Optionally open the toast relation, if any. */
408 2952 : if (ctx.rel->rd_rel->reltoastrelid && check_toast)
409 1392 : {
410 : int offset;
411 :
412 : /* Main relation has associated toast relation */
413 1392 : ctx.toast_rel = table_open(ctx.rel->rd_rel->reltoastrelid,
414 : AccessShareLock);
415 1392 : offset = toast_open_indexes(ctx.toast_rel,
416 : AccessShareLock,
417 : &(ctx.toast_indexes),
418 : &(ctx.num_toast_indexes));
419 1392 : ctx.valid_toast_index = ctx.toast_indexes[offset];
420 : }
421 : else
422 : {
423 : /*
424 : * Main relation has no associated toast relation, or we're
425 : * intentionally skipping it.
426 : */
427 1560 : ctx.toast_rel = NULL;
428 1560 : ctx.toast_indexes = NULL;
429 1560 : ctx.num_toast_indexes = 0;
430 : }
431 :
432 2952 : update_cached_xid_range(&ctx);
433 2952 : update_cached_mxid_range(&ctx);
434 2952 : ctx.relfrozenxid = ctx.rel->rd_rel->relfrozenxid;
435 2952 : ctx.relfrozenfxid = FullTransactionIdFromXidAndCtx(ctx.relfrozenxid, &ctx);
436 2952 : ctx.relminmxid = ctx.rel->rd_rel->relminmxid;
437 :
438 2952 : if (TransactionIdIsNormal(ctx.relfrozenxid))
439 2568 : ctx.oldest_xid = ctx.relfrozenxid;
440 :
441 : /* Now that `ctx` is set up, set up the read stream */
442 2952 : stream_skip_data.range.current_blocknum = first_block;
443 2952 : stream_skip_data.range.last_exclusive = last_block + 1;
444 2952 : stream_skip_data.skip_option = skip_option;
445 2952 : stream_skip_data.rel = ctx.rel;
446 2952 : stream_skip_data.vmbuffer = &vmbuffer;
447 :
448 2952 : if (skip_option == SKIP_PAGES_NONE)
449 : {
450 : /*
451 : * It is safe to use batchmode as block_range_read_stream_cb takes no
452 : * locks.
453 : */
454 2620 : stream_cb = block_range_read_stream_cb;
455 2620 : stream_flags = READ_STREAM_SEQUENTIAL |
456 : READ_STREAM_FULL |
457 : READ_STREAM_USE_BATCHING;
458 2620 : stream_data = &stream_skip_data.range;
459 : }
460 : else
461 : {
462 : /*
463 : * It would not be safe to naively use use batchmode, as
464 : * heapcheck_read_stream_next_unskippable takes locks. It shouldn't be
465 : * too hard to convert though.
466 : */
467 332 : stream_cb = heapcheck_read_stream_next_unskippable;
468 332 : stream_flags = READ_STREAM_DEFAULT;
469 332 : stream_data = &stream_skip_data;
470 : }
471 :
472 2952 : stream = read_stream_begin_relation(stream_flags,
473 : ctx.bstrategy,
474 : ctx.rel,
475 : MAIN_FORKNUM,
476 : stream_cb,
477 : stream_data,
478 : 0);
479 :
480 26826 : while ((ctx.buffer = read_stream_next_buffer(stream, NULL)) != InvalidBuffer)
481 : {
482 : OffsetNumber maxoff;
483 : OffsetNumber predecessor[MaxOffsetNumber];
484 : OffsetNumber successor[MaxOffsetNumber];
485 : bool lp_valid[MaxOffsetNumber];
486 : bool xmin_commit_status_ok[MaxOffsetNumber];
487 : XidCommitStatus xmin_commit_status[MaxOffsetNumber];
488 :
489 23880 : CHECK_FOR_INTERRUPTS();
490 :
491 23880 : memset(predecessor, 0, sizeof(OffsetNumber) * MaxOffsetNumber);
492 :
493 : /* Lock the next page. */
494 : Assert(BufferIsValid(ctx.buffer));
495 23880 : LockBuffer(ctx.buffer, BUFFER_LOCK_SHARE);
496 :
497 23880 : ctx.blkno = BufferGetBlockNumber(ctx.buffer);
498 23880 : ctx.page = BufferGetPage(ctx.buffer);
499 :
500 : /* Perform tuple checks */
501 23880 : maxoff = PageGetMaxOffsetNumber(ctx.page);
502 1135126 : for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
503 1111246 : ctx.offnum = OffsetNumberNext(ctx.offnum))
504 : {
505 : BlockNumber nextblkno;
506 : OffsetNumber nextoffnum;
507 :
508 1111246 : successor[ctx.offnum] = InvalidOffsetNumber;
509 1111246 : lp_valid[ctx.offnum] = false;
510 1111246 : xmin_commit_status_ok[ctx.offnum] = false;
511 1111246 : ctx.itemid = PageGetItemId(ctx.page, ctx.offnum);
512 :
513 : /* Skip over unused/dead line pointers */
514 1111246 : if (!ItemIdIsUsed(ctx.itemid) || ItemIdIsDead(ctx.itemid))
515 19028 : continue;
516 :
517 : /*
518 : * If this line pointer has been redirected, check that it
519 : * redirects to a valid offset within the line pointer array
520 : */
521 1092218 : if (ItemIdIsRedirected(ctx.itemid))
522 : {
523 11446 : OffsetNumber rdoffnum = ItemIdGetRedirect(ctx.itemid);
524 : ItemId rditem;
525 :
526 11446 : if (rdoffnum < FirstOffsetNumber)
527 : {
528 12 : report_corruption(&ctx,
529 : psprintf("line pointer redirection to item at offset %u precedes minimum offset %u",
530 : (unsigned) rdoffnum,
531 : (unsigned) FirstOffsetNumber));
532 12 : continue;
533 : }
534 11434 : if (rdoffnum > maxoff)
535 : {
536 28 : report_corruption(&ctx,
537 : psprintf("line pointer redirection to item at offset %u exceeds maximum offset %u",
538 : (unsigned) rdoffnum,
539 : (unsigned) maxoff));
540 28 : continue;
541 : }
542 :
543 : /*
544 : * Since we've checked that this redirect points to a line
545 : * pointer between FirstOffsetNumber and maxoff, it should now
546 : * be safe to fetch the referenced line pointer. We expect it
547 : * to be LP_NORMAL; if not, that's corruption.
548 : */
549 11406 : rditem = PageGetItemId(ctx.page, rdoffnum);
550 11406 : if (!ItemIdIsUsed(rditem))
551 : {
552 0 : report_corruption(&ctx,
553 : psprintf("redirected line pointer points to an unused item at offset %u",
554 : (unsigned) rdoffnum));
555 0 : continue;
556 : }
557 11406 : else if (ItemIdIsDead(rditem))
558 : {
559 0 : report_corruption(&ctx,
560 : psprintf("redirected line pointer points to a dead item at offset %u",
561 : (unsigned) rdoffnum));
562 0 : continue;
563 : }
564 11406 : else if (ItemIdIsRedirected(rditem))
565 : {
566 2 : report_corruption(&ctx,
567 : psprintf("redirected line pointer points to another redirected line pointer at offset %u",
568 : (unsigned) rdoffnum));
569 2 : continue;
570 : }
571 :
572 : /*
573 : * Record the fact that this line pointer has passed basic
574 : * sanity checking, and also the offset number to which it
575 : * points.
576 : */
577 11404 : lp_valid[ctx.offnum] = true;
578 11404 : successor[ctx.offnum] = rdoffnum;
579 11404 : continue;
580 : }
581 :
582 : /* Sanity-check the line pointer's offset and length values */
583 1080772 : ctx.lp_len = ItemIdGetLength(ctx.itemid);
584 1080772 : ctx.lp_off = ItemIdGetOffset(ctx.itemid);
585 :
586 1080772 : if (ctx.lp_off != MAXALIGN(ctx.lp_off))
587 : {
588 12 : report_corruption(&ctx,
589 : psprintf("line pointer to page offset %u is not maximally aligned",
590 12 : ctx.lp_off));
591 12 : continue;
592 : }
593 1080760 : if (ctx.lp_len < MAXALIGN(SizeofHeapTupleHeader))
594 : {
595 24 : report_corruption(&ctx,
596 : psprintf("line pointer length %u is less than the minimum tuple header size %u",
597 24 : ctx.lp_len,
598 : (unsigned) MAXALIGN(SizeofHeapTupleHeader)));
599 24 : continue;
600 : }
601 1080736 : if (ctx.lp_off + ctx.lp_len > BLCKSZ)
602 : {
603 28 : report_corruption(&ctx,
604 : psprintf("line pointer to page offset %u with length %u ends beyond maximum page offset %u",
605 28 : ctx.lp_off,
606 28 : ctx.lp_len,
607 : (unsigned) BLCKSZ));
608 28 : continue;
609 : }
610 :
611 : /* It should be safe to examine the tuple's header, at least */
612 1080708 : lp_valid[ctx.offnum] = true;
613 1080708 : ctx.tuphdr = (HeapTupleHeader) PageGetItem(ctx.page, ctx.itemid);
614 1080708 : ctx.natts = HeapTupleHeaderGetNatts(ctx.tuphdr);
615 :
616 : /* Ok, ready to check this next tuple */
617 1080708 : check_tuple(&ctx,
618 1080708 : &xmin_commit_status_ok[ctx.offnum],
619 1080708 : &xmin_commit_status[ctx.offnum]);
620 :
621 : /*
622 : * If the CTID field of this tuple seems to point to another tuple
623 : * on the same page, record that tuple as the successor of this
624 : * one.
625 : */
626 1080708 : nextblkno = ItemPointerGetBlockNumber(&(ctx.tuphdr)->t_ctid);
627 1080708 : nextoffnum = ItemPointerGetOffsetNumber(&(ctx.tuphdr)->t_ctid);
628 1080708 : if (nextblkno == ctx.blkno && nextoffnum != ctx.offnum &&
629 388 : nextoffnum >= FirstOffsetNumber && nextoffnum <= maxoff)
630 388 : successor[ctx.offnum] = nextoffnum;
631 : }
632 :
633 : /*
634 : * Update chain validation. Check each line pointer that's got a valid
635 : * successor against that successor.
636 : */
637 23880 : ctx.attnum = -1;
638 1135126 : for (ctx.offnum = FirstOffsetNumber; ctx.offnum <= maxoff;
639 1111246 : ctx.offnum = OffsetNumberNext(ctx.offnum))
640 : {
641 : ItemId curr_lp;
642 : ItemId next_lp;
643 : HeapTupleHeader curr_htup;
644 : HeapTupleHeader next_htup;
645 : TransactionId curr_xmin;
646 : TransactionId curr_xmax;
647 : TransactionId next_xmin;
648 1111246 : OffsetNumber nextoffnum = successor[ctx.offnum];
649 :
650 : /*
651 : * The current line pointer may not have a successor, either
652 : * because it's not valid or because it didn't point to anything.
653 : * In either case, we have to give up.
654 : *
655 : * If the current line pointer does point to something, it's
656 : * possible that the target line pointer isn't valid. We have to
657 : * give up in that case, too.
658 : */
659 1111246 : if (nextoffnum == InvalidOffsetNumber || !lp_valid[nextoffnum])
660 1099454 : continue;
661 :
662 : /* We have two valid line pointers that we can examine. */
663 11792 : curr_lp = PageGetItemId(ctx.page, ctx.offnum);
664 11792 : next_lp = PageGetItemId(ctx.page, nextoffnum);
665 :
666 : /* Handle the cases where the current line pointer is a redirect. */
667 11792 : if (ItemIdIsRedirected(curr_lp))
668 : {
669 : /*
670 : * We should not have set successor[ctx.offnum] to a value
671 : * other than InvalidOffsetNumber unless that line pointer is
672 : * LP_NORMAL.
673 : */
674 : Assert(ItemIdIsNormal(next_lp));
675 :
676 : /* Can only redirect to a HOT tuple. */
677 11404 : next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
678 11404 : if (!HeapTupleHeaderIsHeapOnly(next_htup))
679 : {
680 2 : report_corruption(&ctx,
681 : psprintf("redirected line pointer points to a non-heap-only tuple at offset %u",
682 : (unsigned) nextoffnum));
683 : }
684 :
685 : /* HOT chains should not intersect. */
686 11404 : if (predecessor[nextoffnum] != InvalidOffsetNumber)
687 : {
688 2 : report_corruption(&ctx,
689 : psprintf("redirect line pointer points to offset %u, but offset %u also points there",
690 2 : (unsigned) nextoffnum, (unsigned) predecessor[nextoffnum]));
691 2 : continue;
692 : }
693 :
694 : /*
695 : * This redirect and the tuple to which it points seem to be
696 : * part of an update chain.
697 : */
698 11402 : predecessor[nextoffnum] = ctx.offnum;
699 11402 : continue;
700 : }
701 :
702 : /*
703 : * If the next line pointer is a redirect, or if it's a tuple but
704 : * the XMAX of this tuple doesn't match the XMIN of the next
705 : * tuple, then the two aren't part of the same update chain and
706 : * there is nothing more to do.
707 : */
708 388 : if (ItemIdIsRedirected(next_lp))
709 0 : continue;
710 388 : curr_htup = (HeapTupleHeader) PageGetItem(ctx.page, curr_lp);
711 388 : curr_xmax = HeapTupleHeaderGetUpdateXid(curr_htup);
712 388 : next_htup = (HeapTupleHeader) PageGetItem(ctx.page, next_lp);
713 388 : next_xmin = HeapTupleHeaderGetXmin(next_htup);
714 388 : if (!TransactionIdIsValid(curr_xmax) ||
715 : !TransactionIdEquals(curr_xmax, next_xmin))
716 8 : continue;
717 :
718 : /* HOT chains should not intersect. */
719 380 : if (predecessor[nextoffnum] != InvalidOffsetNumber)
720 : {
721 2 : report_corruption(&ctx,
722 : psprintf("tuple points to new version at offset %u, but offset %u also points there",
723 2 : (unsigned) nextoffnum, (unsigned) predecessor[nextoffnum]));
724 2 : continue;
725 : }
726 :
727 : /*
728 : * This tuple and the tuple to which it points seem to be part of
729 : * an update chain.
730 : */
731 378 : predecessor[nextoffnum] = ctx.offnum;
732 :
733 : /*
734 : * If the current tuple is marked as HOT-updated, then the next
735 : * tuple should be marked as a heap-only tuple. Conversely, if the
736 : * current tuple isn't marked as HOT-updated, then the next tuple
737 : * shouldn't be marked as a heap-only tuple.
738 : *
739 : * NB: Can't use HeapTupleHeaderIsHotUpdated() as it checks if
740 : * hint bits indicate xmin/xmax aborted.
741 : */
742 380 : if (!(curr_htup->t_infomask2 & HEAP_HOT_UPDATED) &&
743 2 : HeapTupleHeaderIsHeapOnly(next_htup))
744 : {
745 2 : report_corruption(&ctx,
746 : psprintf("non-heap-only update produced a heap-only tuple at offset %u",
747 : (unsigned) nextoffnum));
748 : }
749 378 : if ((curr_htup->t_infomask2 & HEAP_HOT_UPDATED) &&
750 376 : !HeapTupleHeaderIsHeapOnly(next_htup))
751 : {
752 2 : report_corruption(&ctx,
753 : psprintf("heap-only update produced a non-heap only tuple at offset %u",
754 : (unsigned) nextoffnum));
755 : }
756 :
757 : /*
758 : * If the current tuple's xmin is still in progress but the
759 : * successor tuple's xmin is committed, that's corruption.
760 : *
761 : * NB: We recheck the commit status of the current tuple's xmin
762 : * here, because it might have committed after we checked it and
763 : * before we checked the commit status of the successor tuple's
764 : * xmin. This should be safe because the xmin itself can't have
765 : * changed, only its commit status.
766 : */
767 378 : curr_xmin = HeapTupleHeaderGetXmin(curr_htup);
768 378 : if (xmin_commit_status_ok[ctx.offnum] &&
769 378 : xmin_commit_status[ctx.offnum] == XID_IN_PROGRESS &&
770 2 : xmin_commit_status_ok[nextoffnum] &&
771 4 : xmin_commit_status[nextoffnum] == XID_COMMITTED &&
772 2 : TransactionIdIsInProgress(curr_xmin))
773 : {
774 2 : report_corruption(&ctx,
775 : psprintf("tuple with in-progress xmin %u was updated to produce a tuple at offset %u with committed xmin %u",
776 : (unsigned) curr_xmin,
777 2 : (unsigned) ctx.offnum,
778 : (unsigned) next_xmin));
779 : }
780 :
781 : /*
782 : * If the current tuple's xmin is aborted but the successor
783 : * tuple's xmin is in-progress or committed, that's corruption.
784 : */
785 378 : if (xmin_commit_status_ok[ctx.offnum] &&
786 378 : xmin_commit_status[ctx.offnum] == XID_ABORTED &&
787 4 : xmin_commit_status_ok[nextoffnum])
788 : {
789 4 : if (xmin_commit_status[nextoffnum] == XID_IN_PROGRESS)
790 2 : report_corruption(&ctx,
791 : psprintf("tuple with aborted xmin %u was updated to produce a tuple at offset %u with in-progress xmin %u",
792 : (unsigned) curr_xmin,
793 2 : (unsigned) ctx.offnum,
794 : (unsigned) next_xmin));
795 2 : else if (xmin_commit_status[nextoffnum] == XID_COMMITTED)
796 2 : report_corruption(&ctx,
797 : psprintf("tuple with aborted xmin %u was updated to produce a tuple at offset %u with committed xmin %u",
798 : (unsigned) curr_xmin,
799 2 : (unsigned) ctx.offnum,
800 : (unsigned) next_xmin));
801 : }
802 : }
803 :
804 : /*
805 : * An update chain can start either with a non-heap-only tuple or with
806 : * a redirect line pointer, but not with a heap-only tuple.
807 : *
808 : * (This check is in a separate loop because we need the predecessor
809 : * array to be fully populated before we can perform it.)
810 : */
811 23880 : for (ctx.offnum = FirstOffsetNumber;
812 1135126 : ctx.offnum <= maxoff;
813 1111246 : ctx.offnum = OffsetNumberNext(ctx.offnum))
814 : {
815 1111246 : if (xmin_commit_status_ok[ctx.offnum] &&
816 1080690 : (xmin_commit_status[ctx.offnum] == XID_COMMITTED ||
817 14 : xmin_commit_status[ctx.offnum] == XID_IN_PROGRESS) &&
818 1080680 : predecessor[ctx.offnum] == InvalidOffsetNumber)
819 : {
820 : ItemId curr_lp;
821 :
822 1068906 : curr_lp = PageGetItemId(ctx.page, ctx.offnum);
823 1068906 : if (!ItemIdIsRedirected(curr_lp))
824 : {
825 : HeapTupleHeader curr_htup;
826 :
827 : curr_htup = (HeapTupleHeader)
828 1068906 : PageGetItem(ctx.page, curr_lp);
829 1068906 : if (HeapTupleHeaderIsHeapOnly(curr_htup))
830 8 : report_corruption(&ctx,
831 : psprintf("tuple is root of chain but is marked as heap-only tuple"));
832 : }
833 : }
834 : }
835 :
836 : /* clean up */
837 23880 : UnlockReleaseBuffer(ctx.buffer);
838 :
839 : /*
840 : * Check any toast pointers from the page whose lock we just released
841 : */
842 23880 : if (ctx.toasted_attributes != NIL)
843 : {
844 : ListCell *cell;
845 :
846 25322 : foreach(cell, ctx.toasted_attributes)
847 23636 : check_toasted_attribute(&ctx, lfirst(cell));
848 1686 : list_free_deep(ctx.toasted_attributes);
849 1686 : ctx.toasted_attributes = NIL;
850 : }
851 :
852 23874 : if (on_error_stop && ctx.is_corrupt)
853 0 : break;
854 : }
855 :
856 : /* Ensure that the stream is completely read */
857 : Assert(read_stream_next_buffer(stream, NULL) == InvalidBuffer);
858 2946 : read_stream_end(stream);
859 :
860 2946 : if (vmbuffer != InvalidBuffer)
861 6 : ReleaseBuffer(vmbuffer);
862 :
863 : /* Close the associated toast table and indexes, if any. */
864 2946 : if (ctx.toast_indexes)
865 1386 : toast_close_indexes(ctx.toast_indexes, ctx.num_toast_indexes,
866 : AccessShareLock);
867 2946 : if (ctx.toast_rel)
868 1386 : table_close(ctx.toast_rel, AccessShareLock);
869 :
870 : /* Close the main relation */
871 2946 : relation_close(ctx.rel, AccessShareLock);
872 :
873 2946 : PG_RETURN_NULL();
874 : }
875 :
876 : /*
877 : * Heap amcheck's read stream callback for getting the next unskippable block.
878 : * This callback is only used when 'all-visible' or 'all-frozen' is provided
879 : * as the skip option to verify_heapam(). With the default 'none',
880 : * block_range_read_stream_cb() is used instead.
881 : */
882 : static BlockNumber
883 1738 : heapcheck_read_stream_next_unskippable(ReadStream *stream,
884 : void *callback_private_data,
885 : void *per_buffer_data)
886 : {
887 1738 : HeapCheckReadStreamData *p = callback_private_data;
888 :
889 : /* Loops over [current_blocknum, last_exclusive) blocks */
890 1804 : for (BlockNumber i; (i = p->range.current_blocknum++) < p->range.last_exclusive;)
891 : {
892 1472 : uint8 mapbits = visibilitymap_get_status(p->rel, i, p->vmbuffer);
893 :
894 1472 : if (p->skip_option == SKIP_PAGES_ALL_FROZEN)
895 : {
896 768 : if ((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0)
897 64 : continue;
898 : }
899 :
900 1408 : if (p->skip_option == SKIP_PAGES_ALL_VISIBLE)
901 : {
902 704 : if ((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0)
903 2 : continue;
904 : }
905 :
906 1406 : return i;
907 : }
908 :
909 332 : return InvalidBlockNumber;
910 : }
911 :
912 : /*
913 : * Shared internal implementation for report_corruption and
914 : * report_toast_corruption.
915 : */
916 : static void
917 172 : report_corruption_internal(Tuplestorestate *tupstore, TupleDesc tupdesc,
918 : BlockNumber blkno, OffsetNumber offnum,
919 : AttrNumber attnum, char *msg)
920 : {
921 172 : Datum values[HEAPCHECK_RELATION_COLS] = {0};
922 172 : bool nulls[HEAPCHECK_RELATION_COLS] = {0};
923 : HeapTuple tuple;
924 :
925 172 : values[0] = Int64GetDatum(blkno);
926 172 : values[1] = Int32GetDatum(offnum);
927 172 : values[2] = Int32GetDatum(attnum);
928 172 : nulls[2] = (attnum < 0);
929 172 : values[3] = CStringGetTextDatum(msg);
930 :
931 : /*
932 : * In principle, there is nothing to prevent a scan over a large, highly
933 : * corrupted table from using work_mem worth of memory building up the
934 : * tuplestore. That's ok, but if we also leak the msg argument memory
935 : * until the end of the query, we could exceed work_mem by more than a
936 : * trivial amount. Therefore, free the msg argument each time we are
937 : * called rather than waiting for our current memory context to be freed.
938 : */
939 172 : pfree(msg);
940 :
941 172 : tuple = heap_form_tuple(tupdesc, values, nulls);
942 172 : tuplestore_puttuple(tupstore, tuple);
943 172 : }
944 :
945 : /*
946 : * Record a single corruption found in the main table. The values in ctx should
947 : * indicate the location of the corruption, and the msg argument should contain
948 : * a human-readable description of the corruption.
949 : *
950 : * The msg argument is pfree'd by this function.
951 : */
952 : static void
953 170 : report_corruption(HeapCheckContext *ctx, char *msg)
954 : {
955 170 : report_corruption_internal(ctx->tupstore, ctx->tupdesc, ctx->blkno,
956 170 : ctx->offnum, ctx->attnum, msg);
957 170 : ctx->is_corrupt = true;
958 170 : }
959 :
960 : /*
961 : * Record corruption found in the toast table. The values in ta should
962 : * indicate the location in the main table where the toast pointer was
963 : * encountered, and the msg argument should contain a human-readable
964 : * description of the toast table corruption.
965 : *
966 : * As above, the msg argument is pfree'd by this function.
967 : */
968 : static void
969 2 : report_toast_corruption(HeapCheckContext *ctx, ToastedAttribute *ta,
970 : char *msg)
971 : {
972 2 : report_corruption_internal(ctx->tupstore, ctx->tupdesc, ta->blkno,
973 2 : ta->offnum, ta->attnum, msg);
974 2 : ctx->is_corrupt = true;
975 2 : }
976 :
977 : /*
978 : * Check for tuple header corruption.
979 : *
980 : * Some kinds of corruption make it unsafe to check the tuple attributes, for
981 : * example when the line pointer refers to a range of bytes outside the page.
982 : * In such cases, we return false (not checkable) after recording appropriate
983 : * corruption messages.
984 : *
985 : * Some other kinds of tuple header corruption confuse the question of where
986 : * the tuple attributes begin, or how long the nulls bitmap is, etc., making it
987 : * unreasonable to attempt to check attributes, even if all candidate answers
988 : * to those questions would not result in reading past the end of the line
989 : * pointer or page. In such cases, like above, we record corruption messages
990 : * about the header and then return false.
991 : *
992 : * Other kinds of tuple header corruption do not bear on the question of
993 : * whether the tuple attributes can be checked, so we record corruption
994 : * messages for them but we do not return false merely because we detected
995 : * them.
996 : *
997 : * Returns whether the tuple is sufficiently sensible to undergo visibility and
998 : * attribute checks.
999 : */
1000 : static bool
1001 1080708 : check_tuple_header(HeapCheckContext *ctx)
1002 : {
1003 1080708 : HeapTupleHeader tuphdr = ctx->tuphdr;
1004 1080708 : uint16 infomask = tuphdr->t_infomask;
1005 1080708 : TransactionId curr_xmax = HeapTupleHeaderGetUpdateXid(tuphdr);
1006 1080708 : bool result = true;
1007 : unsigned expected_hoff;
1008 :
1009 1080708 : if (ctx->tuphdr->t_hoff > ctx->lp_len)
1010 : {
1011 2 : report_corruption(ctx,
1012 : psprintf("data begins at offset %u beyond the tuple length %u",
1013 2 : ctx->tuphdr->t_hoff, ctx->lp_len));
1014 2 : result = false;
1015 : }
1016 :
1017 1080708 : if ((ctx->tuphdr->t_infomask & HEAP_XMAX_COMMITTED) &&
1018 386 : (ctx->tuphdr->t_infomask & HEAP_XMAX_IS_MULTI))
1019 : {
1020 4 : report_corruption(ctx,
1021 : pstrdup("multixact should not be marked committed"));
1022 :
1023 : /*
1024 : * This condition is clearly wrong, but it's not enough to justify
1025 : * skipping further checks, because we don't rely on this to determine
1026 : * whether the tuple is visible or to interpret other relevant header
1027 : * fields.
1028 : */
1029 : }
1030 :
1031 2159348 : if (!TransactionIdIsValid(curr_xmax) &&
1032 1078640 : HeapTupleHeaderIsHotUpdated(tuphdr))
1033 : {
1034 2 : report_corruption(ctx,
1035 : psprintf("tuple has been HOT updated, but xmax is 0"));
1036 :
1037 : /*
1038 : * As above, even though this shouldn't happen, it's not sufficient
1039 : * justification for skipping further checks, we should still be able
1040 : * to perform sensibly.
1041 : */
1042 : }
1043 :
1044 1080708 : if (HeapTupleHeaderIsHeapOnly(tuphdr) &&
1045 11784 : ((tuphdr->t_infomask & HEAP_UPDATED) == 0))
1046 : {
1047 2 : report_corruption(ctx,
1048 : psprintf("tuple is heap only, but not the result of an update"));
1049 :
1050 : /* Here again, we can still perform further checks. */
1051 : }
1052 :
1053 1080708 : if (infomask & HEAP_HASNULL)
1054 484228 : expected_hoff = MAXALIGN(SizeofHeapTupleHeader + BITMAPLEN(ctx->natts));
1055 : else
1056 596480 : expected_hoff = MAXALIGN(SizeofHeapTupleHeader);
1057 1080708 : if (ctx->tuphdr->t_hoff != expected_hoff)
1058 : {
1059 10 : if ((infomask & HEAP_HASNULL) && ctx->natts == 1)
1060 0 : report_corruption(ctx,
1061 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (1 attribute, has nulls)",
1062 0 : expected_hoff, ctx->tuphdr->t_hoff));
1063 10 : else if ((infomask & HEAP_HASNULL))
1064 2 : report_corruption(ctx,
1065 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (%u attributes, has nulls)",
1066 2 : expected_hoff, ctx->tuphdr->t_hoff, ctx->natts));
1067 8 : else if (ctx->natts == 1)
1068 0 : report_corruption(ctx,
1069 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (1 attribute, no nulls)",
1070 0 : expected_hoff, ctx->tuphdr->t_hoff));
1071 : else
1072 8 : report_corruption(ctx,
1073 : psprintf("tuple data should begin at byte %u, but actually begins at byte %u (%u attributes, no nulls)",
1074 8 : expected_hoff, ctx->tuphdr->t_hoff, ctx->natts));
1075 10 : result = false;
1076 : }
1077 :
1078 1080708 : return result;
1079 : }
1080 :
1081 : /*
1082 : * Checks tuple visibility so we know which further checks are safe to
1083 : * perform.
1084 : *
1085 : * If a tuple could have been inserted by a transaction that also added a
1086 : * column to the table, but which ultimately did not commit, or which has not
1087 : * yet committed, then the table's current TupleDesc might differ from the one
1088 : * used to construct this tuple, so we must not check it.
1089 : *
1090 : * As a special case, if our own transaction inserted the tuple, even if we
1091 : * added a column to the table, our TupleDesc should match. We could check the
1092 : * tuple, but choose not to do so.
1093 : *
1094 : * If a tuple has been updated or deleted, we can still read the old tuple for
1095 : * corruption checking purposes, as long as we are careful about concurrent
1096 : * vacuums. The main table tuple itself cannot be vacuumed away because we
1097 : * hold a buffer lock on the page, but if the deleting transaction is older
1098 : * than our transaction snapshot's xmin, then vacuum could remove the toast at
1099 : * any time, so we must not try to follow TOAST pointers.
1100 : *
1101 : * If xmin or xmax values are older than can be checked against clog, or appear
1102 : * to be in the future (possibly due to wrap-around), then we cannot make a
1103 : * determination about the visibility of the tuple, so we skip further checks.
1104 : *
1105 : * Returns true if the tuple itself should be checked, false otherwise. Sets
1106 : * ctx->tuple_could_be_pruned if the tuple -- and thus also any associated
1107 : * TOAST tuples -- are eligible for pruning.
1108 : *
1109 : * Sets *xmin_commit_status_ok to true if the commit status of xmin is known
1110 : * and false otherwise. If it's set to true, then also set *xmin_commit_status
1111 : * to the actual commit status.
1112 : */
1113 : static bool
1114 1080698 : check_tuple_visibility(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
1115 : XidCommitStatus *xmin_commit_status)
1116 : {
1117 : TransactionId xmin;
1118 : TransactionId xvac;
1119 : TransactionId xmax;
1120 : XidCommitStatus xmin_status;
1121 : XidCommitStatus xvac_status;
1122 : XidCommitStatus xmax_status;
1123 1080698 : HeapTupleHeader tuphdr = ctx->tuphdr;
1124 :
1125 1080698 : ctx->tuple_could_be_pruned = true; /* have not yet proven otherwise */
1126 1080698 : *xmin_commit_status_ok = false; /* have not yet proven otherwise */
1127 :
1128 : /* If xmin is normal, it should be within valid range */
1129 1080698 : xmin = HeapTupleHeaderGetXmin(tuphdr);
1130 1080698 : switch (get_xid_status(xmin, ctx, &xmin_status))
1131 : {
1132 0 : case XID_INVALID:
1133 : /* Could be the result of a speculative insertion that aborted. */
1134 0 : return false;
1135 1080690 : case XID_BOUNDS_OK:
1136 1080690 : *xmin_commit_status_ok = true;
1137 1080690 : *xmin_commit_status = xmin_status;
1138 1080690 : break;
1139 2 : case XID_IN_FUTURE:
1140 2 : report_corruption(ctx,
1141 : psprintf("xmin %u equals or exceeds next valid transaction ID %u:%u",
1142 : xmin,
1143 2 : EpochFromFullTransactionId(ctx->next_fxid),
1144 2 : XidFromFullTransactionId(ctx->next_fxid)));
1145 2 : return false;
1146 4 : case XID_PRECEDES_CLUSTERMIN:
1147 4 : report_corruption(ctx,
1148 : psprintf("xmin %u precedes oldest valid transaction ID %u:%u",
1149 : xmin,
1150 4 : EpochFromFullTransactionId(ctx->oldest_fxid),
1151 4 : XidFromFullTransactionId(ctx->oldest_fxid)));
1152 4 : return false;
1153 2 : case XID_PRECEDES_RELMIN:
1154 2 : report_corruption(ctx,
1155 : psprintf("xmin %u precedes relation freeze threshold %u:%u",
1156 : xmin,
1157 2 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1158 2 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1159 2 : return false;
1160 : }
1161 :
1162 : /*
1163 : * Has inserting transaction committed?
1164 : */
1165 1080690 : if (!HeapTupleHeaderXminCommitted(tuphdr))
1166 : {
1167 27168 : if (HeapTupleHeaderXminInvalid(tuphdr))
1168 0 : return false; /* inserter aborted, don't check */
1169 : /* Used by pre-9.0 binary upgrades */
1170 27168 : else if (tuphdr->t_infomask & HEAP_MOVED_OFF)
1171 : {
1172 0 : xvac = HeapTupleHeaderGetXvac(tuphdr);
1173 :
1174 0 : switch (get_xid_status(xvac, ctx, &xvac_status))
1175 : {
1176 0 : case XID_INVALID:
1177 0 : report_corruption(ctx,
1178 : pstrdup("old-style VACUUM FULL transaction ID for moved off tuple is invalid"));
1179 0 : return false;
1180 0 : case XID_IN_FUTURE:
1181 0 : report_corruption(ctx,
1182 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple equals or exceeds next valid transaction ID %u:%u",
1183 : xvac,
1184 0 : EpochFromFullTransactionId(ctx->next_fxid),
1185 0 : XidFromFullTransactionId(ctx->next_fxid)));
1186 0 : return false;
1187 0 : case XID_PRECEDES_RELMIN:
1188 0 : report_corruption(ctx,
1189 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple precedes relation freeze threshold %u:%u",
1190 : xvac,
1191 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1192 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1193 0 : return false;
1194 0 : case XID_PRECEDES_CLUSTERMIN:
1195 0 : report_corruption(ctx,
1196 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple precedes oldest valid transaction ID %u:%u",
1197 : xvac,
1198 0 : EpochFromFullTransactionId(ctx->oldest_fxid),
1199 0 : XidFromFullTransactionId(ctx->oldest_fxid)));
1200 0 : return false;
1201 0 : case XID_BOUNDS_OK:
1202 0 : break;
1203 : }
1204 :
1205 0 : switch (xvac_status)
1206 : {
1207 0 : case XID_IS_CURRENT_XID:
1208 0 : report_corruption(ctx,
1209 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple matches our current transaction ID",
1210 : xvac));
1211 0 : return false;
1212 0 : case XID_IN_PROGRESS:
1213 0 : report_corruption(ctx,
1214 : psprintf("old-style VACUUM FULL transaction ID %u for moved off tuple appears to be in progress",
1215 : xvac));
1216 0 : return false;
1217 :
1218 0 : case XID_COMMITTED:
1219 :
1220 : /*
1221 : * The tuple is dead, because the xvac transaction moved
1222 : * it off and committed. It's checkable, but also
1223 : * prunable.
1224 : */
1225 0 : return true;
1226 :
1227 0 : case XID_ABORTED:
1228 :
1229 : /*
1230 : * The original xmin must have committed, because the xvac
1231 : * transaction tried to move it later. Since xvac is
1232 : * aborted, whether it's still alive now depends on the
1233 : * status of xmax.
1234 : */
1235 0 : break;
1236 : }
1237 0 : }
1238 : /* Used by pre-9.0 binary upgrades */
1239 27168 : else if (tuphdr->t_infomask & HEAP_MOVED_IN)
1240 : {
1241 0 : xvac = HeapTupleHeaderGetXvac(tuphdr);
1242 :
1243 0 : switch (get_xid_status(xvac, ctx, &xvac_status))
1244 : {
1245 0 : case XID_INVALID:
1246 0 : report_corruption(ctx,
1247 : pstrdup("old-style VACUUM FULL transaction ID for moved in tuple is invalid"));
1248 0 : return false;
1249 0 : case XID_IN_FUTURE:
1250 0 : report_corruption(ctx,
1251 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple equals or exceeds next valid transaction ID %u:%u",
1252 : xvac,
1253 0 : EpochFromFullTransactionId(ctx->next_fxid),
1254 0 : XidFromFullTransactionId(ctx->next_fxid)));
1255 0 : return false;
1256 0 : case XID_PRECEDES_RELMIN:
1257 0 : report_corruption(ctx,
1258 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple precedes relation freeze threshold %u:%u",
1259 : xvac,
1260 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1261 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1262 0 : return false;
1263 0 : case XID_PRECEDES_CLUSTERMIN:
1264 0 : report_corruption(ctx,
1265 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple precedes oldest valid transaction ID %u:%u",
1266 : xvac,
1267 0 : EpochFromFullTransactionId(ctx->oldest_fxid),
1268 0 : XidFromFullTransactionId(ctx->oldest_fxid)));
1269 0 : return false;
1270 0 : case XID_BOUNDS_OK:
1271 0 : break;
1272 : }
1273 :
1274 0 : switch (xvac_status)
1275 : {
1276 0 : case XID_IS_CURRENT_XID:
1277 0 : report_corruption(ctx,
1278 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple matches our current transaction ID",
1279 : xvac));
1280 0 : return false;
1281 0 : case XID_IN_PROGRESS:
1282 0 : report_corruption(ctx,
1283 : psprintf("old-style VACUUM FULL transaction ID %u for moved in tuple appears to be in progress",
1284 : xvac));
1285 0 : return false;
1286 :
1287 0 : case XID_COMMITTED:
1288 :
1289 : /*
1290 : * The original xmin must have committed, because the xvac
1291 : * transaction moved it later. Whether it's still alive
1292 : * now depends on the status of xmax.
1293 : */
1294 0 : break;
1295 :
1296 0 : case XID_ABORTED:
1297 :
1298 : /*
1299 : * The tuple is dead, because the xvac transaction moved
1300 : * it off and committed. It's checkable, but also
1301 : * prunable.
1302 : */
1303 0 : return true;
1304 : }
1305 0 : }
1306 27168 : else if (xmin_status != XID_COMMITTED)
1307 : {
1308 : /*
1309 : * Inserting transaction is not in progress, and not committed, so
1310 : * it might have changed the TupleDesc in ways we don't know
1311 : * about. Thus, don't try to check the tuple structure.
1312 : *
1313 : * If xmin_status happens to be XID_IS_CURRENT_XID, then in theory
1314 : * any such DDL changes ought to be visible to us, so perhaps we
1315 : * could check anyway in that case. But, for now, let's be
1316 : * conservative and treat this like any other uncommitted insert.
1317 : */
1318 14 : return false;
1319 : }
1320 : }
1321 :
1322 : /*
1323 : * Okay, the inserter committed, so it was good at some point. Now what
1324 : * about the deleting transaction?
1325 : */
1326 :
1327 1080676 : if (tuphdr->t_infomask & HEAP_XMAX_IS_MULTI)
1328 : {
1329 : /*
1330 : * xmax is a multixact, so sanity-check the MXID. Note that we do this
1331 : * prior to checking for HEAP_XMAX_INVALID or
1332 : * HEAP_XMAX_IS_LOCKED_ONLY. This might therefore complain about
1333 : * things that wouldn't actually be a problem during a normal scan,
1334 : * but eventually we're going to have to freeze, and that process will
1335 : * ignore hint bits.
1336 : *
1337 : * Even if the MXID is out of range, we still know that the original
1338 : * insert committed, so we can check the tuple itself. However, we
1339 : * can't rule out the possibility that this tuple is dead, so don't
1340 : * clear ctx->tuple_could_be_pruned. Possibly we should go ahead and
1341 : * clear that flag anyway if HEAP_XMAX_INVALID is set or if
1342 : * HEAP_XMAX_IS_LOCKED_ONLY is true, but for now we err on the side of
1343 : * avoiding possibly-bogus complaints about missing TOAST entries.
1344 : */
1345 116 : xmax = HeapTupleHeaderGetRawXmax(tuphdr);
1346 116 : switch (check_mxid_valid_in_rel(xmax, ctx))
1347 : {
1348 0 : case XID_INVALID:
1349 0 : report_corruption(ctx,
1350 : pstrdup("multitransaction ID is invalid"));
1351 0 : return true;
1352 2 : case XID_PRECEDES_RELMIN:
1353 2 : report_corruption(ctx,
1354 : psprintf("multitransaction ID %u precedes relation minimum multitransaction ID threshold %u",
1355 : xmax, ctx->relminmxid));
1356 2 : return true;
1357 0 : case XID_PRECEDES_CLUSTERMIN:
1358 0 : report_corruption(ctx,
1359 : psprintf("multitransaction ID %u precedes oldest valid multitransaction ID threshold %u",
1360 : xmax, ctx->oldest_mxact));
1361 0 : return true;
1362 2 : case XID_IN_FUTURE:
1363 2 : report_corruption(ctx,
1364 : psprintf("multitransaction ID %u equals or exceeds next valid multitransaction ID %u",
1365 : xmax,
1366 : ctx->next_mxact));
1367 2 : return true;
1368 112 : case XID_BOUNDS_OK:
1369 112 : break;
1370 : }
1371 1080560 : }
1372 :
1373 1080672 : if (tuphdr->t_infomask & HEAP_XMAX_INVALID)
1374 : {
1375 : /*
1376 : * This tuple is live. A concurrently running transaction could
1377 : * delete it before we get around to checking the toast, but any such
1378 : * running transaction is surely not less than our safe_xmin, so the
1379 : * toast cannot be vacuumed out from under us.
1380 : */
1381 1078614 : ctx->tuple_could_be_pruned = false;
1382 1078614 : return true;
1383 : }
1384 :
1385 2058 : if (HEAP_XMAX_IS_LOCKED_ONLY(tuphdr->t_infomask))
1386 : {
1387 : /*
1388 : * "Deleting" xact really only locked it, so the tuple is live in any
1389 : * case. As above, a concurrently running transaction could delete
1390 : * it, but it cannot be vacuumed out from under us.
1391 : */
1392 56 : ctx->tuple_could_be_pruned = false;
1393 56 : return true;
1394 : }
1395 :
1396 2002 : if (tuphdr->t_infomask & HEAP_XMAX_IS_MULTI)
1397 : {
1398 : /*
1399 : * We already checked above that this multixact is within limits for
1400 : * this table. Now check the update xid from this multixact.
1401 : */
1402 56 : xmax = HeapTupleGetUpdateXid(tuphdr);
1403 56 : switch (get_xid_status(xmax, ctx, &xmax_status))
1404 : {
1405 0 : case XID_INVALID:
1406 : /* not LOCKED_ONLY, so it has to have an xmax */
1407 0 : report_corruption(ctx,
1408 : pstrdup("update xid is invalid"));
1409 0 : return true;
1410 0 : case XID_IN_FUTURE:
1411 0 : report_corruption(ctx,
1412 : psprintf("update xid %u equals or exceeds next valid transaction ID %u:%u",
1413 : xmax,
1414 0 : EpochFromFullTransactionId(ctx->next_fxid),
1415 0 : XidFromFullTransactionId(ctx->next_fxid)));
1416 0 : return true;
1417 0 : case XID_PRECEDES_RELMIN:
1418 0 : report_corruption(ctx,
1419 : psprintf("update xid %u precedes relation freeze threshold %u:%u",
1420 : xmax,
1421 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1422 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1423 0 : return true;
1424 0 : case XID_PRECEDES_CLUSTERMIN:
1425 0 : report_corruption(ctx,
1426 : psprintf("update xid %u precedes oldest valid transaction ID %u:%u",
1427 : xmax,
1428 0 : EpochFromFullTransactionId(ctx->oldest_fxid),
1429 0 : XidFromFullTransactionId(ctx->oldest_fxid)));
1430 0 : return true;
1431 56 : case XID_BOUNDS_OK:
1432 56 : break;
1433 : }
1434 :
1435 56 : switch (xmax_status)
1436 : {
1437 0 : case XID_IS_CURRENT_XID:
1438 : case XID_IN_PROGRESS:
1439 :
1440 : /*
1441 : * The delete is in progress, so it cannot be visible to our
1442 : * snapshot.
1443 : */
1444 0 : ctx->tuple_could_be_pruned = false;
1445 0 : break;
1446 56 : case XID_COMMITTED:
1447 :
1448 : /*
1449 : * The delete committed. Whether the toast can be vacuumed
1450 : * away depends on how old the deleting transaction is.
1451 : */
1452 56 : ctx->tuple_could_be_pruned = TransactionIdPrecedes(xmax,
1453 : ctx->safe_xmin);
1454 56 : break;
1455 0 : case XID_ABORTED:
1456 :
1457 : /*
1458 : * The delete aborted or crashed. The tuple is still live.
1459 : */
1460 0 : ctx->tuple_could_be_pruned = false;
1461 0 : break;
1462 : }
1463 :
1464 : /* Tuple itself is checkable even if it's dead. */
1465 56 : return true;
1466 : }
1467 :
1468 : /* xmax is an XID, not a MXID. Sanity check it. */
1469 1946 : xmax = HeapTupleHeaderGetRawXmax(tuphdr);
1470 1946 : switch (get_xid_status(xmax, ctx, &xmax_status))
1471 : {
1472 2 : case XID_INVALID:
1473 2 : ctx->tuple_could_be_pruned = false;
1474 2 : return true;
1475 0 : case XID_IN_FUTURE:
1476 0 : report_corruption(ctx,
1477 : psprintf("xmax %u equals or exceeds next valid transaction ID %u:%u",
1478 : xmax,
1479 0 : EpochFromFullTransactionId(ctx->next_fxid),
1480 0 : XidFromFullTransactionId(ctx->next_fxid)));
1481 0 : return false; /* corrupt */
1482 0 : case XID_PRECEDES_RELMIN:
1483 0 : report_corruption(ctx,
1484 : psprintf("xmax %u precedes relation freeze threshold %u:%u",
1485 : xmax,
1486 0 : EpochFromFullTransactionId(ctx->relfrozenfxid),
1487 0 : XidFromFullTransactionId(ctx->relfrozenfxid)));
1488 0 : return false; /* corrupt */
1489 2 : case XID_PRECEDES_CLUSTERMIN:
1490 2 : report_corruption(ctx,
1491 : psprintf("xmax %u precedes oldest valid transaction ID %u:%u",
1492 : xmax,
1493 2 : EpochFromFullTransactionId(ctx->oldest_fxid),
1494 2 : XidFromFullTransactionId(ctx->oldest_fxid)));
1495 2 : return false; /* corrupt */
1496 1942 : case XID_BOUNDS_OK:
1497 1942 : break;
1498 : }
1499 :
1500 : /*
1501 : * Whether the toast can be vacuumed away depends on how old the deleting
1502 : * transaction is.
1503 : */
1504 1942 : switch (xmax_status)
1505 : {
1506 0 : case XID_IS_CURRENT_XID:
1507 : case XID_IN_PROGRESS:
1508 :
1509 : /*
1510 : * The delete is in progress, so it cannot be visible to our
1511 : * snapshot.
1512 : */
1513 0 : ctx->tuple_could_be_pruned = false;
1514 0 : break;
1515 :
1516 1936 : case XID_COMMITTED:
1517 :
1518 : /*
1519 : * The delete committed. Whether the toast can be vacuumed away
1520 : * depends on how old the deleting transaction is.
1521 : */
1522 1936 : ctx->tuple_could_be_pruned = TransactionIdPrecedes(xmax,
1523 : ctx->safe_xmin);
1524 1936 : break;
1525 :
1526 6 : case XID_ABORTED:
1527 :
1528 : /*
1529 : * The delete aborted or crashed. The tuple is still live.
1530 : */
1531 6 : ctx->tuple_could_be_pruned = false;
1532 6 : break;
1533 : }
1534 :
1535 : /* Tuple itself is checkable even if it's dead. */
1536 1942 : return true;
1537 : }
1538 :
1539 :
1540 : /*
1541 : * Check the current toast tuple against the state tracked in ctx, recording
1542 : * any corruption found in ctx->tupstore.
1543 : *
1544 : * This is not equivalent to running verify_heapam on the toast table itself,
1545 : * and is not hardened against corruption of the toast table. Rather, when
1546 : * validating a toasted attribute in the main table, the sequence of toast
1547 : * tuples that store the toasted value are retrieved and checked in order, with
1548 : * each toast tuple being checked against where we are in the sequence, as well
1549 : * as each toast tuple having its varlena structure sanity checked.
1550 : *
1551 : * On entry, *expected_chunk_seq should be the chunk_seq value that we expect
1552 : * to find in toasttup. On exit, it will be updated to the value the next call
1553 : * to this function should expect to see.
1554 : */
1555 : static void
1556 83364 : check_toast_tuple(HeapTuple toasttup, HeapCheckContext *ctx,
1557 : ToastedAttribute *ta, int32 *expected_chunk_seq,
1558 : uint32 extsize)
1559 : {
1560 : int32 chunk_seq;
1561 83364 : int32 last_chunk_seq = (extsize - 1) / TOAST_MAX_CHUNK_SIZE;
1562 : Pointer chunk;
1563 : bool isnull;
1564 : int32 chunksize;
1565 : int32 expected_size;
1566 :
1567 : /* Sanity-check the sequence number. */
1568 83364 : chunk_seq = DatumGetInt32(fastgetattr(toasttup, 2,
1569 83364 : ctx->toast_rel->rd_att, &isnull));
1570 83364 : if (isnull)
1571 : {
1572 0 : report_toast_corruption(ctx, ta,
1573 : psprintf("toast value %u has toast chunk with null sequence number",
1574 : ta->toast_pointer.va_valueid));
1575 0 : return;
1576 : }
1577 83364 : if (chunk_seq != *expected_chunk_seq)
1578 : {
1579 : /* Either the TOAST index is corrupt, or we don't have all chunks. */
1580 0 : report_toast_corruption(ctx, ta,
1581 : psprintf("toast value %u index scan returned chunk %d when expecting chunk %d",
1582 : ta->toast_pointer.va_valueid,
1583 : chunk_seq, *expected_chunk_seq));
1584 : }
1585 83364 : *expected_chunk_seq = chunk_seq + 1;
1586 :
1587 : /* Sanity-check the chunk data. */
1588 83364 : chunk = DatumGetPointer(fastgetattr(toasttup, 3,
1589 83364 : ctx->toast_rel->rd_att, &isnull));
1590 83364 : if (isnull)
1591 : {
1592 0 : report_toast_corruption(ctx, ta,
1593 : psprintf("toast value %u chunk %d has null data",
1594 : ta->toast_pointer.va_valueid,
1595 : chunk_seq));
1596 0 : return;
1597 : }
1598 83364 : if (!VARATT_IS_EXTENDED(chunk))
1599 83364 : chunksize = VARSIZE(chunk) - VARHDRSZ;
1600 0 : else if (VARATT_IS_SHORT(chunk))
1601 : {
1602 : /*
1603 : * could happen due to heap_form_tuple doing its thing
1604 : */
1605 0 : chunksize = VARSIZE_SHORT(chunk) - VARHDRSZ_SHORT;
1606 : }
1607 : else
1608 : {
1609 : /* should never happen */
1610 0 : uint32 header = ((varattrib_4b *) chunk)->va_4byte.va_header;
1611 :
1612 0 : report_toast_corruption(ctx, ta,
1613 : psprintf("toast value %u chunk %d has invalid varlena header %0x",
1614 : ta->toast_pointer.va_valueid,
1615 : chunk_seq, header));
1616 0 : return;
1617 : }
1618 :
1619 : /*
1620 : * Some checks on the data we've found
1621 : */
1622 83364 : if (chunk_seq > last_chunk_seq)
1623 : {
1624 0 : report_toast_corruption(ctx, ta,
1625 : psprintf("toast value %u chunk %d follows last expected chunk %d",
1626 : ta->toast_pointer.va_valueid,
1627 : chunk_seq, last_chunk_seq));
1628 0 : return;
1629 : }
1630 :
1631 83364 : expected_size = chunk_seq < last_chunk_seq ? TOAST_MAX_CHUNK_SIZE
1632 23628 : : extsize - (last_chunk_seq * TOAST_MAX_CHUNK_SIZE);
1633 :
1634 83364 : if (chunksize != expected_size)
1635 0 : report_toast_corruption(ctx, ta,
1636 : psprintf("toast value %u chunk %d has size %u, but expected size %u",
1637 : ta->toast_pointer.va_valueid,
1638 : chunk_seq, chunksize, expected_size));
1639 : }
1640 :
1641 : /*
1642 : * Check the current attribute as tracked in ctx, recording any corruption
1643 : * found in ctx->tupstore.
1644 : *
1645 : * This function follows the logic performed by heap_deform_tuple(), and in the
1646 : * case of a toasted value, optionally stores the toast pointer so later it can
1647 : * be checked following the logic of detoast_external_attr(), checking for any
1648 : * conditions that would result in either of those functions Asserting or
1649 : * crashing the backend. The checks performed by Asserts present in those two
1650 : * functions are also performed here and in check_toasted_attribute. In cases
1651 : * where those two functions are a bit cavalier in their assumptions about data
1652 : * being correct, we perform additional checks not present in either of those
1653 : * two functions. Where some condition is checked in both of those functions,
1654 : * we perform it here twice, as we parallel the logical flow of those two
1655 : * functions. The presence of duplicate checks seems a reasonable price to pay
1656 : * for keeping this code tightly coupled with the code it protects.
1657 : *
1658 : * Returns true if the tuple attribute is sane enough for processing to
1659 : * continue on to the next attribute, false otherwise.
1660 : */
1661 : static bool
1662 15534250 : check_tuple_attribute(HeapCheckContext *ctx)
1663 : {
1664 : Datum attdatum;
1665 : struct varlena *attr;
1666 : char *tp; /* pointer to the tuple data */
1667 : uint16 infomask;
1668 : CompactAttribute *thisatt;
1669 : struct varatt_external toast_pointer;
1670 :
1671 15534250 : infomask = ctx->tuphdr->t_infomask;
1672 15534250 : thisatt = TupleDescCompactAttr(RelationGetDescr(ctx->rel), ctx->attnum);
1673 :
1674 15534250 : tp = (char *) ctx->tuphdr + ctx->tuphdr->t_hoff;
1675 :
1676 15534250 : if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len)
1677 : {
1678 0 : report_corruption(ctx,
1679 : psprintf("attribute with length %u starts at offset %u beyond total tuple length %u",
1680 0 : thisatt->attlen,
1681 0 : ctx->tuphdr->t_hoff + ctx->offset,
1682 0 : ctx->lp_len));
1683 0 : return false;
1684 : }
1685 :
1686 : /* Skip null values */
1687 15534250 : if (infomask & HEAP_HASNULL && att_isnull(ctx->attnum, ctx->tuphdr->t_bits))
1688 2658956 : return true;
1689 :
1690 : /* Skip non-varlena values, but update offset first */
1691 12875294 : if (thisatt->attlen != -1)
1692 : {
1693 11812056 : ctx->offset = att_nominal_alignby(ctx->offset, thisatt->attalignby);
1694 11812056 : ctx->offset = att_addlength_pointer(ctx->offset, thisatt->attlen,
1695 : tp + ctx->offset);
1696 11812056 : if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len)
1697 : {
1698 0 : report_corruption(ctx,
1699 : psprintf("attribute with length %u ends at offset %u beyond total tuple length %u",
1700 0 : thisatt->attlen,
1701 0 : ctx->tuphdr->t_hoff + ctx->offset,
1702 0 : ctx->lp_len));
1703 0 : return false;
1704 : }
1705 11812056 : return true;
1706 : }
1707 :
1708 : /* Ok, we're looking at a varlena attribute. */
1709 1063238 : ctx->offset = att_pointer_alignby(ctx->offset, thisatt->attalignby, -1,
1710 : tp + ctx->offset);
1711 :
1712 : /* Get the (possibly corrupt) varlena datum */
1713 1063238 : attdatum = fetchatt(thisatt, tp + ctx->offset);
1714 :
1715 : /*
1716 : * We have the datum, but we cannot decode it carelessly, as it may still
1717 : * be corrupt.
1718 : */
1719 :
1720 : /*
1721 : * Check that VARTAG_SIZE won't hit an Assert on a corrupt va_tag before
1722 : * risking a call into att_addlength_pointer
1723 : */
1724 1063238 : if (VARATT_IS_EXTERNAL(tp + ctx->offset))
1725 : {
1726 52510 : uint8 va_tag = VARTAG_EXTERNAL(tp + ctx->offset);
1727 :
1728 52510 : if (va_tag != VARTAG_ONDISK)
1729 : {
1730 0 : report_corruption(ctx,
1731 : psprintf("toasted attribute has unexpected TOAST tag %u",
1732 : va_tag));
1733 : /* We can't know where the next attribute begins */
1734 0 : return false;
1735 : }
1736 : }
1737 :
1738 : /* Ok, should be safe now */
1739 1063238 : ctx->offset = att_addlength_pointer(ctx->offset, thisatt->attlen,
1740 : tp + ctx->offset);
1741 :
1742 1063238 : if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len)
1743 : {
1744 2 : report_corruption(ctx,
1745 : psprintf("attribute with length %u ends at offset %u beyond total tuple length %u",
1746 2 : thisatt->attlen,
1747 2 : ctx->tuphdr->t_hoff + ctx->offset,
1748 2 : ctx->lp_len));
1749 :
1750 2 : return false;
1751 : }
1752 :
1753 : /*
1754 : * heap_deform_tuple would be done with this attribute at this point,
1755 : * having stored it in values[], and would continue to the next attribute.
1756 : * We go further, because we need to check if the toast datum is corrupt.
1757 : */
1758 :
1759 1063236 : attr = (struct varlena *) DatumGetPointer(attdatum);
1760 :
1761 : /*
1762 : * Now we follow the logic of detoast_external_attr(), with the same
1763 : * caveats about being paranoid about corruption.
1764 : */
1765 :
1766 : /* Skip values that are not external */
1767 1063236 : if (!VARATT_IS_EXTERNAL(attr))
1768 1010726 : return true;
1769 :
1770 : /* It is external, and we're looking at a page on disk */
1771 :
1772 : /*
1773 : * Must copy attr into toast_pointer for alignment considerations
1774 : */
1775 52510 : VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
1776 :
1777 : /* Toasted attributes too large to be untoasted should never be stored */
1778 52510 : if (toast_pointer.va_rawsize > VARLENA_SIZE_LIMIT)
1779 0 : report_corruption(ctx,
1780 : psprintf("toast value %u rawsize %d exceeds limit %d",
1781 : toast_pointer.va_valueid,
1782 : toast_pointer.va_rawsize,
1783 : VARLENA_SIZE_LIMIT));
1784 :
1785 52510 : if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
1786 : {
1787 : ToastCompressionId cmid;
1788 4054 : bool valid = false;
1789 :
1790 : /* Compressed attributes should have a valid compression method */
1791 4054 : cmid = TOAST_COMPRESS_METHOD(&toast_pointer);
1792 4054 : switch (cmid)
1793 : {
1794 : /* List of all valid compression method IDs */
1795 4054 : case TOAST_PGLZ_COMPRESSION_ID:
1796 : case TOAST_LZ4_COMPRESSION_ID:
1797 4054 : valid = true;
1798 4054 : break;
1799 :
1800 : /* Recognized but invalid compression method ID */
1801 0 : case TOAST_INVALID_COMPRESSION_ID:
1802 0 : break;
1803 :
1804 : /* Intentionally no default here */
1805 : }
1806 4054 : if (!valid)
1807 0 : report_corruption(ctx,
1808 : psprintf("toast value %u has invalid compression method id %d",
1809 : toast_pointer.va_valueid, cmid));
1810 : }
1811 :
1812 : /* The tuple header better claim to contain toasted values */
1813 52510 : if (!(infomask & HEAP_HASEXTERNAL))
1814 : {
1815 0 : report_corruption(ctx,
1816 : psprintf("toast value %u is external but tuple header flag HEAP_HASEXTERNAL not set",
1817 : toast_pointer.va_valueid));
1818 0 : return true;
1819 : }
1820 :
1821 : /* The relation better have a toast table */
1822 52510 : if (!ctx->rel->rd_rel->reltoastrelid)
1823 : {
1824 0 : report_corruption(ctx,
1825 : psprintf("toast value %u is external but relation has no toast relation",
1826 : toast_pointer.va_valueid));
1827 0 : return true;
1828 : }
1829 :
1830 : /* If we were told to skip toast checking, then we're done. */
1831 52510 : if (ctx->toast_rel == NULL)
1832 28858 : return true;
1833 :
1834 : /*
1835 : * If this tuple is eligible to be pruned, we cannot check the toast.
1836 : * Otherwise, we push a copy of the toast tuple so we can check it after
1837 : * releasing the main table buffer lock.
1838 : */
1839 23652 : if (!ctx->tuple_could_be_pruned)
1840 : {
1841 : ToastedAttribute *ta;
1842 :
1843 23648 : ta = (ToastedAttribute *) palloc0(sizeof(ToastedAttribute));
1844 :
1845 23648 : VARATT_EXTERNAL_GET_POINTER(ta->toast_pointer, attr);
1846 23648 : ta->blkno = ctx->blkno;
1847 23648 : ta->offnum = ctx->offnum;
1848 23648 : ta->attnum = ctx->attnum;
1849 23648 : ctx->toasted_attributes = lappend(ctx->toasted_attributes, ta);
1850 : }
1851 :
1852 23652 : return true;
1853 : }
1854 :
1855 : /*
1856 : * For each attribute collected in ctx->toasted_attributes, look up the value
1857 : * in the toast table and perform checks on it. This function should only be
1858 : * called on toast pointers which cannot be vacuumed away during our
1859 : * processing.
1860 : */
1861 : static void
1862 23636 : check_toasted_attribute(HeapCheckContext *ctx, ToastedAttribute *ta)
1863 : {
1864 : ScanKeyData toastkey;
1865 : SysScanDesc toastscan;
1866 : bool found_toasttup;
1867 : HeapTuple toasttup;
1868 : uint32 extsize;
1869 23636 : int32 expected_chunk_seq = 0;
1870 : int32 last_chunk_seq;
1871 :
1872 23636 : extsize = VARATT_EXTERNAL_GET_EXTSIZE(ta->toast_pointer);
1873 23636 : last_chunk_seq = (extsize - 1) / TOAST_MAX_CHUNK_SIZE;
1874 :
1875 : /*
1876 : * Setup a scan key to find chunks in toast table with matching va_valueid
1877 : */
1878 23636 : ScanKeyInit(&toastkey,
1879 : (AttrNumber) 1,
1880 : BTEqualStrategyNumber, F_OIDEQ,
1881 : ObjectIdGetDatum(ta->toast_pointer.va_valueid));
1882 :
1883 : /*
1884 : * Check if any chunks for this toasted object exist in the toast table,
1885 : * accessible via the index.
1886 : */
1887 23636 : toastscan = systable_beginscan_ordered(ctx->toast_rel,
1888 : ctx->valid_toast_index,
1889 : get_toast_snapshot(), 1,
1890 : &toastkey);
1891 23636 : found_toasttup = false;
1892 130630 : while ((toasttup =
1893 107000 : systable_getnext_ordered(toastscan,
1894 : ForwardScanDirection)) != NULL)
1895 : {
1896 83364 : found_toasttup = true;
1897 83364 : check_toast_tuple(toasttup, ctx, ta, &expected_chunk_seq, extsize);
1898 : }
1899 23630 : systable_endscan_ordered(toastscan);
1900 :
1901 23630 : if (!found_toasttup)
1902 2 : report_toast_corruption(ctx, ta,
1903 : psprintf("toast value %u not found in toast table",
1904 : ta->toast_pointer.va_valueid));
1905 23628 : else if (expected_chunk_seq <= last_chunk_seq)
1906 0 : report_toast_corruption(ctx, ta,
1907 : psprintf("toast value %u was expected to end at chunk %d, but ended while expecting chunk %d",
1908 : ta->toast_pointer.va_valueid,
1909 : last_chunk_seq, expected_chunk_seq));
1910 23630 : }
1911 :
1912 : /*
1913 : * Check the current tuple as tracked in ctx, recording any corruption found in
1914 : * ctx->tupstore.
1915 : *
1916 : * We return some information about the status of xmin to aid in validating
1917 : * update chains.
1918 : */
1919 : static void
1920 1080708 : check_tuple(HeapCheckContext *ctx, bool *xmin_commit_status_ok,
1921 : XidCommitStatus *xmin_commit_status)
1922 : {
1923 : /*
1924 : * Check various forms of tuple header corruption, and if the header is
1925 : * too corrupt, do not continue with other checks.
1926 : */
1927 1080708 : if (!check_tuple_header(ctx))
1928 10 : return;
1929 :
1930 : /*
1931 : * Check tuple visibility. If the inserting transaction aborted, we
1932 : * cannot assume our relation description matches the tuple structure, and
1933 : * therefore cannot check it.
1934 : */
1935 1080698 : if (!check_tuple_visibility(ctx, xmin_commit_status_ok,
1936 : xmin_commit_status))
1937 24 : return;
1938 :
1939 : /*
1940 : * The tuple is visible, so it must be compatible with the current version
1941 : * of the relation descriptor. It might have fewer columns than are
1942 : * present in the relation descriptor, but it cannot have more.
1943 : */
1944 1080674 : if (RelationGetDescr(ctx->rel)->natts < ctx->natts)
1945 : {
1946 4 : report_corruption(ctx,
1947 : psprintf("number of attributes %u exceeds maximum expected for table %u",
1948 : ctx->natts,
1949 4 : RelationGetDescr(ctx->rel)->natts));
1950 4 : return;
1951 : }
1952 :
1953 : /*
1954 : * Check each attribute unless we hit corruption that confuses what to do
1955 : * next, at which point we abort further attribute checks for this tuple.
1956 : * Note that we don't abort for all types of corruption, only for those
1957 : * types where we don't know how to continue. We also don't abort the
1958 : * checking of toasted attributes collected from the tuple prior to
1959 : * aborting. Those will still be checked later along with other toasted
1960 : * attributes collected from the page.
1961 : */
1962 1080670 : ctx->offset = 0;
1963 16614918 : for (ctx->attnum = 0; ctx->attnum < ctx->natts; ctx->attnum++)
1964 15534250 : if (!check_tuple_attribute(ctx))
1965 2 : break; /* cannot continue */
1966 :
1967 : /* revert attnum to -1 until we again examine individual attributes */
1968 1080670 : ctx->attnum = -1;
1969 : }
1970 :
1971 : /*
1972 : * Convert a TransactionId into a FullTransactionId using our cached values of
1973 : * the valid transaction ID range. It is the caller's responsibility to have
1974 : * already updated the cached values, if necessary. This is akin to
1975 : * FullTransactionIdFromAllowableAt(), but it tolerates corruption in the form
1976 : * of an xid before epoch 0.
1977 : */
1978 : static FullTransactionId
1979 149610 : FullTransactionIdFromXidAndCtx(TransactionId xid, const HeapCheckContext *ctx)
1980 : {
1981 : uint64 nextfxid_i;
1982 : int32 diff;
1983 : FullTransactionId fxid;
1984 :
1985 : Assert(TransactionIdIsNormal(ctx->next_xid));
1986 : Assert(FullTransactionIdIsNormal(ctx->next_fxid));
1987 : Assert(XidFromFullTransactionId(ctx->next_fxid) == ctx->next_xid);
1988 :
1989 149610 : if (!TransactionIdIsNormal(xid))
1990 384 : return FullTransactionIdFromEpochAndXid(0, xid);
1991 :
1992 149226 : nextfxid_i = U64FromFullTransactionId(ctx->next_fxid);
1993 :
1994 : /* compute the 32bit modulo difference */
1995 149226 : diff = (int32) (ctx->next_xid - xid);
1996 :
1997 : /*
1998 : * In cases of corruption we might see a 32bit xid that is before epoch 0.
1999 : * We can't represent that as a 64bit xid, due to 64bit xids being
2000 : * unsigned integers, without the modulo arithmetic of 32bit xid. There's
2001 : * no really nice way to deal with that, but it works ok enough to use
2002 : * FirstNormalFullTransactionId in that case, as a freshly initdb'd
2003 : * cluster already has a newer horizon.
2004 : */
2005 149226 : if (diff > 0 && (nextfxid_i - FirstNormalTransactionId) < (int64) diff)
2006 : {
2007 : Assert(EpochFromFullTransactionId(ctx->next_fxid) == 0);
2008 8 : fxid = FirstNormalFullTransactionId;
2009 : }
2010 : else
2011 149218 : fxid = FullTransactionIdFromU64(nextfxid_i - diff);
2012 :
2013 : Assert(FullTransactionIdIsNormal(fxid));
2014 149226 : return fxid;
2015 : }
2016 :
2017 : /*
2018 : * Update our cached range of valid transaction IDs.
2019 : */
2020 : static void
2021 2960 : update_cached_xid_range(HeapCheckContext *ctx)
2022 : {
2023 : /* Make cached copies */
2024 2960 : LWLockAcquire(XidGenLock, LW_SHARED);
2025 2960 : ctx->next_fxid = TransamVariables->nextXid;
2026 2960 : ctx->oldest_xid = TransamVariables->oldestXid;
2027 2960 : LWLockRelease(XidGenLock);
2028 :
2029 : /* And compute alternate versions of the same */
2030 2960 : ctx->next_xid = XidFromFullTransactionId(ctx->next_fxid);
2031 2960 : ctx->oldest_fxid = FullTransactionIdFromXidAndCtx(ctx->oldest_xid, ctx);
2032 2960 : }
2033 :
2034 : /*
2035 : * Update our cached range of valid multitransaction IDs.
2036 : */
2037 : static void
2038 2956 : update_cached_mxid_range(HeapCheckContext *ctx)
2039 : {
2040 2956 : ReadMultiXactIdRange(&ctx->oldest_mxact, &ctx->next_mxact);
2041 2956 : }
2042 :
2043 : /*
2044 : * Return whether the given FullTransactionId is within our cached valid
2045 : * transaction ID range.
2046 : */
2047 : static inline bool
2048 125420 : fxid_in_cached_range(FullTransactionId fxid, const HeapCheckContext *ctx)
2049 : {
2050 250834 : return (FullTransactionIdPrecedesOrEquals(ctx->oldest_fxid, fxid) &&
2051 125414 : FullTransactionIdPrecedes(fxid, ctx->next_fxid));
2052 : }
2053 :
2054 : /*
2055 : * Checks whether a multitransaction ID is in the cached valid range, returning
2056 : * the nature of the range violation, if any.
2057 : */
2058 : static XidBoundsViolation
2059 120 : check_mxid_in_range(MultiXactId mxid, HeapCheckContext *ctx)
2060 : {
2061 120 : if (!TransactionIdIsValid(mxid))
2062 0 : return XID_INVALID;
2063 120 : if (MultiXactIdPrecedes(mxid, ctx->relminmxid))
2064 4 : return XID_PRECEDES_RELMIN;
2065 116 : if (MultiXactIdPrecedes(mxid, ctx->oldest_mxact))
2066 0 : return XID_PRECEDES_CLUSTERMIN;
2067 116 : if (MultiXactIdPrecedesOrEquals(ctx->next_mxact, mxid))
2068 4 : return XID_IN_FUTURE;
2069 112 : return XID_BOUNDS_OK;
2070 : }
2071 :
2072 : /*
2073 : * Checks whether the given mxid is valid to appear in the heap being checked,
2074 : * returning the nature of the range violation, if any.
2075 : *
2076 : * This function attempts to return quickly by caching the known valid mxid
2077 : * range in ctx. Callers should already have performed the initial setup of
2078 : * the cache prior to the first call to this function.
2079 : */
2080 : static XidBoundsViolation
2081 116 : check_mxid_valid_in_rel(MultiXactId mxid, HeapCheckContext *ctx)
2082 : {
2083 : XidBoundsViolation result;
2084 :
2085 116 : result = check_mxid_in_range(mxid, ctx);
2086 116 : if (result == XID_BOUNDS_OK)
2087 112 : return XID_BOUNDS_OK;
2088 :
2089 : /* The range may have advanced. Recheck. */
2090 4 : update_cached_mxid_range(ctx);
2091 4 : return check_mxid_in_range(mxid, ctx);
2092 : }
2093 :
2094 : /*
2095 : * Checks whether the given transaction ID is (or was recently) valid to appear
2096 : * in the heap being checked, or whether it is too old or too new to appear in
2097 : * the relation, returning information about the nature of the bounds violation.
2098 : *
2099 : * We cache the range of valid transaction IDs. If xid is in that range, we
2100 : * conclude that it is valid, even though concurrent changes to the table might
2101 : * invalidate it under certain corrupt conditions. (For example, if the table
2102 : * contains corrupt all-frozen bits, a concurrent vacuum might skip the page(s)
2103 : * containing the xid and then truncate clog and advance the relfrozenxid
2104 : * beyond xid.) Reporting the xid as valid under such conditions seems
2105 : * acceptable, since if we had checked it earlier in our scan it would have
2106 : * truly been valid at that time.
2107 : *
2108 : * If the status argument is not NULL, and if and only if the transaction ID
2109 : * appears to be valid in this relation, the status argument will be set with
2110 : * the commit status of the transaction ID.
2111 : */
2112 : static XidBoundsViolation
2113 1082700 : get_xid_status(TransactionId xid, HeapCheckContext *ctx,
2114 : XidCommitStatus *status)
2115 : {
2116 : FullTransactionId fxid;
2117 : FullTransactionId clog_horizon;
2118 :
2119 : /* Quick check for special xids */
2120 1082700 : if (!TransactionIdIsValid(xid))
2121 2 : return XID_INVALID;
2122 1082698 : else if (xid == BootstrapTransactionId || xid == FrozenTransactionId)
2123 : {
2124 957278 : if (status != NULL)
2125 957278 : *status = XID_COMMITTED;
2126 957278 : return XID_BOUNDS_OK;
2127 : }
2128 :
2129 : /* Check if the xid is within bounds */
2130 125420 : fxid = FullTransactionIdFromXidAndCtx(xid, ctx);
2131 125420 : if (!fxid_in_cached_range(fxid, ctx))
2132 : {
2133 : /*
2134 : * We may have been checking against stale values. Update the cached
2135 : * range to be sure, and since we relied on the cached range when we
2136 : * performed the full xid conversion, reconvert.
2137 : */
2138 8 : update_cached_xid_range(ctx);
2139 8 : fxid = FullTransactionIdFromXidAndCtx(xid, ctx);
2140 : }
2141 :
2142 125420 : if (FullTransactionIdPrecedesOrEquals(ctx->next_fxid, fxid))
2143 2 : return XID_IN_FUTURE;
2144 125418 : if (FullTransactionIdPrecedes(fxid, ctx->oldest_fxid))
2145 6 : return XID_PRECEDES_CLUSTERMIN;
2146 125412 : if (FullTransactionIdPrecedes(fxid, ctx->relfrozenfxid))
2147 2 : return XID_PRECEDES_RELMIN;
2148 :
2149 : /* Early return if the caller does not request clog checking */
2150 125410 : if (status == NULL)
2151 0 : return XID_BOUNDS_OK;
2152 :
2153 : /* Early return if we just checked this xid in a prior call */
2154 125410 : if (xid == ctx->cached_xid)
2155 : {
2156 107140 : *status = ctx->cached_status;
2157 107140 : return XID_BOUNDS_OK;
2158 : }
2159 :
2160 18270 : *status = XID_COMMITTED;
2161 18270 : LWLockAcquire(XactTruncationLock, LW_SHARED);
2162 : clog_horizon =
2163 18270 : FullTransactionIdFromXidAndCtx(TransamVariables->oldestClogXid,
2164 : ctx);
2165 18270 : if (FullTransactionIdPrecedesOrEquals(clog_horizon, fxid))
2166 : {
2167 18270 : if (TransactionIdIsCurrentTransactionId(xid))
2168 0 : *status = XID_IS_CURRENT_XID;
2169 18270 : else if (TransactionIdIsInProgress(xid))
2170 4 : *status = XID_IN_PROGRESS;
2171 18266 : else if (TransactionIdDidCommit(xid))
2172 18252 : *status = XID_COMMITTED;
2173 : else
2174 14 : *status = XID_ABORTED;
2175 : }
2176 18270 : LWLockRelease(XactTruncationLock);
2177 18270 : ctx->cached_xid = xid;
2178 18270 : ctx->cached_status = *status;
2179 18270 : return XID_BOUNDS_OK;
2180 : }
|
