Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * basebackup_incremental.c
4 : : * code for incremental backup support
5 : : *
6 : : * This code isn't actually in charge of taking an incremental backup;
7 : : * the actual construction of the incremental backup happens in
8 : : * basebackup.c. Here, we're concerned with providing the necessary
9 : : * supports for that operation. In particular, we need to parse the
10 : : * backup manifest supplied by the user taking the incremental backup
11 : : * and extract the required information from it.
12 : : *
13 : : * Portions Copyright (c) 2010-2026, PostgreSQL Global Development Group
14 : : *
15 : : * IDENTIFICATION
16 : : * src/backend/backup/basebackup_incremental.c
17 : : *
18 : : *-------------------------------------------------------------------------
19 : : */
20 : : #include "postgres.h"
21 : :
22 : : #include "access/timeline.h"
23 : : #include "access/xlog.h"
24 : : #include "backup/basebackup_incremental.h"
25 : : #include "backup/walsummary.h"
26 : : #include "common/blkreftable.h"
27 : : #include "common/hashfn.h"
28 : : #include "common/int.h"
29 : : #include "common/parse_manifest.h"
30 : : #include "postmaster/walsummarizer.h"
31 : :
32 : : #define BLOCKS_PER_READ 512
33 : :
34 : : /*
35 : : * We expect to find the last lines of the manifest, including the checksum,
36 : : * in the last MIN_CHUNK bytes of the manifest. We trigger an incremental
37 : : * parse step if we are about to overflow MAX_CHUNK bytes.
38 : : */
39 : : #define MIN_CHUNK 1024
40 : : #define MAX_CHUNK (128 * 1024)
41 : :
42 : : /*
43 : : * Details extracted from the WAL ranges present in the supplied backup manifest.
44 : : */
45 : : typedef struct
46 : : {
47 : : TimeLineID tli;
48 : : XLogRecPtr start_lsn;
49 : : XLogRecPtr end_lsn;
50 : : } backup_wal_range;
51 : :
52 : : /*
53 : : * Details extracted from the file list present in the supplied backup manifest.
54 : : */
55 : : typedef struct
56 : : {
57 : : uint32 status;
58 : : const char *path;
59 : : uint64 size;
60 : : } backup_file_entry;
61 : :
62 : : static uint32 hash_string_pointer(const char *s);
63 : : #define SH_PREFIX backup_file
64 : : #define SH_ELEMENT_TYPE backup_file_entry
65 : : #define SH_KEY_TYPE const char *
66 : : #define SH_KEY path
67 : : #define SH_HASH_KEY(tb, key) hash_string_pointer(key)
68 : : #define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0)
69 : : #define SH_SCOPE static inline
70 : : #define SH_DECLARE
71 : : #define SH_DEFINE
72 : : #include "lib/simplehash.h"
73 : :
74 : : struct IncrementalBackupInfo
75 : : {
76 : : /* Memory context for this object and its subsidiary objects. */
77 : : MemoryContext mcxt;
78 : :
79 : : /* Temporary buffer for storing the manifest while parsing it. */
80 : : StringInfoData buf;
81 : :
82 : : /* WAL ranges extracted from the backup manifest. */
83 : : List *manifest_wal_ranges;
84 : :
85 : : /*
86 : : * Files extracted from the backup manifest.
87 : : *
88 : : * We don't really need this information, because we use WAL summaries to
89 : : * figure out what's changed. It would be unsafe to just rely on the list
90 : : * of files that existed before, because it's possible for a file to be
91 : : * removed and a new one created with the same name and different
92 : : * contents. In such cases, the whole file must still be sent. We can tell
93 : : * from the WAL summaries whether that happened, but not from the file
94 : : * list.
95 : : *
96 : : * Nonetheless, this data is useful for sanity checking. If a file that we
97 : : * think we shouldn't need to send is not present in the manifest for the
98 : : * prior backup, something has gone terribly wrong. We retain the file
99 : : * names and sizes, but not the checksums or last modified times, for
100 : : * which we have no use.
101 : : *
102 : : * One significant downside of storing this data is that it consumes
103 : : * memory. If that turns out to be a problem, we might have to decide not
104 : : * to retain this information, or to make it optional.
105 : : */
106 : : backup_file_hash *manifest_files;
107 : :
108 : : /*
109 : : * Block-reference table for the incremental backup.
110 : : *
111 : : * It's possible that storing the entire block-reference table in memory
112 : : * will be a problem for some users. The in-memory format that we're using
113 : : * here is pretty efficient, converging to little more than 1 bit per
114 : : * block for relation forks with large numbers of modified blocks. It's
115 : : * possible, however, that if you try to perform an incremental backup of
116 : : * a database with a sufficiently large number of relations on a
117 : : * sufficiently small machine, you could run out of memory here. If that
118 : : * turns out to be a problem in practice, we'll need to be more clever.
119 : : */
120 : : BlockRefTable *brtab;
121 : :
122 : : /*
123 : : * State object for incremental JSON parsing
124 : : */
125 : : JsonManifestParseIncrementalState *inc_state;
126 : : };
127 : :
128 : : static void manifest_process_version(JsonManifestParseContext *context,
129 : : int manifest_version);
130 : : static void manifest_process_system_identifier(JsonManifestParseContext *context,
131 : : uint64 manifest_system_identifier);
132 : : static void manifest_process_file(JsonManifestParseContext *context,
133 : : const char *pathname,
134 : : uint64 size,
135 : : pg_checksum_type checksum_type,
136 : : int checksum_length,
137 : : uint8 *checksum_payload);
138 : : static void manifest_process_wal_range(JsonManifestParseContext *context,
139 : : TimeLineID tli,
140 : : XLogRecPtr start_lsn,
141 : : XLogRecPtr end_lsn);
142 : : pg_noreturn static void manifest_report_error(JsonManifestParseContext *context,
143 : : const char *fmt, ...)
144 : : pg_attribute_printf(2, 3);
145 : : static int compare_wal_summary_start_lsn(const ListCell *a, const ListCell *b);
146 : : static int compare_block_numbers(const void *a, const void *b);
147 : :
148 : : /*
149 : : * Create a new object for storing information extracted from the manifest
150 : : * supplied when creating an incremental backup.
151 : : */
152 : : IncrementalBackupInfo *
153 : 16 : CreateIncrementalBackupInfo(MemoryContext mcxt)
154 : : {
155 : : IncrementalBackupInfo *ib;
156 : : MemoryContext oldcontext;
157 : : JsonManifestParseContext *context;
158 : :
159 : 16 : oldcontext = MemoryContextSwitchTo(mcxt);
160 : :
161 : 16 : ib = palloc0_object(IncrementalBackupInfo);
162 : 16 : ib->mcxt = mcxt;
163 : 16 : initStringInfo(&ib->buf);
164 : :
165 : : /*
166 : : * It's hard to guess how many files a "typical" installation will have in
167 : : * the data directory, but a fresh initdb creates almost 1000 files as of
168 : : * this writing, so it seems to make sense for our estimate to
169 : : * substantially higher.
170 : : */
171 : 16 : ib->manifest_files = backup_file_create(mcxt, 10000, NULL);
172 : :
173 : 16 : context = palloc0_object(JsonManifestParseContext);
174 : : /* Parse the manifest. */
175 : 16 : context->private_data = ib;
176 : 16 : context->version_cb = manifest_process_version;
177 : 16 : context->system_identifier_cb = manifest_process_system_identifier;
178 : 16 : context->per_file_cb = manifest_process_file;
179 : 16 : context->per_wal_range_cb = manifest_process_wal_range;
180 : 16 : context->error_cb = manifest_report_error;
181 : :
182 : 16 : ib->inc_state = json_parse_manifest_incremental_init(context);
183 : :
184 : 16 : MemoryContextSwitchTo(oldcontext);
185 : :
186 : 16 : return ib;
187 : : }
188 : :
189 : : /*
190 : : * Before taking an incremental backup, the caller must supply the backup
191 : : * manifest from a prior backup. Each chunk of manifest data received
192 : : * from the client should be passed to this function.
193 : : */
194 : : void
195 : 48 : AppendIncrementalManifestData(IncrementalBackupInfo *ib, const char *data,
196 : : int len)
197 : : {
198 : : MemoryContext oldcontext;
199 : :
200 : : /* Switch to our memory context. */
201 : 48 : oldcontext = MemoryContextSwitchTo(ib->mcxt);
202 : :
203 [ + + + + ]: 48 : if (ib->buf.len > MIN_CHUNK && ib->buf.len + len > MAX_CHUNK)
204 : : {
205 : : /*
206 : : * time for an incremental parse. We'll do all but the last MIN_CHUNK
207 : : * so that we have enough left for the final piece.
208 : : */
209 : 16 : json_parse_manifest_incremental_chunk(ib->inc_state, ib->buf.data,
210 : 16 : ib->buf.len - MIN_CHUNK, false);
211 : : /* now remove what we just parsed */
212 : 15 : memmove(ib->buf.data, ib->buf.data + (ib->buf.len - MIN_CHUNK),
213 : : MIN_CHUNK + 1);
214 : 15 : ib->buf.len = MIN_CHUNK;
215 : : }
216 : :
217 : 47 : appendBinaryStringInfo(&ib->buf, data, len);
218 : :
219 : : /* Switch back to previous memory context. */
220 : 47 : MemoryContextSwitchTo(oldcontext);
221 : 47 : }
222 : :
223 : : /*
224 : : * Finalize an IncrementalBackupInfo object after all manifest data has
225 : : * been supplied via calls to AppendIncrementalManifestData.
226 : : */
227 : : void
228 : 15 : FinalizeIncrementalManifest(IncrementalBackupInfo *ib)
229 : : {
230 : : MemoryContext oldcontext;
231 : :
232 : : /* Switch to our memory context. */
233 : 15 : oldcontext = MemoryContextSwitchTo(ib->mcxt);
234 : :
235 : : /* Parse the last chunk of the manifest */
236 : 15 : json_parse_manifest_incremental_chunk(ib->inc_state, ib->buf.data,
237 : 15 : ib->buf.len, true);
238 : :
239 : : /* Done with the buffer, so release memory. */
240 : 15 : pfree(ib->buf.data);
241 : 15 : ib->buf.data = NULL;
242 : :
243 : : /* Done with inc_state, so release that memory too */
244 : 15 : json_parse_manifest_incremental_shutdown(ib->inc_state);
245 : :
246 : : /* Switch back to previous memory context. */
247 : 15 : MemoryContextSwitchTo(oldcontext);
248 : 15 : }
249 : :
250 : : /*
251 : : * Prepare to take an incremental backup.
252 : : *
253 : : * Before this function is called, AppendIncrementalManifestData and
254 : : * FinalizeIncrementalManifest should have already been called to pass all
255 : : * the manifest data to this object.
256 : : *
257 : : * This function performs sanity checks on the data extracted from the
258 : : * manifest and figures out for which WAL ranges we need summaries, and
259 : : * whether those summaries are available. Then, it reads and combines the
260 : : * data from those summary files. It also updates the backup_state with the
261 : : * reference TLI and LSN for the prior backup.
262 : : */
263 : : void
264 : 15 : PrepareForIncrementalBackup(IncrementalBackupInfo *ib,
265 : : BackupState *backup_state)
266 : : {
267 : : MemoryContext oldcontext;
268 : : List *expectedTLEs;
269 : : List *all_wslist,
270 : 15 : *required_wslist = NIL;
271 : : ListCell *lc;
272 : : TimeLineHistoryEntry **tlep;
273 : : int num_wal_ranges;
274 : 15 : bool found_backup_start_tli = false;
275 : 15 : TimeLineID earliest_wal_range_tli = 0;
276 : 15 : XLogRecPtr earliest_wal_range_start_lsn = InvalidXLogRecPtr;
277 : 15 : TimeLineID latest_wal_range_tli = 0;
278 : :
279 : : Assert(ib->buf.data == NULL);
280 : :
281 : : /* Switch to our memory context. */
282 : 15 : oldcontext = MemoryContextSwitchTo(ib->mcxt);
283 : :
284 : : /*
285 : : * A valid backup manifest must always contain at least one WAL range
286 : : * (usually exactly one, unless the backup spanned a timeline switch).
287 : : */
288 : 15 : num_wal_ranges = list_length(ib->manifest_wal_ranges);
289 [ - + ]: 15 : if (num_wal_ranges == 0)
290 [ # # ]: 0 : ereport(ERROR,
291 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
292 : : errmsg("manifest contains no required WAL ranges")));
293 : :
294 : : /*
295 : : * Match up the TLIs that appear in the WAL ranges of the backup manifest
296 : : * with those that appear in this server's timeline history. We expect
297 : : * every backup_wal_range to match to a TimeLineHistoryEntry; if it does
298 : : * not, that's an error.
299 : : *
300 : : * This loop also decides which of the WAL ranges is the manifest is most
301 : : * ancient and which one is the newest, according to the timeline history
302 : : * of this server, and stores TLIs of those WAL ranges into
303 : : * earliest_wal_range_tli and latest_wal_range_tli. It also updates
304 : : * earliest_wal_range_start_lsn to the start LSN of the WAL range for
305 : : * earliest_wal_range_tli.
306 : : *
307 : : * Note that the return value of readTimeLineHistory puts the latest
308 : : * timeline at the beginning of the list, not the end. Hence, the earliest
309 : : * TLI is the one that occurs nearest the end of the list returned by
310 : : * readTimeLineHistory, and the latest TLI is the one that occurs closest
311 : : * to the beginning.
312 : : */
313 : 15 : expectedTLEs = readTimeLineHistory(backup_state->starttli);
314 : 15 : tlep = palloc0_array(TimeLineHistoryEntry *, num_wal_ranges);
315 [ + + ]: 30 : for (int i = 0; i < num_wal_ranges; ++i)
316 : : {
317 : 15 : backup_wal_range *range = list_nth(ib->manifest_wal_ranges, i);
318 : 15 : bool saw_earliest_wal_range_tli = false;
319 : 15 : bool saw_latest_wal_range_tli = false;
320 : :
321 : : /* Search this server's history for this WAL range's TLI. */
322 [ + - + - : 18 : foreach(lc, expectedTLEs)
+ - ]
323 : : {
324 : 18 : TimeLineHistoryEntry *tle = lfirst(lc);
325 : :
326 [ + + ]: 18 : if (tle->tli == range->tli)
327 : : {
328 : 15 : tlep[i] = tle;
329 : 15 : break;
330 : : }
331 : :
332 [ - + ]: 3 : if (tle->tli == earliest_wal_range_tli)
333 : 0 : saw_earliest_wal_range_tli = true;
334 [ - + ]: 3 : if (tle->tli == latest_wal_range_tli)
335 : 0 : saw_latest_wal_range_tli = true;
336 : : }
337 : :
338 : : /*
339 : : * An incremental backup can only be taken relative to a backup that
340 : : * represents a previous state of this server. If the backup requires
341 : : * WAL from a timeline that's not in our history, that definitely
342 : : * isn't the case.
343 : : */
344 [ - + ]: 15 : if (tlep[i] == NULL)
345 [ # # ]: 0 : ereport(ERROR,
346 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
347 : : errmsg("timeline %u found in manifest, but not in this server's history",
348 : : range->tli)));
349 : :
350 : : /*
351 : : * If we found this TLI in the server's history before encountering
352 : : * the latest TLI seen so far in the server's history, then this TLI
353 : : * is the latest one seen so far.
354 : : *
355 : : * If on the other hand we saw the earliest TLI seen so far before
356 : : * finding this TLI, this TLI is earlier than the earliest one seen so
357 : : * far. And if this is the first TLI for which we've searched, it's
358 : : * also the earliest one seen so far.
359 : : *
360 : : * On the first loop iteration, both things should necessarily be
361 : : * true.
362 : : */
363 [ + - ]: 15 : if (!saw_latest_wal_range_tli)
364 : 15 : latest_wal_range_tli = range->tli;
365 [ - + - - ]: 15 : if (earliest_wal_range_tli == 0 || saw_earliest_wal_range_tli)
366 : : {
367 : 15 : earliest_wal_range_tli = range->tli;
368 : 15 : earliest_wal_range_start_lsn = range->start_lsn;
369 : : }
370 : : }
371 : :
372 : : /*
373 : : * Propagate information about the prior backup into the backup_label that
374 : : * will be generated for this backup.
375 : : */
376 : 15 : backup_state->istartpoint = earliest_wal_range_start_lsn;
377 : 15 : backup_state->istarttli = earliest_wal_range_tli;
378 : :
379 : : /*
380 : : * Sanity check start and end LSNs for the WAL ranges in the manifest.
381 : : *
382 : : * Commonly, there won't be any timeline switches during the prior backup
383 : : * at all, but if there are, they should happen at the same LSNs that this
384 : : * server switched timelines.
385 : : *
386 : : * Whether there are any timeline switches during the prior backup or not,
387 : : * the prior backup shouldn't require any WAL from a timeline prior to the
388 : : * start of that timeline. It also shouldn't require any WAL from later
389 : : * than the start of this backup.
390 : : *
391 : : * If any of these sanity checks fail, one possible explanation is that
392 : : * the user has generated WAL on the same timeline with the same LSNs more
393 : : * than once. For instance, if two standbys running on timeline 1 were
394 : : * both promoted and (due to a broken archiving setup) both selected new
395 : : * timeline ID 2, then it's possible that one of these checks might trip.
396 : : *
397 : : * Note that there are lots of ways for the user to do something very bad
398 : : * without tripping any of these checks, and they are not intended to be
399 : : * comprehensive. It's pretty hard to see how we could be certain of
400 : : * anything here. However, if there's a problem staring us right in the
401 : : * face, it's best to report it, so we do.
402 : : */
403 [ + + ]: 29 : for (int i = 0; i < num_wal_ranges; ++i)
404 : : {
405 : 15 : backup_wal_range *range = list_nth(ib->manifest_wal_ranges, i);
406 : :
407 [ + - ]: 15 : if (range->tli == earliest_wal_range_tli)
408 : : {
409 [ - + ]: 15 : if (range->start_lsn < tlep[i]->begin)
410 [ # # ]: 0 : ereport(ERROR,
411 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
412 : : errmsg("manifest requires WAL from initial timeline %u starting at %X/%08X, but that timeline begins at %X/%08X",
413 : : range->tli,
414 : : LSN_FORMAT_ARGS(range->start_lsn),
415 : : LSN_FORMAT_ARGS(tlep[i]->begin))));
416 : : }
417 : : else
418 : : {
419 [ # # ]: 0 : if (range->start_lsn != tlep[i]->begin)
420 [ # # ]: 0 : ereport(ERROR,
421 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
422 : : errmsg("manifest requires WAL from continuation timeline %u starting at %X/%08X, but that timeline begins at %X/%08X",
423 : : range->tli,
424 : : LSN_FORMAT_ARGS(range->start_lsn),
425 : : LSN_FORMAT_ARGS(tlep[i]->begin))));
426 : : }
427 : :
428 [ + - ]: 15 : if (range->tli == latest_wal_range_tli)
429 : : {
430 : : /*
431 : : * If the prior backup's final timeline is now historic, its WAL
432 : : * range must not extend past the switch point. Otherwise, the
433 : : * backup may contain changes outside this server's history.
434 : : */
435 [ + + + + ]: 15 : if (XLogRecPtrIsValid(tlep[i]->end) && range->end_lsn > tlep[i]->end)
436 [ + - ]: 1 : ereport(ERROR,
437 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
438 : : errmsg("manifest requires WAL from final timeline %u ending at %X/%08X, but this server switched timelines at %X/%08X",
439 : : range->tli,
440 : : LSN_FORMAT_ARGS(range->end_lsn),
441 : : LSN_FORMAT_ARGS(tlep[i]->end))));
442 : :
443 [ - + ]: 14 : if (range->end_lsn > backup_state->startpoint)
444 [ # # ]: 0 : ereport(ERROR,
445 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
446 : : errmsg("manifest requires WAL from final timeline %u ending at %X/%08X, but this backup starts at %X/%08X",
447 : : range->tli,
448 : : LSN_FORMAT_ARGS(range->end_lsn),
449 : : LSN_FORMAT_ARGS(backup_state->startpoint)),
450 : : errhint("This can happen for incremental backups on a standby if there was little activity since the previous backup.")));
451 : : }
452 : : else
453 : : {
454 [ # # ]: 0 : if (range->end_lsn != tlep[i]->end)
455 [ # # ]: 0 : ereport(ERROR,
456 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
457 : : errmsg("manifest requires WAL from non-final timeline %u ending at %X/%08X, but this server switched timelines at %X/%08X",
458 : : range->tli,
459 : : LSN_FORMAT_ARGS(range->end_lsn),
460 : : LSN_FORMAT_ARGS(tlep[i]->end))));
461 : : }
462 : :
463 : : }
464 : :
465 : : /*
466 : : * Wait for WAL summarization to catch up to the backup start LSN. This
467 : : * will throw an error if the WAL summarizer appears to be stuck. If WAL
468 : : * summarization gets disabled while we're waiting, this will return
469 : : * immediately, and we'll error out further down if the WAL summaries are
470 : : * incomplete.
471 : : */
472 : 14 : WaitForWalSummarization(backup_state->startpoint);
473 : :
474 : : /*
475 : : * Retrieve a list of all WAL summaries on any timeline that overlap with
476 : : * the LSN range of interest. We could instead call GetWalSummaries() once
477 : : * per timeline in the loop that follows, but that would involve reading
478 : : * the directory multiple times. It should be mildly faster - and perhaps
479 : : * a bit safer - to do it just once.
480 : : */
481 : 14 : all_wslist = GetWalSummaries(0, earliest_wal_range_start_lsn,
482 : : backup_state->startpoint);
483 : :
484 : : /*
485 : : * We need WAL summaries for everything that happened during the prior
486 : : * backup and everything that happened afterward up until the point where
487 : : * the current backup started.
488 : : */
489 [ + - + - : 16 : foreach(lc, expectedTLEs)
+ - ]
490 : : {
491 : 16 : TimeLineHistoryEntry *tle = lfirst(lc);
492 : 16 : XLogRecPtr tli_start_lsn = tle->begin;
493 : 16 : XLogRecPtr tli_end_lsn = tle->end;
494 : 16 : XLogRecPtr tli_missing_lsn = InvalidXLogRecPtr;
495 : : List *tli_wslist;
496 : :
497 : : /*
498 : : * Working through the history of this server from the current
499 : : * timeline backwards, we skip everything until we find the timeline
500 : : * where this backup started. Most of the time, this means we won't
501 : : * skip anything at all, as it's unlikely that the timeline has
502 : : * changed since the beginning of the backup moments ago.
503 : : */
504 [ + + ]: 16 : if (tle->tli == backup_state->starttli)
505 : : {
506 : 14 : found_backup_start_tli = true;
507 : 14 : tli_end_lsn = backup_state->startpoint;
508 : : }
509 [ - + ]: 2 : else if (!found_backup_start_tli)
510 : 0 : continue;
511 : :
512 : : /*
513 : : * Find the summaries that overlap the LSN range of interest for this
514 : : * timeline. If this is the earliest timeline involved, the range of
515 : : * interest begins with the start LSN of the prior backup; otherwise,
516 : : * it begins at the LSN at which this timeline came into existence. If
517 : : * this is the latest TLI involved, the range of interest ends at the
518 : : * start LSN of the current backup; otherwise, it ends at the point
519 : : * where we switched from this timeline to the next one.
520 : : */
521 [ + + ]: 16 : if (tle->tli == earliest_wal_range_tli)
522 : 14 : tli_start_lsn = earliest_wal_range_start_lsn;
523 : 16 : tli_wslist = FilterWalSummaries(all_wslist, tle->tli,
524 : : tli_start_lsn, tli_end_lsn);
525 : :
526 : : /*
527 : : * There is no guarantee that the WAL summaries we found cover the
528 : : * entire range of LSNs for which summaries are required, or indeed
529 : : * that we found any WAL summaries at all. Check whether we have a
530 : : * problem of that sort.
531 : : */
532 [ - + ]: 16 : if (!WalSummariesAreComplete(tli_wslist, tli_start_lsn, tli_end_lsn,
533 : : &tli_missing_lsn))
534 : : {
535 [ # # ]: 0 : if (!XLogRecPtrIsValid(tli_missing_lsn))
536 [ # # ]: 0 : ereport(ERROR,
537 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
538 : : errmsg("WAL summaries are required on timeline %u from %X/%08X to %X/%08X, but no summaries for that timeline and LSN range exist",
539 : : tle->tli,
540 : : LSN_FORMAT_ARGS(tli_start_lsn),
541 : : LSN_FORMAT_ARGS(tli_end_lsn))));
542 : : else
543 [ # # ]: 0 : ereport(ERROR,
544 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
545 : : errmsg("WAL summaries are required on timeline %u from %X/%08X to %X/%08X, but the summaries for that timeline and LSN range are incomplete",
546 : : tle->tli,
547 : : LSN_FORMAT_ARGS(tli_start_lsn),
548 : : LSN_FORMAT_ARGS(tli_end_lsn)),
549 : : errdetail("The first unsummarized LSN in this range is %X/%08X.",
550 : : LSN_FORMAT_ARGS(tli_missing_lsn))));
551 : : }
552 : :
553 : : /*
554 : : * Remember that we need to read these summaries.
555 : : *
556 : : * Technically, it's possible that this could read more files than
557 : : * required, since tli_wslist in theory could contain redundant
558 : : * summaries. For instance, if we have a summary from 0/10000000 to
559 : : * 0/20000000 and also one from 0/00000000 to 0/30000000, then the
560 : : * latter subsumes the former and the former could be ignored.
561 : : *
562 : : * We ignore this possibility because the WAL summarizer only tries to
563 : : * generate summaries that do not overlap. If somehow they exist,
564 : : * we'll do a bit of extra work but the results should still be
565 : : * correct.
566 : : */
567 : 16 : required_wslist = list_concat(required_wslist, tli_wslist);
568 : :
569 : : /*
570 : : * Timelines earlier than the one in which the prior backup began are
571 : : * not relevant.
572 : : */
573 [ + + ]: 16 : if (tle->tli == earliest_wal_range_tli)
574 : 14 : break;
575 : : }
576 : :
577 : : /*
578 : : * Merge summaries in ascending start LSN order so that applying a limit
579 : : * block does not discard block references from later summaries.
580 : : */
581 : 14 : list_sort(required_wslist, compare_wal_summary_start_lsn);
582 : :
583 : : /*
584 : : * Read all of the required block reference table files and merge all of
585 : : * the data into a single in-memory block reference table.
586 : : *
587 : : * See the comments for struct IncrementalBackupInfo for some thoughts on
588 : : * memory usage.
589 : : */
590 : 14 : ib->brtab = CreateEmptyBlockRefTable();
591 [ + - + + : 39 : foreach(lc, required_wslist)
+ + ]
592 : : {
593 : 25 : WalSummaryFile *ws = lfirst(lc);
594 : : WalSummaryIO wsio;
595 : : BlockRefTableReader *reader;
596 : : RelFileLocator rlocator;
597 : : ForkNumber forknum;
598 : : BlockNumber limit_block;
599 : : BlockNumber blocks[BLOCKS_PER_READ];
600 : :
601 : 25 : wsio.file = OpenWalSummaryFile(ws, false);
602 : 25 : wsio.filepos = 0;
603 [ + + ]: 25 : ereport(DEBUG1,
604 : : (errmsg_internal("reading WAL summary file \"%s\"",
605 : : FilePathName(wsio.file))));
606 : 25 : reader = CreateBlockRefTableReader(ReadWalSummary, &wsio,
607 : : FilePathName(wsio.file),
608 : : ReportWalSummaryError, NULL);
609 [ + + ]: 397 : while (BlockRefTableReaderNextRelation(reader, &rlocator, &forknum,
610 : : &limit_block))
611 : : {
612 : 372 : BlockRefTableSetLimitBlock(ib->brtab, &rlocator,
613 : : forknum, limit_block);
614 : :
615 : : while (1)
616 : 272 : {
617 : : unsigned int nblocks;
618 : :
619 : 644 : nblocks = BlockRefTableReaderGetBlocks(reader, blocks,
620 : : BLOCKS_PER_READ);
621 [ + + ]: 644 : if (nblocks == 0)
622 : 372 : break;
623 : :
624 [ + + ]: 1210 : for (unsigned int i = 0; i < nblocks; ++i)
625 : 938 : BlockRefTableMarkBlockModified(ib->brtab, &rlocator,
626 : : forknum, blocks[i]);
627 : : }
628 : : }
629 : 25 : DestroyBlockRefTableReader(reader);
630 : 25 : FileClose(wsio.file);
631 : : }
632 : :
633 : : /* Switch back to previous memory context. */
634 : 14 : MemoryContextSwitchTo(oldcontext);
635 : 14 : }
636 : :
637 : : /*
638 : : * Get the pathname that should be used when a file is sent incrementally.
639 : : *
640 : : * The result is a palloc'd string.
641 : : */
642 : : char *
643 : 3210 : GetIncrementalFilePath(Oid dboid, Oid spcoid, RelFileNumber relfilenumber,
644 : : ForkNumber forknum, unsigned segno)
645 : : {
646 : : RelPathStr path;
647 : : char *lastslash;
648 : : char *ipath;
649 : :
650 : 3210 : path = GetRelationPath(dboid, spcoid, relfilenumber, INVALID_PROC_NUMBER,
651 : : forknum);
652 : :
653 : 3210 : lastslash = strrchr(path.str, '/');
654 : : Assert(lastslash != NULL);
655 : 3210 : *lastslash = '\0';
656 : :
657 [ - + ]: 3210 : if (segno > 0)
658 : 0 : ipath = psprintf("%s/INCREMENTAL.%s.%u", path.str, lastslash + 1, segno);
659 : : else
660 : 3210 : ipath = psprintf("%s/INCREMENTAL.%s", path.str, lastslash + 1);
661 : :
662 : 3210 : return ipath;
663 : : }
664 : :
665 : : /*
666 : : * How should we back up a particular file as part of an incremental backup?
667 : : *
668 : : * If the return value is BACK_UP_FILE_FULLY, caller should back up the whole
669 : : * file just as if this were not an incremental backup. The contents of the
670 : : * relative_block_numbers array are unspecified in this case.
671 : : *
672 : : * If the return value is BACK_UP_FILE_INCREMENTALLY, caller should include
673 : : * an incremental file in the backup instead of the entire file. On return,
674 : : * *num_blocks_required will be set to the number of blocks that need to be
675 : : * sent, and the actual block numbers will have been stored in
676 : : * relative_block_numbers, which should be an array of at least RELSEG_SIZE.
677 : : * In addition, *truncation_block_length will be set to the value that should
678 : : * be included in the incremental file.
679 : : */
680 : : FileBackupMethod
681 : 27936 : GetFileBackupMethod(IncrementalBackupInfo *ib, const char *path,
682 : : Oid dboid, Oid spcoid,
683 : : RelFileNumber relfilenumber, ForkNumber forknum,
684 : : unsigned segno, size_t size,
685 : : unsigned *num_blocks_required,
686 : : BlockNumber *relative_block_numbers,
687 : : unsigned *truncation_block_length)
688 : : {
689 : : BlockNumber limit_block;
690 : : BlockNumber start_blkno;
691 : : BlockNumber stop_blkno;
692 : : RelFileLocator rlocator;
693 : : BlockRefTableEntry *brtentry;
694 : : unsigned i;
695 : : unsigned nblocks;
696 : :
697 : : /* Should only be called after PrepareForIncrementalBackup. */
698 : : Assert(ib->buf.data == NULL);
699 : :
700 : : /*
701 : : * dboid could be InvalidOid if shared rel, but spcoid and relfilenumber
702 : : * should have legal values.
703 : : */
704 : : Assert(OidIsValid(spcoid));
705 : : Assert(RelFileNumberIsValid(relfilenumber));
706 : :
707 : : /*
708 : : * If the file size is too large or not a multiple of BLCKSZ, then
709 : : * something weird is happening, so give up and send the whole file.
710 : : */
711 [ + - - + ]: 27936 : if ((size % BLCKSZ) != 0 || size / BLCKSZ > RELSEG_SIZE)
712 : 0 : return BACK_UP_FILE_FULLY;
713 : :
714 : : /*
715 : : * The free-space map fork is not properly WAL-logged, so we need to
716 : : * backup the entire file every time.
717 : : */
718 [ + + ]: 27936 : if (forknum == FSM_FORKNUM)
719 : 3418 : return BACK_UP_FILE_FULLY;
720 : :
721 : : /*
722 : : * If this file was not part of the prior backup, back it up fully.
723 : : *
724 : : * If this file was created after the prior backup and before the start of
725 : : * the current backup, then the WAL summary information will tell us to
726 : : * back up the whole file. However, if this file was created after the
727 : : * start of the current backup, then the WAL summary won't know anything
728 : : * about it. Without this logic, we would erroneously conclude that it was
729 : : * OK to send it incrementally.
730 : : *
731 : : * Note that the file could have existed at the time of the prior backup,
732 : : * gotten deleted, and then a new file with the same name could have been
733 : : * created. In that case, this logic won't prevent the file from being
734 : : * backed up incrementally. But, if the deletion happened before the start
735 : : * of the current backup, the limit block will be 0, inducing a full
736 : : * backup. If the deletion happened after the start of the current backup,
737 : : * reconstruction will erroneously combine blocks from the current
738 : : * lifespan of the file with blocks from the previous lifespan -- but in
739 : : * this type of case, WAL replay to reach backup consistency should remove
740 : : * and recreate the file anyway, so the initial bogus contents should not
741 : : * matter.
742 : : */
743 [ + + ]: 24518 : if (backup_file_lookup(ib->manifest_files, path) == NULL)
744 : : {
745 : : char *ipath;
746 : :
747 : 3210 : ipath = GetIncrementalFilePath(dboid, spcoid, relfilenumber,
748 : : forknum, segno);
749 [ + + ]: 3210 : if (backup_file_lookup(ib->manifest_files, ipath) == NULL)
750 : : {
751 : 548 : pfree(ipath);
752 : 548 : return BACK_UP_FILE_FULLY;
753 : : }
754 : 2662 : pfree(ipath);
755 : : }
756 : :
757 : : /*
758 : : * Look up the special block reference table entry for the database as a
759 : : * whole.
760 : : */
761 : 23970 : rlocator.spcOid = spcoid;
762 : 23970 : rlocator.dbOid = dboid;
763 : 23970 : rlocator.relNumber = 0;
764 [ + + ]: 23970 : if (BlockRefTableGetEntry(ib->brtab, &rlocator, MAIN_FORKNUM,
765 : : &limit_block) != NULL)
766 : : {
767 : : /*
768 : : * According to the WAL summary, this database OID/tablespace OID
769 : : * pairing has been created since the previous backup. So, everything
770 : : * in it must be backed up fully.
771 : : */
772 : 522 : return BACK_UP_FILE_FULLY;
773 : : }
774 : :
775 : : /* Look up the block reference table entry for this relfilenode. */
776 : 23448 : rlocator.relNumber = relfilenumber;
777 : 23448 : brtentry = BlockRefTableGetEntry(ib->brtab, &rlocator, forknum,
778 : : &limit_block);
779 : :
780 : : /*
781 : : * If there is no entry, then there have been no WAL-logged changes to the
782 : : * relation since the predecessor backup was taken, so we can back it up
783 : : * incrementally and need not include any modified blocks.
784 : : *
785 : : * However, if the file is zero-length, we should do a full backup,
786 : : * because an incremental file is always more than zero length, and it's
787 : : * silly to take an incremental backup when a full backup would be
788 : : * smaller.
789 : : */
790 [ + + ]: 23448 : if (brtentry == NULL)
791 : : {
792 [ + + ]: 23342 : if (size == 0)
793 : 4754 : return BACK_UP_FILE_FULLY;
794 : 18588 : *num_blocks_required = 0;
795 : 18588 : *truncation_block_length = size / BLCKSZ;
796 : 18588 : return BACK_UP_FILE_INCREMENTALLY;
797 : : }
798 : :
799 : : /*
800 : : * If the limit_block is less than or equal to the point where this
801 : : * segment starts, send the whole file.
802 : : */
803 [ - + ]: 106 : if (limit_block <= segno * RELSEG_SIZE)
804 : 0 : return BACK_UP_FILE_FULLY;
805 : :
806 : : /*
807 : : * Get relevant entries from the block reference table entry.
808 : : *
809 : : * We shouldn't overflow computing the start or stop block numbers, but if
810 : : * it manages to happen somehow, detect it and throw an error.
811 : : */
812 : 106 : start_blkno = segno * RELSEG_SIZE;
813 : 106 : stop_blkno = start_blkno + (size / BLCKSZ);
814 [ + - - + ]: 106 : if (start_blkno / RELSEG_SIZE != segno || stop_blkno < start_blkno)
815 [ # # ]: 0 : ereport(ERROR,
816 : : errcode(ERRCODE_INTERNAL_ERROR),
817 : : errmsg_internal("overflow computing block number bounds for segment %u with size %zu",
818 : : segno, size));
819 : :
820 : : /*
821 : : * This will write *absolute* block numbers into the output array, but
822 : : * we'll transpose them below.
823 : : */
824 : 106 : nblocks = BlockRefTableEntryGetBlocks(brtentry, start_blkno, stop_blkno,
825 : : relative_block_numbers, RELSEG_SIZE);
826 : : Assert(nblocks <= RELSEG_SIZE);
827 : :
828 : : /*
829 : : * If we're going to have to send nearly all of the blocks, then just send
830 : : * the whole file, because that won't require much extra storage or
831 : : * transfer and will speed up and simplify backup restoration. It's not
832 : : * clear what threshold is most appropriate here and perhaps it ought to
833 : : * be configurable, but for now we're just going to say that if we'd need
834 : : * to send 90% of the blocks anyway, give up and send the whole file.
835 : : *
836 : : * NB: If you change the threshold here, at least make sure to back up the
837 : : * file fully when every single block must be sent, because there's
838 : : * nothing good about sending an incremental file in that case.
839 : : */
840 [ + + ]: 106 : if (nblocks * BLCKSZ > size * 0.9)
841 : 44 : return BACK_UP_FILE_FULLY;
842 : :
843 : : /*
844 : : * Looks like we can send an incremental file, so sort the block numbers
845 : : * and then transpose them from absolute block numbers to relative block
846 : : * numbers if necessary.
847 : : *
848 : : * NB: If the block reference table was using the bitmap representation
849 : : * for a given chunk, the block numbers in that chunk will already be
850 : : * sorted, but when the array-of-offsets representation is used, we can
851 : : * receive block numbers here out of order.
852 : : */
853 : 62 : qsort(relative_block_numbers, nblocks, sizeof(BlockNumber),
854 : : compare_block_numbers);
855 [ - + ]: 62 : if (start_blkno != 0)
856 : : {
857 [ # # ]: 0 : for (i = 0; i < nblocks; ++i)
858 : 0 : relative_block_numbers[i] -= start_blkno;
859 : : }
860 : 62 : *num_blocks_required = nblocks;
861 : :
862 : : /*
863 : : * The truncation block length is the minimum length of the reconstructed
864 : : * file. Any block numbers below this threshold that are not present in
865 : : * the backup need to be fetched from the prior backup. At or above this
866 : : * threshold, blocks should only be included in the result if they are
867 : : * present in the backup. (This may require inserting zero blocks if the
868 : : * blocks included in the backup are non-consecutive.)
869 : : */
870 : 62 : *truncation_block_length = size / BLCKSZ;
871 [ + + ]: 62 : if (BlockNumberIsValid(limit_block))
872 : : {
873 : 4 : unsigned relative_limit = limit_block - segno * RELSEG_SIZE;
874 : :
875 : : /*
876 : : * We can't set a truncation_block_length in excess of the limit block
877 : : * number (relativized to the current segment). To do so would be to
878 : : * treat blocks from older backups as valid current contents even if
879 : : * they were subsequently truncated away.
880 : : */
881 [ - + ]: 4 : if (*truncation_block_length < relative_limit)
882 : 0 : *truncation_block_length = relative_limit;
883 : :
884 : : /*
885 : : * We also can't set a truncation_block_length in excess of the
886 : : * segment size, since the reconstructed file can't be larger than
887 : : * that.
888 : : */
889 [ - + ]: 4 : if (*truncation_block_length > RELSEG_SIZE)
890 : 0 : *truncation_block_length = RELSEG_SIZE;
891 : : }
892 : :
893 : : /* Send it incrementally. */
894 : 62 : return BACK_UP_FILE_INCREMENTALLY;
895 : : }
896 : :
897 : : /*
898 : : * Compute the size for a header of an incremental file containing a given
899 : : * number of blocks. The header is rounded to a multiple of BLCKSZ, but
900 : : * only if the file will store some block data.
901 : : */
902 : : size_t
903 : 18650 : GetIncrementalHeaderSize(unsigned num_blocks_required)
904 : : {
905 : : size_t result;
906 : :
907 : : /* Make sure we're not going to overflow. */
908 : : Assert(num_blocks_required <= RELSEG_SIZE);
909 : :
910 : : /*
911 : : * Three four byte quantities (magic number, truncation block length,
912 : : * block count) followed by block numbers.
913 : : */
914 : 18650 : result = 3 * sizeof(uint32) + (sizeof(BlockNumber) * num_blocks_required);
915 : :
916 : : /*
917 : : * Round the header size to a multiple of BLCKSZ - when not a multiple of
918 : : * BLCKSZ, add the missing fraction of a block. But do this only if the
919 : : * file will store data for some blocks, otherwise keep it small.
920 : : */
921 [ + + + - ]: 18650 : if ((num_blocks_required > 0) && (result % BLCKSZ != 0))
922 : 58 : result += BLCKSZ - (result % BLCKSZ);
923 : :
924 : 18650 : return result;
925 : : }
926 : :
927 : : /*
928 : : * Compute the size for an incremental file containing a given number of blocks.
929 : : */
930 : : size_t
931 : 18650 : GetIncrementalFileSize(unsigned num_blocks_required)
932 : : {
933 : : size_t result;
934 : :
935 : : /* Make sure we're not going to overflow. */
936 : : Assert(num_blocks_required <= RELSEG_SIZE);
937 : :
938 : : /*
939 : : * Header with three four byte quantities (magic number, truncation block
940 : : * length, block count) followed by block numbers, rounded to a multiple
941 : : * of BLCKSZ (for files with block data), followed by block contents.
942 : : */
943 : 18650 : result = GetIncrementalHeaderSize(num_blocks_required);
944 : 18650 : result += BLCKSZ * num_blocks_required;
945 : :
946 : 18650 : return result;
947 : : }
948 : :
949 : : /*
950 : : * Helper function for filemap hash table.
951 : : */
952 : : static uint32
953 : 43402 : hash_string_pointer(const char *s)
954 : : {
955 : 43402 : const unsigned char *ss = (const unsigned char *) s;
956 : :
957 : 43402 : return hash_bytes(ss, strlen(s));
958 : : }
959 : :
960 : : /*
961 : : * This callback to validate the manifest version for incremental backup.
962 : : */
963 : : static void
964 : 16 : manifest_process_version(JsonManifestParseContext *context,
965 : : int manifest_version)
966 : : {
967 : : /* Incremental backups don't work with manifest version 1 */
968 [ - + ]: 16 : if (manifest_version == 1)
969 : 0 : context->error_cb(context,
970 : : "backup manifest version 1 does not support incremental backup");
971 : 16 : }
972 : :
973 : : /*
974 : : * This callback to validate the manifest system identifier against the current
975 : : * database server.
976 : : */
977 : : static void
978 : 16 : manifest_process_system_identifier(JsonManifestParseContext *context,
979 : : uint64 manifest_system_identifier)
980 : : {
981 : : uint64 system_identifier;
982 : :
983 : : /* Get system identifier of current system */
984 : 16 : system_identifier = GetSystemIdentifier();
985 : :
986 [ + + ]: 16 : if (manifest_system_identifier != system_identifier)
987 : 1 : context->error_cb(context,
988 : : "system identifier in backup manifest is %" PRIu64 ", but database system identifier is %" PRIu64,
989 : : manifest_system_identifier,
990 : : system_identifier);
991 : 15 : }
992 : :
993 : : /*
994 : : * This callback is invoked for each file mentioned in the backup manifest.
995 : : *
996 : : * We store the path to each file and the size of each file for sanity-checking
997 : : * purposes. For further details, see comments for IncrementalBackupInfo.
998 : : */
999 : : static void
1000 : 15209 : manifest_process_file(JsonManifestParseContext *context,
1001 : : const char *pathname, uint64 size,
1002 : : pg_checksum_type checksum_type,
1003 : : int checksum_length,
1004 : : uint8 *checksum_payload)
1005 : : {
1006 : 15209 : IncrementalBackupInfo *ib = context->private_data;
1007 : : backup_file_entry *entry;
1008 : : bool found;
1009 : :
1010 : 15209 : entry = backup_file_insert(ib->manifest_files, pathname, &found);
1011 [ + - ]: 15209 : if (!found)
1012 : : {
1013 : 15209 : entry->path = MemoryContextStrdup(ib->manifest_files->ctx,
1014 : : pathname);
1015 : 15209 : entry->size = size;
1016 : : }
1017 : 15209 : }
1018 : :
1019 : : /*
1020 : : * This callback is invoked for each WAL range mentioned in the backup
1021 : : * manifest.
1022 : : *
1023 : : * We're just interested in learning the oldest LSN and the corresponding TLI
1024 : : * that appear in any WAL range.
1025 : : */
1026 : : static void
1027 : 15 : manifest_process_wal_range(JsonManifestParseContext *context,
1028 : : TimeLineID tli, XLogRecPtr start_lsn,
1029 : : XLogRecPtr end_lsn)
1030 : : {
1031 : 15 : IncrementalBackupInfo *ib = context->private_data;
1032 : 15 : backup_wal_range *range = palloc_object(backup_wal_range);
1033 : :
1034 : 15 : range->tli = tli;
1035 : 15 : range->start_lsn = start_lsn;
1036 : 15 : range->end_lsn = end_lsn;
1037 : 15 : ib->manifest_wal_ranges = lappend(ib->manifest_wal_ranges, range);
1038 : 15 : }
1039 : :
1040 : : /*
1041 : : * This callback is invoked if an error occurs while parsing the backup
1042 : : * manifest.
1043 : : */
1044 : : static void
1045 : 1 : manifest_report_error(JsonManifestParseContext *context, const char *fmt, ...)
1046 : : {
1047 : : StringInfoData errbuf;
1048 : :
1049 : 1 : initStringInfo(&errbuf);
1050 : :
1051 : : for (;;)
1052 : 0 : {
1053 : : va_list ap;
1054 : : int needed;
1055 : :
1056 : 1 : va_start(ap, fmt);
1057 : 1 : needed = appendStringInfoVA(&errbuf, fmt, ap);
1058 : 1 : va_end(ap);
1059 [ + - ]: 1 : if (needed == 0)
1060 : 1 : break;
1061 : 0 : enlargeStringInfo(&errbuf, needed);
1062 : : }
1063 : :
1064 [ + - ]: 1 : ereport(ERROR,
1065 : : errmsg_internal("%s", errbuf.data));
1066 : : }
1067 : :
1068 : : /*
1069 : : * List comparator for WAL summary files, ordered by ascending start LSN.
1070 : : */
1071 : : static int
1072 : 19 : compare_wal_summary_start_lsn(const ListCell *a, const ListCell *b)
1073 : : {
1074 : 19 : WalSummaryFile *ws1 = lfirst(a);
1075 : 19 : WalSummaryFile *ws2 = lfirst(b);
1076 : :
1077 : 19 : return pg_cmp_u64(ws1->start_lsn, ws2->start_lsn);
1078 : : }
1079 : :
1080 : : /*
1081 : : * Quicksort comparator for block numbers.
1082 : : */
1083 : : static int
1084 : 28 : compare_block_numbers(const void *a, const void *b)
1085 : : {
1086 : 28 : BlockNumber aa = *(const BlockNumber *) a;
1087 : 28 : BlockNumber bb = *(const BlockNumber *) b;
1088 : :
1089 : 28 : return pg_cmp_u32(aa, bb);
1090 : : }
|
