Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * reconstruct.c
4 : : * Reconstruct full file from incremental file and backup chain.
5 : : *
6 : : * Copyright (c) 2017-2026, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/bin/pg_combinebackup/reconstruct.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres_fe.h"
14 : :
15 : : #include <unistd.h>
16 : :
17 : : #include "backup/basebackup_incremental.h"
18 : : #include "common/file_perm.h"
19 : : #include "common/logging.h"
20 : : #include "copy_file.h"
21 : : #include "lib/stringinfo.h"
22 : : #include "reconstruct.h"
23 : : #include "storage/block.h"
24 : :
25 : : /*
26 : : * An rfile stores the data that we need in order to be able to use some file
27 : : * on disk for reconstruction. For any given output file, we create one rfile
28 : : * per backup that we need to consult when we constructing that output file.
29 : : *
30 : : * If we find a full version of the file in the backup chain, then only
31 : : * filename and fd are initialized; the remaining fields are 0 or NULL.
32 : : * For an incremental file, header_length, num_blocks, relative_block_numbers,
33 : : * and truncation_block_length are also set.
34 : : *
35 : : * num_blocks_read and highest_offset_read always start out as 0.
36 : : */
37 : : typedef struct rfile
38 : : {
39 : : char *filename;
40 : : int fd;
41 : : size_t header_length;
42 : : unsigned num_blocks;
43 : : BlockNumber *relative_block_numbers;
44 : : unsigned truncation_block_length;
45 : : unsigned num_blocks_read;
46 : : off_t highest_offset_read;
47 : : } rfile;
48 : :
49 : : static void debug_reconstruction(int n_source,
50 : : rfile **sources,
51 : : bool dry_run);
52 : : static unsigned find_reconstructed_block_length(rfile *s);
53 : : static rfile *make_incremental_rfile(char *filename);
54 : : static rfile *make_rfile(char *filename, bool missing_ok);
55 : : static void write_reconstructed_file(char *input_filename,
56 : : char *output_filename,
57 : : unsigned block_length,
58 : : rfile **sourcemap,
59 : : off_t *offsetmap,
60 : : pg_checksum_context *checksum_ctx,
61 : : CopyMethod copy_method,
62 : : bool debug,
63 : : bool dry_run);
64 : : static void read_bytes(rfile *rf, void *buffer, unsigned length);
65 : : static void write_block(int fd, char *output_filename,
66 : : uint8 *buffer,
67 : : pg_checksum_context *checksum_ctx);
68 : : static void read_block(rfile *s, off_t off, uint8 *buffer);
69 : :
70 : : /*
71 : : * Reconstruct a full file from an incremental file and a chain of prior
72 : : * backups.
73 : : *
74 : : * input_filename should be the path to the incremental file, and
75 : : * output_filename should be the path where the reconstructed file is to be
76 : : * written.
77 : : *
78 : : * relative_path should be the path to the directory containing this file,
79 : : * relative to the root of the backup (NOT relative to the root of the
80 : : * tablespace). It must always end with a trailing slash. bare_file_name
81 : : * should be the name of the file within that directory, without
82 : : * "INCREMENTAL.".
83 : : *
84 : : * n_prior_backups is the number of prior backups, and prior_backup_dirs is
85 : : * an array of pathnames where those backups can be found.
86 : : */
87 : : void
88 : 7535 : reconstruct_from_incremental_file(char *input_filename,
89 : : char *output_filename,
90 : : char *relative_path,
91 : : char *bare_file_name,
92 : : int n_prior_backups,
93 : : char **prior_backup_dirs,
94 : : manifest_data **manifests,
95 : : char *manifest_path,
96 : : pg_checksum_type checksum_type,
97 : : int *checksum_length,
98 : : uint8 **checksum_payload,
99 : : CopyMethod copy_method,
100 : : bool debug,
101 : : bool dry_run)
102 : : {
103 : : rfile **source;
104 : 7535 : rfile *latest_source = NULL;
105 : : rfile **sourcemap;
106 : : off_t *offsetmap;
107 : : unsigned block_length;
108 : 7535 : unsigned sidx = n_prior_backups;
109 : 7535 : bool full_copy_possible = true;
110 : 7535 : int copy_source_index = -1;
111 : 7535 : rfile *copy_source = NULL;
112 : : pg_checksum_context checksum_ctx;
113 : :
114 : : /* Sanity check the relative_path. */
115 : : Assert(relative_path[0] != '\0');
116 : : Assert(relative_path[strlen(relative_path) - 1] == '/');
117 : :
118 : : /*
119 : : * Every block must come either from the latest version of the file or
120 : : * from one of the prior backups.
121 : : */
122 : 7535 : source = pg_malloc0_array(rfile *, 1 + n_prior_backups);
123 : :
124 : : /*
125 : : * Use the information from the latest incremental file to figure out how
126 : : * long the reconstructed file should be.
127 : : */
128 : 7535 : latest_source = make_incremental_rfile(input_filename);
129 : 7535 : source[n_prior_backups] = latest_source;
130 : 7535 : block_length = find_reconstructed_block_length(latest_source);
131 : :
132 : : /*
133 : : * For each block in the output file, we need to know from which file we
134 : : * need to obtain it and at what offset in that file it's stored.
135 : : * sourcemap gives us the first of these things, and offsetmap the latter.
136 : : */
137 : 7535 : sourcemap = pg_malloc0_array(rfile *, block_length);
138 : 7535 : offsetmap = pg_malloc0_array(off_t, block_length);
139 : :
140 : : /*
141 : : * Every block that is present in the newest incremental file should be
142 : : * sourced from that file. If it precedes the truncation_block_length,
143 : : * it's a block that we would otherwise have had to find in an older
144 : : * backup and thus reduces the number of blocks remaining to be found by
145 : : * one; otherwise, it's an extra block that needs to be included in the
146 : : * output but would not have needed to be found in an older backup if it
147 : : * had not been present.
148 : : */
149 [ + + ]: 7571 : for (unsigned i = 0; i < latest_source->num_blocks; ++i)
150 : : {
151 : 36 : BlockNumber b = latest_source->relative_block_numbers[i];
152 : :
153 : : Assert(b < block_length);
154 : 36 : sourcemap[b] = latest_source;
155 : 36 : offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
156 : :
157 : : /*
158 : : * A full copy of a file from an earlier backup is only possible if no
159 : : * blocks are needed from any later incremental file.
160 : : */
161 : 36 : full_copy_possible = false;
162 : : }
163 : :
164 : : while (1)
165 : 1410 : {
166 : : char source_filename[MAXPGPATH];
167 : : rfile *s;
168 : :
169 : : /*
170 : : * Move to the next backup in the chain. If there are no more, then
171 : : * we're done.
172 : : */
173 [ + + ]: 8945 : if (sidx == 0)
174 : 1 : break;
175 : 8944 : --sidx;
176 : :
177 : : /*
178 : : * Look for the full file in the previous backup. If not found, then
179 : : * look for an incremental file instead.
180 : : */
181 : 8944 : snprintf(source_filename, MAXPGPATH, "%s/%s%s",
182 : 8944 : prior_backup_dirs[sidx], relative_path, bare_file_name);
183 [ + + ]: 8944 : if ((s = make_rfile(source_filename, true)) == NULL)
184 : : {
185 : 1410 : snprintf(source_filename, MAXPGPATH, "%s/%sINCREMENTAL.%s",
186 : 1410 : prior_backup_dirs[sidx], relative_path, bare_file_name);
187 : 1410 : s = make_incremental_rfile(source_filename);
188 : : }
189 : 8944 : source[sidx] = s;
190 : :
191 : : /*
192 : : * If s->header_length == 0, then this is a full file; otherwise, it's
193 : : * an incremental file.
194 : : */
195 [ + + ]: 8944 : if (s->header_length == 0)
196 : : {
197 : : struct stat sb;
198 : : BlockNumber b;
199 : : BlockNumber blocklength;
200 : :
201 : : /* We need to know the length of the file. */
202 [ - + ]: 7534 : if (fstat(s->fd, &sb) < 0)
203 : 0 : pg_fatal("could not stat file \"%s\": %m", s->filename);
204 : :
205 : : /*
206 : : * Since we found a full file, source all blocks from it that
207 : : * exist in the file.
208 : : *
209 : : * Note that there may be blocks that don't exist either in this
210 : : * file or in any incremental file but that precede
211 : : * truncation_block_length. These are, presumably, zero-filled
212 : : * blocks that result from the server extending the file but
213 : : * taking no action on those blocks that generated any WAL.
214 : : *
215 : : * Sadly, we have no way of validating that this is really what
216 : : * happened, and neither does the server. From its perspective,
217 : : * an unmodified block that contains data looks exactly the same
218 : : * as a zero-filled block that never had any data: either way,
219 : : * it's not mentioned in any WAL summary and the server has no
220 : : * reason to read it. From our perspective, all we know is that
221 : : * nobody had a reason to back up the block. That certainly means
222 : : * that the block didn't exist at the time of the full backup, but
223 : : * the supposition that it was all zeroes at the time of every
224 : : * later backup is one that we can't validate.
225 : : */
226 : 7534 : blocklength = sb.st_size / BLCKSZ;
227 [ + + ]: 35540 : for (b = 0; b < latest_source->truncation_block_length; ++b)
228 : : {
229 [ + + + - ]: 28006 : if (sourcemap[b] == NULL && b < blocklength)
230 : : {
231 : 27970 : sourcemap[b] = s;
232 : 27970 : offsetmap[b] = b * BLCKSZ;
233 : : }
234 : : }
235 : :
236 : : /*
237 : : * If a full copy looks possible, check whether the resulting file
238 : : * should be exactly as long as the source file is. If so, a full
239 : : * copy is acceptable, otherwise not.
240 : : */
241 [ + + ]: 7534 : if (full_copy_possible)
242 : : {
243 : : uint64 expected_length;
244 : :
245 : 7509 : expected_length =
246 : 7509 : (uint64) latest_source->truncation_block_length;
247 : 7509 : expected_length *= BLCKSZ;
248 [ + + ]: 7509 : if (expected_length == sb.st_size)
249 : : {
250 : 7508 : copy_source = s;
251 : 7508 : copy_source_index = sidx;
252 : : }
253 : : }
254 : :
255 : : /* We don't need to consider any further sources. */
256 : 7534 : break;
257 : : }
258 : :
259 : : /*
260 : : * Since we found another incremental file, source all blocks from it
261 : : * that we need but don't yet have.
262 : : */
263 [ - + ]: 1410 : for (unsigned i = 0; i < s->num_blocks; ++i)
264 : : {
265 : 0 : BlockNumber b = s->relative_block_numbers[i];
266 : :
267 [ # # ]: 0 : if (b < latest_source->truncation_block_length &&
268 [ # # ]: 0 : sourcemap[b] == NULL)
269 : : {
270 : 0 : sourcemap[b] = s;
271 : 0 : offsetmap[b] = s->header_length + (i * BLCKSZ);
272 : :
273 : : /*
274 : : * A full copy of a file from an earlier backup is only
275 : : * possible if no blocks are needed from any later incremental
276 : : * file.
277 : : */
278 : 0 : full_copy_possible = false;
279 : : }
280 : : }
281 : : }
282 : :
283 : : /*
284 : : * If a checksum of the required type already exists in the
285 : : * backup_manifest for the relevant input directory, we can save some work
286 : : * by reusing that checksum instead of computing a new one.
287 : : */
288 [ + + + - : 7535 : if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
+ - ]
289 : : checksum_type != CHECKSUM_TYPE_NONE)
290 : : {
291 : : manifest_file *mfile;
292 : :
293 : 7508 : mfile = manifest_files_lookup(manifests[copy_source_index]->files,
294 : : manifest_path);
295 [ - + ]: 7508 : if (mfile == NULL)
296 : : {
297 : 0 : char *path = psprintf("%s/backup_manifest",
298 : 0 : prior_backup_dirs[copy_source_index]);
299 : :
300 : : /*
301 : : * The directory is out of sync with the backup_manifest, so emit
302 : : * a warning.
303 : : */
304 : 0 : pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
305 : : path,
306 : : manifest_path);
307 : 0 : pfree(path);
308 : : }
309 [ + - ]: 7508 : else if (mfile->checksum_type == checksum_type)
310 : : {
311 : 7508 : *checksum_length = mfile->checksum_length;
312 : 7508 : *checksum_payload = pg_malloc(*checksum_length);
313 : 7508 : memcpy(*checksum_payload, mfile->checksum_payload,
314 : 7508 : *checksum_length);
315 : 7508 : checksum_type = CHECKSUM_TYPE_NONE;
316 : : }
317 : : }
318 : :
319 : : /* Prepare for checksum calculation, if required. */
320 : 7535 : pg_checksum_init(&checksum_ctx, checksum_type);
321 : :
322 : : /*
323 : : * If the full file can be created by copying a file from an older backup
324 : : * in the chain without needing to overwrite any blocks or truncate the
325 : : * result, then forget about performing reconstruction and just copy that
326 : : * file in its entirety.
327 : : *
328 : : * If we have only incremental files, and there's no full file at any
329 : : * point in the backup chain, something has gone wrong. Emit an error.
330 : : *
331 : : * Otherwise, reconstruct.
332 : : */
333 [ + + ]: 7535 : if (copy_source != NULL)
334 : 7508 : copy_file(copy_source->filename, output_filename,
335 : : &checksum_ctx, copy_method, dry_run);
336 [ + - + + ]: 27 : else if (sidx == 0 && source[0]->header_length != 0)
337 : : {
338 : 1 : pg_fatal("full backup contains unexpected incremental file \"%s\"",
339 : : source[0]->filename);
340 : : }
341 : : else
342 : : {
343 : 26 : write_reconstructed_file(input_filename, output_filename,
344 : : block_length, sourcemap, offsetmap,
345 : : &checksum_ctx, copy_method,
346 : : debug, dry_run);
347 : 26 : debug_reconstruction(n_prior_backups + 1, source, dry_run);
348 : : }
349 : :
350 : : /* Save results of checksum calculation. */
351 [ + + ]: 7534 : if (checksum_type != CHECKSUM_TYPE_NONE)
352 : : {
353 : 26 : *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
354 : 26 : *checksum_length = pg_checksum_final(&checksum_ctx,
355 : : *checksum_payload);
356 : : }
357 : :
358 : : /*
359 : : * Close files and release memory.
360 : : */
361 [ + + ]: 24011 : for (int i = 0; i <= n_prior_backups; ++i)
362 : : {
363 : 16477 : rfile *s = source[i];
364 : :
365 [ - + ]: 16477 : if (s == NULL)
366 : 0 : continue;
367 [ - + ]: 16477 : if (close(s->fd) != 0)
368 : 0 : pg_fatal("could not close file \"%s\": %m", s->filename);
369 [ + + ]: 16477 : if (s->relative_block_numbers != NULL)
370 : 25 : pfree(s->relative_block_numbers);
371 : 16477 : pg_free(s->filename);
372 : 16477 : pg_free(s);
373 : : }
374 : 7534 : pg_free(sourcemap);
375 : 7534 : pg_free(offsetmap);
376 : 7534 : pg_free(source);
377 : 7534 : }
378 : :
379 : : /*
380 : : * Perform post-reconstruction logging and sanity checks.
381 : : */
382 : : static void
383 : 26 : debug_reconstruction(int n_source, rfile **sources, bool dry_run)
384 : : {
385 [ + + ]: 80 : for (int i = 0; i < n_source; ++i)
386 : : {
387 : 54 : rfile *s = sources[i];
388 : :
389 : : /* Ignore source if not used. */
390 [ - + ]: 54 : if (s == NULL)
391 : 0 : continue;
392 : :
393 : : /* If no data is needed from this file, we can ignore it. */
394 [ + + ]: 54 : if (s->num_blocks_read == 0)
395 : 3 : continue;
396 : :
397 : : /* Debug logging. */
398 [ - + ]: 51 : if (dry_run)
399 [ # # ]: 0 : pg_log_debug("would have read %u blocks from \"%s\"",
400 : : s->num_blocks_read, s->filename);
401 : : else
402 [ + - ]: 51 : pg_log_debug("read %u blocks from \"%s\"",
403 : : s->num_blocks_read, s->filename);
404 : :
405 : : /*
406 : : * In dry-run mode, we don't actually try to read data from the file,
407 : : * but we do try to verify that the file is long enough that we could
408 : : * have read the data if we'd tried.
409 : : *
410 : : * If this fails, then it means that a non-dry-run attempt would fail,
411 : : * complaining of not being able to read the required bytes from the
412 : : * file.
413 : : */
414 [ - + ]: 51 : if (dry_run)
415 : : {
416 : : struct stat sb;
417 : :
418 [ # # ]: 0 : if (fstat(s->fd, &sb) < 0)
419 : 0 : pg_fatal("could not stat file \"%s\": %m", s->filename);
420 [ # # ]: 0 : if (sb.st_size < s->highest_offset_read)
421 : 0 : pg_fatal("file \"%s\" is too short: expected %lld, found %lld",
422 : : s->filename,
423 : : (long long) s->highest_offset_read,
424 : : (long long) sb.st_size);
425 : : }
426 : : }
427 : 26 : }
428 : :
429 : : /*
430 : : * When we perform reconstruction using an incremental file, the output file
431 : : * should be at least as long as the truncation_block_length. Any blocks
432 : : * present in the incremental file increase the output length as far as is
433 : : * necessary to include those blocks.
434 : : */
435 : : static unsigned
436 : 7535 : find_reconstructed_block_length(rfile *s)
437 : : {
438 : 7535 : unsigned block_length = s->truncation_block_length;
439 : : unsigned i;
440 : :
441 [ + + ]: 7571 : for (i = 0; i < s->num_blocks; ++i)
442 [ - + ]: 36 : if (s->relative_block_numbers[i] >= block_length)
443 : 0 : block_length = s->relative_block_numbers[i] + 1;
444 : :
445 : 7535 : return block_length;
446 : : }
447 : :
448 : : /*
449 : : * Initialize an incremental rfile, reading the header so that we know which
450 : : * blocks it contains.
451 : : */
452 : : static rfile *
453 : 8945 : make_incremental_rfile(char *filename)
454 : : {
455 : : rfile *rf;
456 : : unsigned magic;
457 : :
458 : 8945 : rf = make_rfile(filename, false);
459 : :
460 : : /* Read and validate magic number. */
461 : 8945 : read_bytes(rf, &magic, sizeof(magic));
462 [ - + ]: 8945 : if (magic != INCREMENTAL_MAGIC)
463 : 0 : pg_fatal("file \"%s\" has bad incremental magic number (0x%x, expected 0x%x)",
464 : : filename, magic, INCREMENTAL_MAGIC);
465 : :
466 : : /* Read block count. */
467 : 8945 : read_bytes(rf, &rf->num_blocks, sizeof(rf->num_blocks));
468 [ - + ]: 8945 : if (rf->num_blocks > RELSEG_SIZE)
469 : 0 : pg_fatal("file \"%s\" has block count %u in excess of segment size %u",
470 : : filename, rf->num_blocks, RELSEG_SIZE);
471 : :
472 : : /* Read truncation block length. */
473 : 8945 : read_bytes(rf, &rf->truncation_block_length,
474 : : sizeof(rf->truncation_block_length));
475 [ - + ]: 8945 : if (rf->truncation_block_length > RELSEG_SIZE)
476 : 0 : pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
477 : : filename, rf->truncation_block_length, RELSEG_SIZE);
478 : :
479 : : /* Read block numbers if there are any. */
480 [ + + ]: 8945 : if (rf->num_blocks > 0)
481 : : {
482 : 25 : rf->relative_block_numbers =
483 : 25 : pg_malloc0_array(BlockNumber, rf->num_blocks);
484 : 25 : read_bytes(rf, rf->relative_block_numbers,
485 : 25 : sizeof(BlockNumber) * rf->num_blocks);
486 : : }
487 : :
488 : : /* Remember length of header. */
489 : 8945 : rf->header_length = sizeof(magic) + sizeof(rf->num_blocks) +
490 : 8945 : sizeof(rf->truncation_block_length) +
491 : 8945 : sizeof(BlockNumber) * rf->num_blocks;
492 : :
493 : : /*
494 : : * Round header length to a multiple of BLCKSZ, so that blocks contents
495 : : * are properly aligned. Only do this when the file actually has data for
496 : : * some blocks.
497 : : */
498 [ + + + - ]: 8945 : if ((rf->num_blocks > 0) && ((rf->header_length % BLCKSZ) != 0))
499 : 25 : rf->header_length += (BLCKSZ - (rf->header_length % BLCKSZ));
500 : :
501 : 8945 : return rf;
502 : : }
503 : :
504 : : /*
505 : : * Allocate and perform basic initialization of an rfile.
506 : : */
507 : : static rfile *
508 : 17889 : make_rfile(char *filename, bool missing_ok)
509 : : {
510 : : rfile *rf;
511 : :
512 : 17889 : rf = pg_malloc0_object(rfile);
513 : 17889 : rf->filename = pstrdup(filename);
514 [ + + ]: 17889 : if ((rf->fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0)
515 : : {
516 [ + - + - ]: 1410 : if (missing_ok && errno == ENOENT)
517 : : {
518 : 1410 : pfree(rf->filename);
519 : 1410 : pg_free(rf);
520 : 1410 : return NULL;
521 : : }
522 : 0 : pg_fatal("could not open file \"%s\": %m", filename);
523 : : }
524 : :
525 : 16479 : return rf;
526 : : }
527 : :
528 : : /*
529 : : * Read the indicated number of bytes from an rfile into the buffer.
530 : : */
531 : : static void
532 : 26860 : read_bytes(rfile *rf, void *buffer, unsigned length)
533 : : {
534 : 26860 : int rb = read(rf->fd, buffer, length);
535 : :
536 [ - + ]: 26860 : if (rb != length)
537 : : {
538 [ # # ]: 0 : if (rb < 0)
539 : 0 : pg_fatal("could not read file \"%s\": %m", rf->filename);
540 : : else
541 : 0 : pg_fatal("could not read file \"%s\": read %d of %u",
542 : : rf->filename, rb, length);
543 : : }
544 : 26860 : }
545 : :
546 : : /*
547 : : * Write out a reconstructed file.
548 : : */
549 : : static void
550 : 26 : write_reconstructed_file(char *input_filename,
551 : : char *output_filename,
552 : : unsigned block_length,
553 : : rfile **sourcemap,
554 : : off_t *offsetmap,
555 : : pg_checksum_context *checksum_ctx,
556 : : CopyMethod copy_method,
557 : : bool debug,
558 : : bool dry_run)
559 : : {
560 : 26 : int wfd = -1;
561 : : unsigned i;
562 : 26 : unsigned zero_blocks = 0;
563 : :
564 : : /* Debugging output. */
565 [ + - ]: 26 : if (debug)
566 : : {
567 : : StringInfoData debug_buf;
568 : 26 : unsigned start_of_range = 0;
569 : 26 : unsigned current_block = 0;
570 : :
571 : : /* Basic information about the output file to be produced. */
572 [ - + ]: 26 : if (dry_run)
573 [ # # ]: 0 : pg_log_debug("would reconstruct \"%s\" (%u blocks, checksum %s)",
574 : : output_filename, block_length,
575 : : pg_checksum_type_name(checksum_ctx->type));
576 : : else
577 [ + - ]: 26 : pg_log_debug("reconstructing \"%s\" (%u blocks, checksum %s)",
578 : : output_filename, block_length,
579 : : pg_checksum_type_name(checksum_ctx->type));
580 : :
581 : : /* Print out the plan for reconstructing this file. */
582 : 26 : initStringInfo(&debug_buf);
583 [ + + ]: 357 : while (current_block < block_length)
584 : : {
585 : 331 : rfile *s = sourcemap[current_block];
586 : :
587 : : /* Extend range, if possible. */
588 [ + + ]: 331 : if (current_block + 1 < block_length &&
589 [ + + ]: 305 : s == sourcemap[current_block + 1])
590 : : {
591 : 268 : ++current_block;
592 : 268 : continue;
593 : : }
594 : :
595 : : /* Add details about this range. */
596 [ - + ]: 63 : if (s == NULL)
597 : : {
598 [ # # ]: 0 : if (current_block == start_of_range)
599 : 0 : appendStringInfo(&debug_buf, " %u:zero", current_block);
600 : : else
601 : 0 : appendStringInfo(&debug_buf, " %u-%u:zero",
602 : : start_of_range, current_block);
603 : : }
604 : : else
605 : : {
606 [ + + ]: 63 : if (current_block == start_of_range)
607 : 37 : appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT,
608 : : current_block, s->filename,
609 : 37 : (uint64) offsetmap[current_block]);
610 : : else
611 : 26 : appendStringInfo(&debug_buf, " %u-%u:%s@" UINT64_FORMAT,
612 : : start_of_range, current_block,
613 : : s->filename,
614 : 26 : (uint64) offsetmap[current_block]);
615 : : }
616 : :
617 : : /* Begin new range. */
618 : 63 : start_of_range = ++current_block;
619 : :
620 : : /* If the output is very long or we are done, dump it now. */
621 [ + + - + ]: 63 : if (current_block == block_length || debug_buf.len > 1024)
622 : : {
623 [ + - ]: 26 : pg_log_debug("reconstruction plan:%s", debug_buf.data);
624 : 26 : resetStringInfo(&debug_buf);
625 : : }
626 : : }
627 : :
628 : : /* Free memory. */
629 : 26 : pfree(debug_buf.data);
630 : : }
631 : :
632 : : /* Open the output file, except in dry_run mode. */
633 [ + - - + ]: 52 : if (!dry_run &&
634 : 26 : (wfd = open(output_filename,
635 : : O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
636 : : pg_file_create_mode)) < 0)
637 : 0 : pg_fatal("could not open file \"%s\": %m", output_filename);
638 : :
639 : : /* Read and write the blocks as required. */
640 [ + + ]: 357 : for (i = 0; i < block_length; ++i)
641 : : {
642 : : uint8 buffer[BLCKSZ];
643 : 331 : rfile *s = sourcemap[i];
644 : :
645 : : /* Update accounting information. */
646 [ - + ]: 331 : if (s == NULL)
647 : 0 : ++zero_blocks;
648 : : else
649 : : {
650 : 331 : s->num_blocks_read++;
651 : 331 : s->highest_offset_read = Max(s->highest_offset_read,
652 : : offsetmap[i] + BLCKSZ);
653 : : }
654 : :
655 : : /* Skip the rest of this in dry-run mode. */
656 [ - + ]: 331 : if (dry_run)
657 : 0 : continue;
658 : :
659 : : /* Read or zero-fill the block as appropriate. */
660 [ - + ]: 331 : if (s == NULL)
661 : : {
662 : : /*
663 : : * New block not mentioned in the WAL summary. Should have been an
664 : : * uninitialized block, so just zero-fill it.
665 : : */
666 : 0 : memset(buffer, 0, BLCKSZ);
667 : :
668 : : /* Write out the block, update the checksum if needed. */
669 : 0 : write_block(wfd, output_filename, buffer, checksum_ctx);
670 : :
671 : : /* Nothing else to do for zero-filled blocks. */
672 : 0 : continue;
673 : : }
674 : :
675 : : /* Copy the block using the appropriate copy method. */
676 [ + - ]: 331 : if (copy_method != COPY_METHOD_COPY_FILE_RANGE)
677 : : {
678 : : /*
679 : : * Read the block from the correct source file, and then write it
680 : : * out, possibly with a checksum update.
681 : : */
682 : 331 : read_block(s, offsetmap[i], buffer);
683 : 331 : write_block(wfd, output_filename, buffer, checksum_ctx);
684 : : }
685 : : else /* use copy_file_range */
686 : : {
687 : : #if defined(HAVE_COPY_FILE_RANGE)
688 : : /* copy_file_range modifies the offset, so use a local copy */
689 : 0 : off_t off = offsetmap[i];
690 : 0 : size_t nwritten = 0;
691 : :
692 : : /*
693 : : * Retry until we've written all the bytes (the offset is updated
694 : : * by copy_file_range, and so is the wfd file offset).
695 : : */
696 : : do
697 : : {
698 : : int wb;
699 : :
700 : 0 : wb = copy_file_range(s->fd, &off, wfd, NULL, BLCKSZ - nwritten, 0);
701 : :
702 [ # # ]: 0 : if (wb < 0)
703 : 0 : pg_fatal("error while copying file range from \"%s\" to \"%s\": %m",
704 : : input_filename, output_filename);
705 [ # # ]: 0 : else if (wb == 0)
706 : 0 : pg_fatal("unexpected end of file while copying file range from \"%s\" to \"%s\"",
707 : : input_filename, output_filename);
708 : :
709 : 0 : nwritten += wb;
710 : :
711 [ # # ]: 0 : } while (BLCKSZ > nwritten);
712 : :
713 : : /*
714 : : * When checksum calculation not needed, we're done, otherwise
715 : : * read the block and pass it to the checksum calculation.
716 : : */
717 [ # # ]: 0 : if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
718 : 0 : continue;
719 : :
720 : 0 : read_block(s, offsetmap[i], buffer);
721 : :
722 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
723 : 0 : pg_fatal("could not update checksum of file \"%s\"",
724 : : output_filename);
725 : : #else
726 : : pg_fatal("copy_file_range not supported on this platform");
727 : : #endif
728 : : }
729 : : }
730 : :
731 : : /* Debugging output. */
732 [ - + ]: 26 : if (zero_blocks > 0)
733 : : {
734 [ # # ]: 0 : if (dry_run)
735 [ # # ]: 0 : pg_log_debug("would have zero-filled %u blocks", zero_blocks);
736 : : else
737 [ # # ]: 0 : pg_log_debug("zero-filled %u blocks", zero_blocks);
738 : : }
739 : :
740 : : /* Close the output file. */
741 [ + - - + ]: 26 : if (wfd >= 0 && close(wfd) != 0)
742 : 0 : pg_fatal("could not close file \"%s\": %m", output_filename);
743 : 26 : }
744 : :
745 : : /*
746 : : * Write the block into the file (using the file descriptor), and
747 : : * if needed update the checksum calculation.
748 : : *
749 : : * The buffer is expected to contain BLCKSZ bytes. The filename is
750 : : * provided only for the error message.
751 : : */
752 : : static void
753 : 331 : write_block(int fd, char *output_filename,
754 : : uint8 *buffer, pg_checksum_context *checksum_ctx)
755 : : {
756 : : int wb;
757 : :
758 [ - + ]: 331 : if ((wb = write(fd, buffer, BLCKSZ)) != BLCKSZ)
759 : : {
760 [ # # ]: 0 : if (wb < 0)
761 : 0 : pg_fatal("could not write file \"%s\": %m", output_filename);
762 : : else
763 : 0 : pg_fatal("could not write file \"%s\": wrote %d of %d",
764 : : output_filename, wb, BLCKSZ);
765 : : }
766 : :
767 : : /* Update the checksum computation. */
768 [ - + ]: 331 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
769 : 0 : pg_fatal("could not update checksum of file \"%s\"",
770 : : output_filename);
771 : 331 : }
772 : :
773 : : /*
774 : : * Read a block of data (BLCKSZ bytes) into the buffer.
775 : : */
776 : : static void
777 : 331 : read_block(rfile *s, off_t off, uint8 *buffer)
778 : : {
779 : : int rb;
780 : :
781 : : /* Read the block from the correct source, except if dry-run. */
782 : 331 : rb = pg_pread(s->fd, buffer, BLCKSZ, off);
783 [ - + ]: 331 : if (rb != BLCKSZ)
784 : : {
785 [ # # ]: 0 : if (rb < 0)
786 : 0 : pg_fatal("could not read from file \"%s\": %m", s->filename);
787 : : else
788 : 0 : pg_fatal("could not read from file \"%s\", offset %lld: read %d of %d",
789 : : s->filename, (long long) off, rb, BLCKSZ);
790 : : }
791 : 331 : }
|
