Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * reconstruct.c
4 : : * Reconstruct full file from incremental file and backup chain.
5 : : *
6 : : * Copyright (c) 2017-2026, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/bin/pg_combinebackup/reconstruct.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres_fe.h"
14 : :
15 : : #include <unistd.h>
16 : :
17 : : #include "backup/basebackup_incremental.h"
18 : : #include "common/file_perm.h"
19 : : #include "common/logging.h"
20 : : #include "copy_file.h"
21 : : #include "lib/stringinfo.h"
22 : : #include "reconstruct.h"
23 : : #include "storage/block.h"
24 : :
25 : : /*
26 : : * An rfile stores the data that we need in order to be able to use some file
27 : : * on disk for reconstruction. For any given output file, we create one rfile
28 : : * per backup that we need to consult when we constructing that output file.
29 : : *
30 : : * If we find a full version of the file in the backup chain, then only
31 : : * filename and fd are initialized; the remaining fields are 0 or NULL.
32 : : * For an incremental file, header_length, num_blocks, relative_block_numbers,
33 : : * and truncation_block_length are also set.
34 : : *
35 : : * num_blocks_read and highest_offset_read always start out as 0.
36 : : */
37 : : typedef struct rfile
38 : : {
39 : : char *filename;
40 : : int fd;
41 : : size_t header_length;
42 : : unsigned num_blocks;
43 : : BlockNumber *relative_block_numbers;
44 : : unsigned truncation_block_length;
45 : : unsigned num_blocks_read;
46 : : off_t highest_offset_read;
47 : : } rfile;
48 : :
49 : : static void debug_reconstruction(int n_source,
50 : : rfile **sources,
51 : : bool dry_run);
52 : : static unsigned find_reconstructed_block_length(const rfile *s);
53 : : static rfile *make_incremental_rfile(const char *filename);
54 : : static rfile *make_rfile(const char *filename, bool missing_ok);
55 : : static void write_reconstructed_file(const char *input_filename,
56 : : const char *output_filename,
57 : : unsigned block_length,
58 : : rfile **sourcemap,
59 : : const off_t *offsetmap,
60 : : pg_checksum_context *checksum_ctx,
61 : : CopyMethod copy_method,
62 : : bool debug,
63 : : bool dry_run);
64 : : static void read_bytes(const rfile *rf, void *buffer, size_t length);
65 : : static void write_block(int fd, const char *output_filename,
66 : : const uint8 *buffer,
67 : : pg_checksum_context *checksum_ctx);
68 : : static void read_block(const rfile *s, off_t off, uint8 *buffer);
69 : :
70 : : /*
71 : : * Reconstruct a full file from an incremental file and a chain of prior
72 : : * backups.
73 : : *
74 : : * input_filename should be the path to the incremental file, and
75 : : * output_filename should be the path where the reconstructed file is to be
76 : : * written.
77 : : *
78 : : * relative_path should be the path to the directory containing this file,
79 : : * relative to the root of the backup (NOT relative to the root of the
80 : : * tablespace). It must always end with a trailing slash. bare_file_name
81 : : * should be the name of the file within that directory, without
82 : : * "INCREMENTAL.".
83 : : *
84 : : * n_prior_backups is the number of prior backups, and prior_backup_dirs is
85 : : * an array of pathnames where those backups can be found.
86 : : */
87 : : void
948 rhaas@postgresql.org 88 :CBC 8241 : reconstruct_from_incremental_file(char *input_filename,
89 : : char *output_filename,
90 : : char *relative_path,
91 : : char *bare_file_name,
92 : : int n_prior_backups,
93 : : char **prior_backup_dirs,
94 : : manifest_data **manifests,
95 : : char *manifest_path,
96 : : pg_checksum_type checksum_type,
97 : : int *checksum_length,
98 : : uint8 **checksum_payload,
99 : : CopyMethod copy_method,
100 : : bool debug,
101 : : bool dry_run)
102 : : {
103 : : rfile **source;
104 : 8241 : rfile *latest_source = NULL;
105 : : rfile **sourcemap;
106 : : off_t *offsetmap;
107 : : unsigned block_length;
108 : 8241 : unsigned sidx = n_prior_backups;
109 : 8241 : bool full_copy_possible = true;
110 : 8241 : int copy_source_index = -1;
111 : 8241 : rfile *copy_source = NULL;
112 : : pg_checksum_context checksum_ctx;
113 : :
114 : : /* Sanity check the relative_path. */
628 115 [ - + ]: 8241 : Assert(relative_path[0] != '\0');
116 [ - + ]: 8241 : Assert(relative_path[strlen(relative_path) - 1] == '/');
117 : :
118 : : /*
119 : : * Every block must come either from the latest version of the file or
120 : : * from one of the prior backups.
121 : : */
148 michael@paquier.xyz 122 : 8241 : source = pg_malloc0_array(rfile *, 1 + n_prior_backups);
123 : :
124 : : /*
125 : : * Use the information from the latest incremental file to figure out how
126 : : * long the reconstructed file should be.
127 : : */
948 rhaas@postgresql.org 128 : 8241 : latest_source = make_incremental_rfile(input_filename);
129 : 8241 : source[n_prior_backups] = latest_source;
130 : 8241 : block_length = find_reconstructed_block_length(latest_source);
131 : :
132 : : /*
133 : : * For each block in the output file, we need to know from which file we
134 : : * need to obtain it and at what offset in that file it's stored.
135 : : * sourcemap gives us the first of these things, and offsetmap the latter.
136 : : */
148 michael@paquier.xyz 137 : 8241 : sourcemap = pg_malloc0_array(rfile *, block_length);
138 : 8241 : offsetmap = pg_malloc0_array(off_t, block_length);
139 : :
140 : : /*
141 : : * Every block that is present in the newest incremental file should be
142 : : * sourced from that file. If it precedes the truncation_block_length,
143 : : * it's a block that we would otherwise have had to find in an older
144 : : * backup and thus reduces the number of blocks remaining to be found by
145 : : * one; otherwise, it's an extra block that needs to be included in the
146 : : * output but would not have needed to be found in an older backup if it
147 : : * had not been present.
148 : : */
14 peter@eisentraut.org 149 [ + + ]:GNC 8280 : for (unsigned i = 0; i < latest_source->num_blocks; ++i)
150 : : {
948 rhaas@postgresql.org 151 :CBC 39 : BlockNumber b = latest_source->relative_block_numbers[i];
152 : :
153 [ - + ]: 39 : Assert(b < block_length);
154 : 39 : sourcemap[b] = latest_source;
155 : 39 : offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
156 : :
157 : : /*
158 : : * A full copy of a file from an earlier backup is only possible if no
159 : : * blocks are needed from any later incremental file.
160 : : */
161 : 39 : full_copy_possible = false;
162 : : }
163 : :
164 : : while (1)
165 : 1410 : {
166 : : char source_filename[MAXPGPATH];
167 : : rfile *s;
168 : :
169 : : /*
170 : : * Move to the next backup in the chain. If there are no more, then
171 : : * we're done.
172 : : */
173 [ + + ]: 9651 : if (sidx == 0)
174 : 1 : break;
175 : 9650 : --sidx;
176 : :
177 : : /*
178 : : * Look for the full file in the previous backup. If not found, then
179 : : * look for an incremental file instead.
180 : : */
628 181 : 9650 : snprintf(source_filename, MAXPGPATH, "%s/%s%s",
948 182 : 9650 : prior_backup_dirs[sidx], relative_path, bare_file_name);
183 [ + + ]: 9650 : if ((s = make_rfile(source_filename, true)) == NULL)
184 : : {
628 185 : 1410 : snprintf(source_filename, MAXPGPATH, "%s/%sINCREMENTAL.%s",
948 186 : 1410 : prior_backup_dirs[sidx], relative_path, bare_file_name);
187 : 1410 : s = make_incremental_rfile(source_filename);
188 : : }
189 : 9650 : source[sidx] = s;
190 : :
191 : : /*
192 : : * If s->header_length == 0, then this is a full file; otherwise, it's
193 : : * an incremental file.
194 : : */
195 [ + + ]: 9650 : if (s->header_length == 0)
196 : : {
197 : : struct stat sb;
198 : : BlockNumber b;
199 : : BlockNumber blocklength;
200 : :
201 : : /* We need to know the length of the file. */
202 [ - + ]: 8240 : if (fstat(s->fd, &sb) < 0)
764 peter@eisentraut.org 203 :UBC 0 : pg_fatal("could not stat file \"%s\": %m", s->filename);
204 : :
205 : : /*
206 : : * Since we found a full file, source all blocks from it that
207 : : * exist in the file.
208 : : *
209 : : * Note that there may be blocks that don't exist either in this
210 : : * file or in any incremental file but that precede
211 : : * truncation_block_length. These are, presumably, zero-filled
212 : : * blocks that result from the server extending the file but
213 : : * taking no action on those blocks that generated any WAL.
214 : : *
215 : : * Sadly, we have no way of validating that this is really what
216 : : * happened, and neither does the server. From its perspective,
217 : : * an unmodified block that contains data looks exactly the same
218 : : * as a zero-filled block that never had any data: either way,
219 : : * it's not mentioned in any WAL summary and the server has no
220 : : * reason to read it. From our perspective, all we know is that
221 : : * nobody had a reason to back up the block. That certainly means
222 : : * that the block didn't exist at the time of the full backup, but
223 : : * the supposition that it was all zeroes at the time of every
224 : : * later backup is one that we can't validate.
225 : : */
948 rhaas@postgresql.org 226 :CBC 8240 : blocklength = sb.st_size / BLCKSZ;
227 [ + + ]: 38870 : for (b = 0; b < latest_source->truncation_block_length; ++b)
228 : : {
229 [ + + + - ]: 30630 : if (sourcemap[b] == NULL && b < blocklength)
230 : : {
231 : 30591 : sourcemap[b] = s;
232 : 30591 : offsetmap[b] = b * BLCKSZ;
233 : : }
234 : : }
235 : :
236 : : /*
237 : : * If a full copy looks possible, check whether the resulting file
238 : : * should be exactly as long as the source file is. If so, a full
239 : : * copy is acceptable, otherwise not.
240 : : */
241 [ + + ]: 8240 : if (full_copy_possible)
242 : : {
243 : : uint64 expected_length;
244 : :
245 : 8213 : expected_length =
246 : 8213 : (uint64) latest_source->truncation_block_length;
247 : 8213 : expected_length *= BLCKSZ;
248 [ + + ]: 8213 : if (expected_length == sb.st_size)
249 : : {
250 : 8212 : copy_source = s;
251 : 8212 : copy_source_index = sidx;
252 : : }
253 : : }
254 : :
255 : : /* We don't need to consider any further sources. */
256 : 8240 : break;
257 : : }
258 : :
259 : : /*
260 : : * Since we found another incremental file, source all blocks from it
261 : : * that we need but don't yet have.
262 : : */
14 peter@eisentraut.org 263 [ - + ]:GNC 1410 : for (unsigned i = 0; i < s->num_blocks; ++i)
264 : : {
948 rhaas@postgresql.org 265 :UBC 0 : BlockNumber b = s->relative_block_numbers[i];
266 : :
267 [ # # ]: 0 : if (b < latest_source->truncation_block_length &&
268 [ # # ]: 0 : sourcemap[b] == NULL)
269 : : {
270 : 0 : sourcemap[b] = s;
271 : 0 : offsetmap[b] = s->header_length + (i * BLCKSZ);
272 : :
273 : : /*
274 : : * A full copy of a file from an earlier backup is only
275 : : * possible if no blocks are needed from any later incremental
276 : : * file.
277 : : */
278 : 0 : full_copy_possible = false;
279 : : }
280 : : }
281 : : }
282 : :
283 : : /*
284 : : * If a checksum of the required type already exists in the
285 : : * backup_manifest for the relevant input directory, we can save some work
286 : : * by reusing that checksum instead of computing a new one.
287 : : */
948 rhaas@postgresql.org 288 [ + + + - :CBC 8241 : if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
+ - ]
289 : : checksum_type != CHECKSUM_TYPE_NONE)
290 : : {
291 : : manifest_file *mfile;
292 : :
293 : 8212 : mfile = manifest_files_lookup(manifests[copy_source_index]->files,
294 : : manifest_path);
295 [ - + ]: 8212 : if (mfile == NULL)
296 : : {
948 rhaas@postgresql.org 297 :UBC 0 : char *path = psprintf("%s/backup_manifest",
298 : 0 : prior_backup_dirs[copy_source_index]);
299 : :
300 : : /*
301 : : * The directory is out of sync with the backup_manifest, so emit
302 : : * a warning.
303 : : */
764 peter@eisentraut.org 304 : 0 : pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
305 : : path,
306 : : manifest_path);
948 rhaas@postgresql.org 307 : 0 : pfree(path);
308 : : }
948 rhaas@postgresql.org 309 [ + - ]:CBC 8212 : else if (mfile->checksum_type == checksum_type)
310 : : {
311 : 8212 : *checksum_length = mfile->checksum_length;
312 : 8212 : *checksum_payload = pg_malloc(*checksum_length);
313 : 8212 : memcpy(*checksum_payload, mfile->checksum_payload,
314 : 8212 : *checksum_length);
315 : 8212 : checksum_type = CHECKSUM_TYPE_NONE;
316 : : }
317 : : }
318 : :
319 : : /* Prepare for checksum calculation, if required. */
320 : 8241 : pg_checksum_init(&checksum_ctx, checksum_type);
321 : :
322 : : /*
323 : : * If the full file can be created by copying a file from an older backup
324 : : * in the chain without needing to overwrite any blocks or truncate the
325 : : * result, then forget about performing reconstruction and just copy that
326 : : * file in its entirety.
327 : : *
328 : : * If we have only incremental files, and there's no full file at any
329 : : * point in the backup chain, something has gone wrong. Emit an error.
330 : : *
331 : : * Otherwise, reconstruct.
332 : : */
333 [ + + ]: 8241 : if (copy_source != NULL)
334 : 8212 : copy_file(copy_source->filename, output_filename,
335 : : &checksum_ctx, copy_method, dry_run);
628 336 [ + - + + ]: 29 : else if (sidx == 0 && source[0]->header_length != 0)
337 : : {
338 : 1 : pg_fatal("full backup contains unexpected incremental file \"%s\"",
339 : : source[0]->filename);
340 : : }
341 : : else
342 : : {
948 343 : 28 : write_reconstructed_file(input_filename, output_filename,
344 : : block_length, sourcemap, offsetmap,
345 : : &checksum_ctx, copy_method,
346 : : debug, dry_run);
347 : 28 : debug_reconstruction(n_prior_backups + 1, source, dry_run);
348 : : }
349 : :
350 : : /* Save results of checksum calculation. */
351 [ + + ]: 8240 : if (checksum_type != CHECKSUM_TYPE_NONE)
352 : : {
353 : 28 : *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
354 : 28 : *checksum_length = pg_checksum_final(&checksum_ctx,
355 : : *checksum_payload);
356 : : }
357 : :
358 : : /*
359 : : * Close files and release memory.
360 : : */
14 peter@eisentraut.org 361 [ + + ]:GNC 26129 : for (int i = 0; i <= n_prior_backups; ++i)
362 : : {
948 rhaas@postgresql.org 363 :CBC 17889 : rfile *s = source[i];
364 : :
365 [ - + ]: 17889 : if (s == NULL)
948 rhaas@postgresql.org 366 :UBC 0 : continue;
948 rhaas@postgresql.org 367 [ - + ]:CBC 17889 : if (close(s->fd) != 0)
764 peter@eisentraut.org 368 :UBC 0 : pg_fatal("could not close file \"%s\": %m", s->filename);
948 rhaas@postgresql.org 369 [ + + ]:CBC 17889 : if (s->relative_block_numbers != NULL)
370 : 27 : pfree(s->relative_block_numbers);
371 : 17889 : pg_free(s->filename);
276 tgl@sss.pgh.pa.us 372 : 17889 : pg_free(s);
373 : : }
24 peter@eisentraut.org 374 :GNC 8240 : pg_free(sourcemap);
375 : 8240 : pg_free(offsetmap);
376 : 8240 : pg_free(source);
948 rhaas@postgresql.org 377 :CBC 8240 : }
378 : :
379 : : /*
380 : : * Perform post-reconstruction logging and sanity checks.
381 : : */
382 : : static void
383 : 28 : debug_reconstruction(int n_source, rfile **sources, bool dry_run)
384 : : {
14 peter@eisentraut.org 385 [ + + ]:GNC 86 : for (int i = 0; i < n_source; ++i)
386 : : {
948 rhaas@postgresql.org 387 :CBC 58 : rfile *s = sources[i];
388 : :
389 : : /* Ignore source if not used. */
390 [ - + ]: 58 : if (s == NULL)
948 rhaas@postgresql.org 391 :UBC 0 : continue;
392 : :
393 : : /* If no data is needed from this file, we can ignore it. */
948 rhaas@postgresql.org 394 [ + + ]:CBC 58 : if (s->num_blocks_read == 0)
395 : 3 : continue;
396 : :
397 : : /* Debug logging. */
398 [ - + ]: 55 : if (dry_run)
948 rhaas@postgresql.org 399 [ # # ]:UBC 0 : pg_log_debug("would have read %u blocks from \"%s\"",
400 : : s->num_blocks_read, s->filename);
401 : : else
948 rhaas@postgresql.org 402 [ + - ]:CBC 55 : pg_log_debug("read %u blocks from \"%s\"",
403 : : s->num_blocks_read, s->filename);
404 : :
405 : : /*
406 : : * In dry-run mode, we don't actually try to read data from the file,
407 : : * but we do try to verify that the file is long enough that we could
408 : : * have read the data if we'd tried.
409 : : *
410 : : * If this fails, then it means that a non-dry-run attempt would fail,
411 : : * complaining of not being able to read the required bytes from the
412 : : * file.
413 : : */
414 [ - + ]: 55 : if (dry_run)
415 : : {
416 : : struct stat sb;
417 : :
948 rhaas@postgresql.org 418 [ # # ]:UBC 0 : if (fstat(s->fd, &sb) < 0)
764 peter@eisentraut.org 419 : 0 : pg_fatal("could not stat file \"%s\": %m", s->filename);
948 rhaas@postgresql.org 420 [ # # ]: 0 : if (sb.st_size < s->highest_offset_read)
18 peter@eisentraut.org 421 :UNC 0 : pg_fatal("file \"%s\" is too short: expected %lld, found %lld",
422 : : s->filename,
423 : : (long long) s->highest_offset_read,
424 : : (long long) sb.st_size);
425 : : }
426 : : }
948 rhaas@postgresql.org 427 :CBC 28 : }
428 : :
429 : : /*
430 : : * When we perform reconstruction using an incremental file, the output file
431 : : * should be at least as long as the truncation_block_length. Any blocks
432 : : * present in the incremental file increase the output length as far as is
433 : : * necessary to include those blocks.
434 : : */
435 : : static unsigned
12 peter@eisentraut.org 436 :GNC 8241 : find_reconstructed_block_length(const rfile *s)
437 : : {
948 rhaas@postgresql.org 438 :CBC 8241 : unsigned block_length = s->truncation_block_length;
439 : : unsigned i;
440 : :
441 [ + + ]: 8280 : for (i = 0; i < s->num_blocks; ++i)
442 [ - + ]: 39 : if (s->relative_block_numbers[i] >= block_length)
948 rhaas@postgresql.org 443 :UBC 0 : block_length = s->relative_block_numbers[i] + 1;
444 : :
948 rhaas@postgresql.org 445 :CBC 8241 : return block_length;
446 : : }
447 : :
448 : : /*
449 : : * Initialize an incremental rfile, reading the header so that we know which
450 : : * blocks it contains.
451 : : */
452 : : static rfile *
12 peter@eisentraut.org 453 :GNC 9651 : make_incremental_rfile(const char *filename)
454 : : {
455 : : rfile *rf;
456 : : unsigned magic;
457 : :
948 rhaas@postgresql.org 458 :CBC 9651 : rf = make_rfile(filename, false);
459 : :
460 : : /* Read and validate magic number. */
461 : 9651 : read_bytes(rf, &magic, sizeof(magic));
462 [ - + ]: 9651 : if (magic != INCREMENTAL_MAGIC)
697 peter@eisentraut.org 463 :UBC 0 : pg_fatal("file \"%s\" has bad incremental magic number (0x%x, expected 0x%x)",
464 : : filename, magic, INCREMENTAL_MAGIC);
465 : :
466 : : /* Read block count. */
948 rhaas@postgresql.org 467 :CBC 9651 : read_bytes(rf, &rf->num_blocks, sizeof(rf->num_blocks));
468 [ - + ]: 9651 : if (rf->num_blocks > RELSEG_SIZE)
948 rhaas@postgresql.org 469 :UBC 0 : pg_fatal("file \"%s\" has block count %u in excess of segment size %u",
470 : : filename, rf->num_blocks, RELSEG_SIZE);
471 : :
472 : : /* Read truncation block length. */
948 rhaas@postgresql.org 473 :CBC 9651 : read_bytes(rf, &rf->truncation_block_length,
474 : : sizeof(rf->truncation_block_length));
475 [ - + ]: 9651 : if (rf->truncation_block_length > RELSEG_SIZE)
948 rhaas@postgresql.org 476 :UBC 0 : pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
477 : : filename, rf->truncation_block_length, RELSEG_SIZE);
478 : :
479 : : /* Read block numbers if there are any. */
948 rhaas@postgresql.org 480 [ + + ]:CBC 9651 : if (rf->num_blocks > 0)
481 : : {
482 : 27 : rf->relative_block_numbers =
148 michael@paquier.xyz 483 : 27 : pg_malloc0_array(BlockNumber, rf->num_blocks);
948 rhaas@postgresql.org 484 : 27 : read_bytes(rf, rf->relative_block_numbers,
485 : 27 : sizeof(BlockNumber) * rf->num_blocks);
486 : : }
487 : :
488 : : /* Remember length of header. */
489 : 9651 : rf->header_length = sizeof(magic) + sizeof(rf->num_blocks) +
490 : 9651 : sizeof(rf->truncation_block_length) +
491 : 9651 : sizeof(BlockNumber) * rf->num_blocks;
492 : :
493 : : /*
494 : : * Round header length to a multiple of BLCKSZ, so that blocks contents
495 : : * are properly aligned. Only do this when the file actually has data for
496 : : * some blocks.
497 : : */
841 tomas.vondra@postgre 498 [ + + + - ]: 9651 : if ((rf->num_blocks > 0) && ((rf->header_length % BLCKSZ) != 0))
499 : 27 : rf->header_length += (BLCKSZ - (rf->header_length % BLCKSZ));
500 : :
948 rhaas@postgresql.org 501 : 9651 : return rf;
502 : : }
503 : :
504 : : /*
505 : : * Allocate and perform basic initialization of an rfile.
506 : : */
507 : : static rfile *
12 peter@eisentraut.org 508 :GNC 19301 : make_rfile(const char *filename, bool missing_ok)
509 : : {
510 : : rfile *rf;
511 : :
148 michael@paquier.xyz 512 :CBC 19301 : rf = pg_malloc0_object(rfile);
948 rhaas@postgresql.org 513 : 19301 : rf->filename = pstrdup(filename);
514 [ + + ]: 19301 : if ((rf->fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0)
515 : : {
516 [ + - + - ]: 1410 : if (missing_ok && errno == ENOENT)
517 : : {
24 peter@eisentraut.org 518 :GNC 1410 : pfree(rf->filename);
948 rhaas@postgresql.org 519 :CBC 1410 : pg_free(rf);
520 : 1410 : return NULL;
521 : : }
948 rhaas@postgresql.org 522 :UBC 0 : pg_fatal("could not open file \"%s\": %m", filename);
523 : : }
524 : :
948 rhaas@postgresql.org 525 :CBC 17891 : return rf;
526 : : }
527 : :
528 : : /*
529 : : * Read the indicated number of bytes from an rfile into the buffer.
530 : : */
531 : : static void
10 peter@eisentraut.org 532 :GNC 28980 : read_bytes(const rfile *rf, void *buffer, size_t length)
533 : : {
534 : : ssize_t rb;
535 : :
536 : 28980 : rb = read(rf->fd, buffer, length);
537 : :
948 rhaas@postgresql.org 538 [ - + ]:CBC 28980 : if (rb != length)
539 : : {
948 rhaas@postgresql.org 540 [ # # ]:UBC 0 : if (rb < 0)
541 : 0 : pg_fatal("could not read file \"%s\": %m", rf->filename);
542 : : else
10 peter@eisentraut.org 543 :UNC 0 : pg_fatal("could not read file \"%s\": read %zd of %zu",
544 : : rf->filename, rb, length);
545 : : }
948 rhaas@postgresql.org 546 :CBC 28980 : }
547 : :
548 : : /*
549 : : * Write out a reconstructed file.
550 : : */
551 : : static void
12 peter@eisentraut.org 552 :GNC 28 : write_reconstructed_file(const char *input_filename,
553 : : const char *output_filename,
554 : : unsigned block_length,
555 : : rfile **sourcemap,
556 : : const off_t *offsetmap,
557 : : pg_checksum_context *checksum_ctx,
558 : : CopyMethod copy_method,
559 : : bool debug,
560 : : bool dry_run)
561 : : {
948 rhaas@postgresql.org 562 :CBC 28 : int wfd = -1;
563 : : unsigned i;
564 : 28 : unsigned zero_blocks = 0;
565 : :
566 : : /* Debugging output. */
567 [ + - ]: 28 : if (debug)
568 : : {
569 : : StringInfoData debug_buf;
570 : 28 : unsigned start_of_range = 0;
571 : 28 : unsigned current_block = 0;
572 : :
573 : : /* Basic information about the output file to be produced. */
574 [ - + ]: 28 : if (dry_run)
948 rhaas@postgresql.org 575 [ # # ]:UBC 0 : pg_log_debug("would reconstruct \"%s\" (%u blocks, checksum %s)",
576 : : output_filename, block_length,
577 : : pg_checksum_type_name(checksum_ctx->type));
578 : : else
948 rhaas@postgresql.org 579 [ + - ]:CBC 28 : pg_log_debug("reconstructing \"%s\" (%u blocks, checksum %s)",
580 : : output_filename, block_length,
581 : : pg_checksum_type_name(checksum_ctx->type));
582 : :
583 : : /* Print out the plan for reconstructing this file. */
584 : 28 : initStringInfo(&debug_buf);
585 [ + + ]: 423 : while (current_block < block_length)
586 : : {
587 : 395 : rfile *s = sourcemap[current_block];
588 : :
589 : : /* Extend range, if possible. */
590 [ + + ]: 395 : if (current_block + 1 < block_length &&
591 [ + + ]: 367 : s == sourcemap[current_block + 1])
592 : : {
593 : 327 : ++current_block;
594 : 327 : continue;
595 : : }
596 : :
597 : : /* Add details about this range. */
598 [ - + ]: 68 : if (s == NULL)
599 : : {
948 rhaas@postgresql.org 600 [ # # ]:UBC 0 : if (current_block == start_of_range)
601 : 0 : appendStringInfo(&debug_buf, " %u:zero", current_block);
602 : : else
603 : 0 : appendStringInfo(&debug_buf, " %u-%u:zero",
604 : : start_of_range, current_block);
605 : : }
606 : : else
607 : : {
948 rhaas@postgresql.org 608 [ + + ]:CBC 68 : if (current_block == start_of_range)
609 : 41 : appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT,
610 : : current_block, s->filename,
611 : 41 : (uint64) offsetmap[current_block]);
612 : : else
613 : 27 : appendStringInfo(&debug_buf, " %u-%u:%s@" UINT64_FORMAT,
614 : : start_of_range, current_block,
615 : : s->filename,
616 : 27 : (uint64) offsetmap[current_block]);
617 : : }
618 : :
619 : : /* Begin new range. */
620 : 68 : start_of_range = ++current_block;
621 : :
622 : : /* If the output is very long or we are done, dump it now. */
623 [ + + - + ]: 68 : if (current_block == block_length || debug_buf.len > 1024)
624 : : {
625 [ + - ]: 28 : pg_log_debug("reconstruction plan:%s", debug_buf.data);
626 : 28 : resetStringInfo(&debug_buf);
627 : : }
628 : : }
629 : :
630 : : /* Free memory. */
631 : 28 : pfree(debug_buf.data);
632 : : }
633 : :
634 : : /* Open the output file, except in dry_run mode. */
635 [ + - - + ]: 56 : if (!dry_run &&
636 : 28 : (wfd = open(output_filename,
637 : : O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
638 : : pg_file_create_mode)) < 0)
948 rhaas@postgresql.org 639 :UBC 0 : pg_fatal("could not open file \"%s\": %m", output_filename);
640 : :
641 : : /* Read and write the blocks as required. */
948 rhaas@postgresql.org 642 [ + + ]:CBC 423 : for (i = 0; i < block_length; ++i)
643 : : {
644 : : uint8 buffer[BLCKSZ];
645 : 395 : rfile *s = sourcemap[i];
646 : :
647 : : /* Update accounting information. */
648 [ - + ]: 395 : if (s == NULL)
948 rhaas@postgresql.org 649 :UBC 0 : ++zero_blocks;
650 : : else
651 : : {
948 rhaas@postgresql.org 652 :CBC 395 : s->num_blocks_read++;
653 : 395 : s->highest_offset_read = Max(s->highest_offset_read,
654 : : offsetmap[i] + BLCKSZ);
655 : : }
656 : :
657 : : /* Skip the rest of this in dry-run mode. */
658 [ - + ]: 395 : if (dry_run)
948 rhaas@postgresql.org 659 :UBC 0 : continue;
660 : :
661 : : /* Read or zero-fill the block as appropriate. */
948 rhaas@postgresql.org 662 [ - + ]:CBC 395 : if (s == NULL)
663 : : {
664 : : /*
665 : : * New block not mentioned in the WAL summary. Should have been an
666 : : * uninitialized block, so just zero-fill it.
667 : : */
948 rhaas@postgresql.org 668 :UBC 0 : memset(buffer, 0, BLCKSZ);
669 : :
670 : : /* Write out the block, update the checksum if needed. */
841 tomas.vondra@postgre 671 : 0 : write_block(wfd, output_filename, buffer, checksum_ctx);
672 : :
673 : : /* Nothing else to do for zero-filled blocks. */
674 : 0 : continue;
675 : : }
676 : :
677 : : /* Copy the block using the appropriate copy method. */
841 tomas.vondra@postgre 678 [ + - ]:CBC 395 : if (copy_method != COPY_METHOD_COPY_FILE_RANGE)
679 : : {
680 : : /*
681 : : * Read the block from the correct source file, and then write it
682 : : * out, possibly with a checksum update.
683 : : */
684 : 395 : read_block(s, offsetmap[i], buffer);
685 : 395 : write_block(wfd, output_filename, buffer, checksum_ctx);
686 : : }
687 : : else /* use copy_file_range */
688 : : {
689 : : #if defined(HAVE_COPY_FILE_RANGE)
690 : : /* copy_file_range modifies the offset, so use a local copy */
841 tomas.vondra@postgre 691 :UBC 0 : off_t off = offsetmap[i];
692 : 0 : size_t nwritten = 0;
693 : :
694 : : /*
695 : : * Retry until we've written all the bytes (the offset is updated
696 : : * by copy_file_range, and so is the wfd file offset).
697 : : */
698 : : do
699 : : {
700 : : ssize_t wb;
701 : :
702 : 0 : wb = copy_file_range(s->fd, &off, wfd, NULL, BLCKSZ - nwritten, 0);
703 : :
704 [ # # ]: 0 : if (wb < 0)
705 : 0 : pg_fatal("error while copying file range from \"%s\" to \"%s\": %m",
706 : : input_filename, output_filename);
26 peter@eisentraut.org 707 [ # # ]: 0 : else if (wb == 0)
708 : 0 : pg_fatal("unexpected end of file while copying file range from \"%s\" to \"%s\"",
709 : : input_filename, output_filename);
710 : :
841 tomas.vondra@postgre 711 : 0 : nwritten += wb;
712 : :
713 [ # # ]: 0 : } while (BLCKSZ > nwritten);
714 : :
715 : : /*
716 : : * When checksum calculation not needed, we're done, otherwise
717 : : * read the block and pass it to the checksum calculation.
718 : : */
719 [ # # ]: 0 : if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
720 : 0 : continue;
721 : :
722 : 0 : read_block(s, offsetmap[i], buffer);
723 : :
724 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
725 : 0 : pg_fatal("could not update checksum of file \"%s\"",
726 : : output_filename);
727 : : #else
728 : : pg_fatal("copy_file_range not supported on this platform");
729 : : #endif
730 : : }
731 : : }
732 : :
733 : : /* Debugging output. */
948 rhaas@postgresql.org 734 [ - + ]:CBC 28 : if (zero_blocks > 0)
735 : : {
948 rhaas@postgresql.org 736 [ # # ]:UBC 0 : if (dry_run)
737 [ # # ]: 0 : pg_log_debug("would have zero-filled %u blocks", zero_blocks);
738 : : else
739 [ # # ]: 0 : pg_log_debug("zero-filled %u blocks", zero_blocks);
740 : : }
741 : :
742 : : /* Close the output file. */
948 rhaas@postgresql.org 743 [ + - - + ]:CBC 28 : if (wfd >= 0 && close(wfd) != 0)
764 peter@eisentraut.org 744 :UBC 0 : pg_fatal("could not close file \"%s\": %m", output_filename);
948 rhaas@postgresql.org 745 :CBC 28 : }
746 : :
747 : : /*
748 : : * Write the block into the file (using the file descriptor), and
749 : : * if needed update the checksum calculation.
750 : : *
751 : : * The buffer is expected to contain BLCKSZ bytes. The filename is
752 : : * provided only for the error message.
753 : : */
754 : : static void
12 peter@eisentraut.org 755 :GNC 395 : write_block(int fd, const char *output_filename,
756 : : const uint8 *buffer, pg_checksum_context *checksum_ctx)
757 : : {
758 : : ssize_t wb;
759 : :
841 tomas.vondra@postgre 760 [ - + ]:CBC 395 : if ((wb = write(fd, buffer, BLCKSZ)) != BLCKSZ)
761 : : {
841 tomas.vondra@postgre 762 [ # # ]:UBC 0 : if (wb < 0)
763 : 0 : pg_fatal("could not write file \"%s\": %m", output_filename);
764 : : else
10 peter@eisentraut.org 765 :UNC 0 : pg_fatal("could not write file \"%s\": wrote %zd of %zu",
766 : : output_filename, wb, (size_t) BLCKSZ);
767 : : }
768 : :
769 : : /* Update the checksum computation. */
841 tomas.vondra@postgre 770 [ - + ]:CBC 395 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
841 tomas.vondra@postgre 771 :UBC 0 : pg_fatal("could not update checksum of file \"%s\"",
772 : : output_filename);
841 tomas.vondra@postgre 773 :CBC 395 : }
774 : :
775 : : /*
776 : : * Read a block of data (BLCKSZ bytes) into the buffer.
777 : : */
778 : : static void
12 peter@eisentraut.org 779 :GNC 395 : read_block(const rfile *s, off_t off, uint8 *buffer)
780 : : {
781 : : ssize_t rb;
782 : :
783 : : /* Read the block from the correct source, except if dry-run. */
841 tomas.vondra@postgre 784 :CBC 395 : rb = pg_pread(s->fd, buffer, BLCKSZ, off);
785 [ - + ]: 395 : if (rb != BLCKSZ)
786 : : {
841 tomas.vondra@postgre 787 [ # # ]:UBC 0 : if (rb < 0)
764 peter@eisentraut.org 788 : 0 : pg_fatal("could not read from file \"%s\": %m", s->filename);
789 : : else
10 peter@eisentraut.org 790 :UNC 0 : pg_fatal("could not read from file \"%s\", offset %lld: read %zd of %zu",
791 : : s->filename, (long long) off, rb, (size_t) BLCKSZ);
792 : : }
841 tomas.vondra@postgre 793 :CBC 395 : }
|
