Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * reconstruct.c
4 : : * Reconstruct full file from incremental file and backup chain.
5 : : *
6 : : * Copyright (c) 2017-2026, PostgreSQL Global Development Group
7 : : *
8 : : * IDENTIFICATION
9 : : * src/bin/pg_combinebackup/reconstruct.c
10 : : *
11 : : *-------------------------------------------------------------------------
12 : : */
13 : : #include "postgres_fe.h"
14 : :
15 : : #include <unistd.h>
16 : :
17 : : #include "backup/basebackup_incremental.h"
18 : : #include "common/file_perm.h"
19 : : #include "common/logging.h"
20 : : #include "copy_file.h"
21 : : #include "lib/stringinfo.h"
22 : : #include "reconstruct.h"
23 : : #include "storage/block.h"
24 : :
25 : : /*
26 : : * An rfile stores the data that we need in order to be able to use some file
27 : : * on disk for reconstruction. For any given output file, we create one rfile
28 : : * per backup that we need to consult when we constructing that output file.
29 : : *
30 : : * If we find a full version of the file in the backup chain, then only
31 : : * filename and fd are initialized; the remaining fields are 0 or NULL.
32 : : * For an incremental file, header_length, num_blocks, relative_block_numbers,
33 : : * and truncation_block_length are also set.
34 : : *
35 : : * num_blocks_read and highest_offset_read always start out as 0.
36 : : */
37 : : typedef struct rfile
38 : : {
39 : : char *filename;
40 : : int fd;
41 : : size_t header_length;
42 : : unsigned num_blocks;
43 : : BlockNumber *relative_block_numbers;
44 : : unsigned truncation_block_length;
45 : : unsigned num_blocks_read;
46 : : off_t highest_offset_read;
47 : : } rfile;
48 : :
49 : : static void debug_reconstruction(int n_source,
50 : : rfile **sources,
51 : : bool dry_run);
52 : : static unsigned find_reconstructed_block_length(const rfile *s);
53 : : static rfile *make_incremental_rfile(const char *filename);
54 : : static rfile *make_rfile(const char *filename, bool missing_ok);
55 : : static void write_reconstructed_file(const char *input_filename,
56 : : const char *output_filename,
57 : : unsigned block_length,
58 : : rfile **sourcemap,
59 : : const off_t *offsetmap,
60 : : pg_checksum_context *checksum_ctx,
61 : : CopyMethod copy_method,
62 : : bool debug,
63 : : bool dry_run);
64 : : static void read_bytes(const rfile *rf, void *buffer, size_t length);
65 : : static void write_block(int fd, const char *output_filename,
66 : : const uint8 *buffer,
67 : : pg_checksum_context *checksum_ctx);
68 : : static void read_block(const rfile *s, off_t off, uint8 *buffer);
69 : :
70 : : /*
71 : : * Reconstruct a full file from an incremental file and a chain of prior
72 : : * backups.
73 : : *
74 : : * input_filename should be the path to the incremental file, and
75 : : * output_filename should be the path where the reconstructed file is to be
76 : : * written.
77 : : *
78 : : * relative_path should be the path to the directory containing this file,
79 : : * relative to the root of the backup (NOT relative to the root of the
80 : : * tablespace). It must always end with a trailing slash. bare_file_name
81 : : * should be the name of the file within that directory, without
82 : : * "INCREMENTAL.".
83 : : *
84 : : * n_prior_backups is the number of prior backups, and prior_backup_dirs is
85 : : * an array of pathnames where those backups can be found.
86 : : */
87 : : void
1051 rhaas@postgresql.org 88 :CBC 9117 : reconstruct_from_incremental_file(char *input_filename,
89 : : char *output_filename,
90 : : char *relative_path,
91 : : char *bare_file_name,
92 : : int n_prior_backups,
93 : : char **prior_backup_dirs,
94 : : manifest_data **manifests,
95 : : char *manifest_path,
96 : : pg_checksum_type checksum_type,
97 : : int *checksum_length,
98 : : uint8 **checksum_payload,
99 : : CopyMethod copy_method,
100 : : bool debug,
101 : : bool dry_run)
102 : : {
103 : : rfile **source;
104 : 9117 : rfile *latest_source = NULL;
105 : : rfile **sourcemap;
106 : : off_t *offsetmap;
107 : : unsigned block_length;
108 : 9117 : unsigned sidx = n_prior_backups;
109 : 9117 : bool full_copy_possible = true;
110 : 9117 : int copy_source_index = -1;
111 : 9117 : rfile *copy_source = NULL;
112 : : pg_checksum_context checksum_ctx;
113 : :
114 : : /* Sanity check the relative_path. */
731 115 [ - + ]: 9117 : Assert(relative_path[0] != '\0');
116 [ - + ]: 9117 : Assert(relative_path[strlen(relative_path) - 1] == '/');
117 : :
118 : : /*
119 : : * Every block must come either from the latest version of the file or
120 : : * from one of the prior backups.
121 : : */
251 michael@paquier.xyz 122 : 9117 : source = pg_malloc0_array(rfile *, 1 + n_prior_backups);
123 : :
124 : : /*
125 : : * Use the information from the latest incremental file to figure out how
126 : : * long the reconstructed file should be.
127 : : */
1051 rhaas@postgresql.org 128 : 9117 : latest_source = make_incremental_rfile(input_filename);
129 : 9117 : source[n_prior_backups] = latest_source;
130 : 9117 : block_length = find_reconstructed_block_length(latest_source);
131 : :
132 : : /*
133 : : * For each block in the output file, we need to know from which file we
134 : : * need to obtain it and at what offset in that file it's stored.
135 : : * sourcemap gives us the first of these things, and offsetmap the latter.
136 : : */
251 michael@paquier.xyz 137 : 9117 : sourcemap = pg_malloc0_array(rfile *, block_length);
138 : 9117 : offsetmap = pg_malloc0_array(off_t, block_length);
139 : :
140 : : /*
141 : : * Every block that is present in the newest incremental file should be
142 : : * sourced from that file. If it precedes the truncation_block_length,
143 : : * it's a block that we would otherwise have had to find in an older
144 : : * backup and thus reduces the number of blocks remaining to be found by
145 : : * one; otherwise, it's an extra block that needs to be included in the
146 : : * output but would not have needed to be found in an older backup if it
147 : : * had not been present.
148 : : */
117 peter@eisentraut.org 149 [ + + ]:GNC 9158 : for (unsigned i = 0; i < latest_source->num_blocks; ++i)
150 : : {
1051 rhaas@postgresql.org 151 :CBC 41 : BlockNumber b = latest_source->relative_block_numbers[i];
152 : :
153 [ - + ]: 41 : Assert(b < block_length);
154 : 41 : sourcemap[b] = latest_source;
155 : 41 : offsetmap[b] = latest_source->header_length + (i * BLCKSZ);
156 : :
157 : : /*
158 : : * A full copy of a file from an earlier backup is only possible if no
159 : : * blocks are needed from any later incremental file.
160 : : */
161 : 41 : full_copy_possible = false;
162 : : }
163 : :
164 : : while (1)
165 : 1332 : {
166 : : char source_filename[MAXPGPATH];
167 : : rfile *s;
168 : :
169 : : /*
170 : : * Move to the next backup in the chain. If there are no more, then
171 : : * we're done.
172 : : */
173 [ + + ]: 10449 : if (sidx == 0)
174 : 1 : break;
175 : 10448 : --sidx;
176 : :
177 : : /*
178 : : * Look for the full file in the previous backup. If not found, then
179 : : * look for an incremental file instead.
180 : : */
731 181 : 10448 : snprintf(source_filename, MAXPGPATH, "%s/%s%s",
1051 182 : 10448 : prior_backup_dirs[sidx], relative_path, bare_file_name);
183 [ + + ]: 10448 : if ((s = make_rfile(source_filename, true)) == NULL)
184 : : {
731 185 : 1332 : snprintf(source_filename, MAXPGPATH, "%s/%sINCREMENTAL.%s",
1051 186 : 1332 : prior_backup_dirs[sidx], relative_path, bare_file_name);
187 : 1332 : s = make_incremental_rfile(source_filename);
188 : : }
189 : 10448 : source[sidx] = s;
190 : :
191 : : /*
192 : : * If s->header_length == 0, then this is a full file; otherwise, it's
193 : : * an incremental file.
194 : : */
195 [ + + ]: 10448 : if (s->header_length == 0)
196 : : {
197 : : struct stat sb;
198 : : BlockNumber b;
199 : : BlockNumber blocklength;
200 : :
201 : : /* We need to know the length of the file. */
202 [ - + ]: 9116 : if (fstat(s->fd, &sb) < 0)
867 peter@eisentraut.org 203 :UBC 0 : pg_fatal("could not stat file \"%s\": %m", s->filename);
204 : :
205 : : /*
206 : : * Since we found a full file, source all blocks from it that
207 : : * exist in the file.
208 : : *
209 : : * Note that there may be blocks that don't exist either in this
210 : : * file or in any incremental file but that precede
211 : : * truncation_block_length. These are, presumably, zero-filled
212 : : * blocks that result from the server extending the file but
213 : : * taking no action on those blocks that generated any WAL.
214 : : *
215 : : * Sadly, we have no way of validating that this is really what
216 : : * happened, and neither does the server. From its perspective,
217 : : * an unmodified block that contains data looks exactly the same
218 : : * as a zero-filled block that never had any data: either way,
219 : : * it's not mentioned in any WAL summary and the server has no
220 : : * reason to read it. From our perspective, all we know is that
221 : : * nobody had a reason to back up the block. That certainly means
222 : : * that the block didn't exist at the time of the full backup, but
223 : : * the supposition that it was all zeroes at the time of every
224 : : * later backup is one that we can't validate.
225 : : */
1051 rhaas@postgresql.org 226 :CBC 9116 : blocklength = sb.st_size / BLCKSZ;
227 [ + + ]: 43875 : for (b = 0; b < latest_source->truncation_block_length; ++b)
228 : : {
229 [ + + + - ]: 34759 : if (sourcemap[b] == NULL && b < blocklength)
230 : : {
231 : 34718 : sourcemap[b] = s;
232 : 34718 : offsetmap[b] = b * BLCKSZ;
233 : : }
234 : : }
235 : :
236 : : /*
237 : : * If a full copy looks possible, check whether the resulting file
238 : : * should be exactly as long as the source file is. If so, a full
239 : : * copy is acceptable, otherwise not.
240 : : */
241 [ + + ]: 9116 : if (full_copy_possible)
242 : : {
243 : : uint64 expected_length;
244 : :
245 : 9087 : expected_length =
246 : 9087 : (uint64) latest_source->truncation_block_length;
247 : 9087 : expected_length *= BLCKSZ;
248 [ + + ]: 9087 : if (expected_length == sb.st_size)
249 : : {
250 : 9085 : copy_source = s;
251 : 9085 : copy_source_index = sidx;
252 : : }
253 : : }
254 : :
255 : : /* We don't need to consider any further sources. */
256 : 9116 : break;
257 : : }
258 : :
259 : : /*
260 : : * Since we found another incremental file, source all blocks from it
261 : : * that we need but don't yet have.
262 : : */
117 peter@eisentraut.org 263 [ - + ]:GNC 1332 : for (unsigned i = 0; i < s->num_blocks; ++i)
264 : : {
1051 rhaas@postgresql.org 265 :UBC 0 : BlockNumber b = s->relative_block_numbers[i];
266 : :
267 [ # # ]: 0 : if (b < latest_source->truncation_block_length &&
268 [ # # ]: 0 : sourcemap[b] == NULL)
269 : : {
270 : 0 : sourcemap[b] = s;
271 : 0 : offsetmap[b] = s->header_length + (i * BLCKSZ);
272 : :
273 : : /*
274 : : * A full copy of a file from an earlier backup is only
275 : : * possible if no blocks are needed from any later incremental
276 : : * file.
277 : : */
278 : 0 : full_copy_possible = false;
279 : : }
280 : : }
281 : : }
282 : :
283 : : /*
284 : : * If a checksum of the required type already exists in the
285 : : * backup_manifest for the relevant input directory, we can save some work
286 : : * by reusing that checksum instead of computing a new one.
287 : : */
1051 rhaas@postgresql.org 288 [ + + + - :CBC 9117 : if (copy_source_index >= 0 && manifests[copy_source_index] != NULL &&
+ - ]
289 : : checksum_type != CHECKSUM_TYPE_NONE)
290 : : {
291 : : manifest_file *mfile;
292 : :
293 : 9085 : mfile = manifest_files_lookup(manifests[copy_source_index]->files,
294 : : manifest_path);
295 [ - + ]: 9085 : if (mfile == NULL)
296 : : {
1051 rhaas@postgresql.org 297 :UBC 0 : char *path = psprintf("%s/backup_manifest",
298 : 0 : prior_backup_dirs[copy_source_index]);
299 : :
300 : : /*
301 : : * The directory is out of sync with the backup_manifest, so emit
302 : : * a warning.
303 : : */
867 peter@eisentraut.org 304 : 0 : pg_log_warning("manifest file \"%s\" contains no entry for file \"%s\"",
305 : : path,
306 : : manifest_path);
1051 rhaas@postgresql.org 307 : 0 : pfree(path);
308 : : }
1051 rhaas@postgresql.org 309 [ + - ]:CBC 9085 : else if (mfile->checksum_type == checksum_type)
310 : : {
311 : 9085 : *checksum_length = mfile->checksum_length;
312 : 9085 : *checksum_payload = pg_malloc(*checksum_length);
313 : 9085 : memcpy(*checksum_payload, mfile->checksum_payload,
314 : 9085 : *checksum_length);
315 : 9085 : checksum_type = CHECKSUM_TYPE_NONE;
316 : : }
317 : : }
318 : :
319 : : /* Prepare for checksum calculation, if required. */
320 : 9117 : pg_checksum_init(&checksum_ctx, checksum_type);
321 : :
322 : : /*
323 : : * If the full file can be created by copying a file from an older backup
324 : : * in the chain without needing to overwrite any blocks or truncate the
325 : : * result, then forget about performing reconstruction and just copy that
326 : : * file in its entirety.
327 : : *
328 : : * If we have only incremental files, and there's no full file at any
329 : : * point in the backup chain, something has gone wrong. Emit an error.
330 : : *
331 : : * Otherwise, reconstruct.
332 : : */
333 [ + + ]: 9117 : if (copy_source != NULL)
334 : 9085 : copy_file(copy_source->filename, output_filename,
335 : : &checksum_ctx, copy_method, dry_run);
731 336 [ + - + + ]: 32 : else if (sidx == 0 && source[0]->header_length != 0)
337 : : {
338 : 1 : pg_fatal("full backup contains unexpected incremental file \"%s\"",
339 : : source[0]->filename);
340 : : }
341 : : else
342 : : {
1051 343 : 31 : write_reconstructed_file(input_filename, output_filename,
344 : : block_length, sourcemap, offsetmap,
345 : : &checksum_ctx, copy_method,
346 : : debug, dry_run);
347 : 31 : debug_reconstruction(n_prior_backups + 1, source, dry_run);
348 : : }
349 : :
350 : : /* Save results of checksum calculation. */
351 [ + + ]: 9116 : if (checksum_type != CHECKSUM_TYPE_NONE)
352 : : {
353 : 31 : *checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
354 : 31 : *checksum_length = pg_checksum_final(&checksum_ctx,
355 : : *checksum_payload);
356 : : }
357 : :
358 : : /*
359 : : * Close files and release memory.
360 : : */
117 peter@eisentraut.org 361 [ + + ]:GNC 28679 : for (int i = 0; i <= n_prior_backups; ++i)
362 : : {
1051 rhaas@postgresql.org 363 :CBC 19563 : rfile *s = source[i];
364 : :
365 [ - + ]: 19563 : if (s == NULL)
1051 rhaas@postgresql.org 366 :UBC 0 : continue;
1051 rhaas@postgresql.org 367 [ - + ]:CBC 19563 : if (close(s->fd) != 0)
867 peter@eisentraut.org 368 :UBC 0 : pg_fatal("could not close file \"%s\": %m", s->filename);
1051 rhaas@postgresql.org 369 [ + + ]:CBC 19563 : if (s->relative_block_numbers != NULL)
370 : 29 : pfree(s->relative_block_numbers);
371 : 19563 : pg_free(s->filename);
379 tgl@sss.pgh.pa.us 372 : 19563 : pg_free(s);
373 : : }
127 peter@eisentraut.org 374 :GNC 9116 : pg_free(sourcemap);
375 : 9116 : pg_free(offsetmap);
376 : 9116 : pg_free(source);
1051 rhaas@postgresql.org 377 :CBC 9116 : }
378 : :
379 : : /*
380 : : * Perform post-reconstruction logging and sanity checks.
381 : : */
382 : : static void
383 : 31 : debug_reconstruction(int n_source, rfile **sources, bool dry_run)
384 : : {
117 peter@eisentraut.org 385 [ + + ]:GNC 95 : for (int i = 0; i < n_source; ++i)
386 : : {
1051 rhaas@postgresql.org 387 :CBC 64 : rfile *s = sources[i];
388 : :
389 : : /* Ignore source if not used. */
390 [ - + ]: 64 : if (s == NULL)
1051 rhaas@postgresql.org 391 :UBC 0 : continue;
392 : :
393 : : /* If no data is needed from this file, we can ignore it. */
1051 rhaas@postgresql.org 394 [ + + ]:CBC 64 : if (s->num_blocks_read == 0)
395 : 4 : continue;
396 : :
397 : : /* Debug logging. */
398 [ - + ]: 60 : if (dry_run)
1051 rhaas@postgresql.org 399 [ # # ]:UBC 0 : pg_log_debug("would have read %u blocks from \"%s\"",
400 : : s->num_blocks_read, s->filename);
401 : : else
1051 rhaas@postgresql.org 402 [ + - ]:CBC 60 : pg_log_debug("read %u blocks from \"%s\"",
403 : : s->num_blocks_read, s->filename);
404 : :
405 : : /*
406 : : * In dry-run mode, we don't actually try to read data from the file,
407 : : * but we do try to verify that the file is long enough that we could
408 : : * have read the data if we'd tried.
409 : : *
410 : : * If this fails, then it means that a non-dry-run attempt would fail,
411 : : * complaining of not being able to read the required bytes from the
412 : : * file.
413 : : */
414 [ - + ]: 60 : if (dry_run)
415 : : {
416 : : struct stat sb;
417 : :
1051 rhaas@postgresql.org 418 [ # # ]:UBC 0 : if (fstat(s->fd, &sb) < 0)
867 peter@eisentraut.org 419 : 0 : pg_fatal("could not stat file \"%s\": %m", s->filename);
1051 rhaas@postgresql.org 420 [ # # ]: 0 : if (sb.st_size < s->highest_offset_read)
121 peter@eisentraut.org 421 :UNC 0 : pg_fatal("file \"%s\" is too short: expected %lld, found %lld",
422 : : s->filename,
423 : : (long long) s->highest_offset_read,
424 : : (long long) sb.st_size);
425 : : }
426 : : }
1051 rhaas@postgresql.org 427 :CBC 31 : }
428 : :
429 : : /*
430 : : * When we perform reconstruction using an incremental file, the output file
431 : : * should be at least as long as the truncation_block_length. Any blocks
432 : : * present in the incremental file increase the output length as far as is
433 : : * necessary to include those blocks.
434 : : */
435 : : static unsigned
115 peter@eisentraut.org 436 :GNC 9117 : find_reconstructed_block_length(const rfile *s)
437 : : {
1051 rhaas@postgresql.org 438 :CBC 9117 : unsigned block_length = s->truncation_block_length;
439 : : unsigned i;
440 : :
441 [ + + ]: 9158 : for (i = 0; i < s->num_blocks; ++i)
442 [ - + ]: 41 : if (s->relative_block_numbers[i] >= block_length)
1051 rhaas@postgresql.org 443 :UBC 0 : block_length = s->relative_block_numbers[i] + 1;
444 : :
1051 rhaas@postgresql.org 445 :CBC 9117 : return block_length;
446 : : }
447 : :
448 : : /*
449 : : * Initialize an incremental rfile, reading the header so that we know which
450 : : * blocks it contains.
451 : : */
452 : : static rfile *
115 peter@eisentraut.org 453 :GNC 10449 : make_incremental_rfile(const char *filename)
454 : : {
455 : : rfile *rf;
456 : : unsigned magic;
457 : :
1051 rhaas@postgresql.org 458 :CBC 10449 : rf = make_rfile(filename, false);
459 : :
460 : : /* Read and validate magic number. */
461 : 10449 : read_bytes(rf, &magic, sizeof(magic));
462 [ - + ]: 10449 : if (magic != INCREMENTAL_MAGIC)
800 peter@eisentraut.org 463 :UBC 0 : pg_fatal("file \"%s\" has bad incremental magic number (0x%x, expected 0x%x)",
464 : : filename, magic, INCREMENTAL_MAGIC);
465 : :
466 : : /* Read block count. */
1051 rhaas@postgresql.org 467 :CBC 10449 : read_bytes(rf, &rf->num_blocks, sizeof(rf->num_blocks));
468 [ - + ]: 10449 : if (rf->num_blocks > RELSEG_SIZE)
1051 rhaas@postgresql.org 469 :UBC 0 : pg_fatal("file \"%s\" has block count %u in excess of segment size %u",
470 : : filename, rf->num_blocks, RELSEG_SIZE);
471 : :
472 : : /* Read truncation block length. */
1051 rhaas@postgresql.org 473 :CBC 10449 : read_bytes(rf, &rf->truncation_block_length,
474 : : sizeof(rf->truncation_block_length));
475 [ - + ]: 10449 : if (rf->truncation_block_length > RELSEG_SIZE)
1051 rhaas@postgresql.org 476 :UBC 0 : pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
477 : : filename, rf->truncation_block_length, RELSEG_SIZE);
478 : :
479 : : /* Read block numbers if there are any. */
1051 rhaas@postgresql.org 480 [ + + ]:CBC 10449 : if (rf->num_blocks > 0)
481 : : {
482 : 29 : rf->relative_block_numbers =
251 michael@paquier.xyz 483 : 29 : pg_malloc0_array(BlockNumber, rf->num_blocks);
1051 rhaas@postgresql.org 484 : 29 : read_bytes(rf, rf->relative_block_numbers,
485 : 29 : sizeof(BlockNumber) * rf->num_blocks);
10 peter@eisentraut.org 486 [ + + ]: 70 : for (unsigned i = 0; i < rf->num_blocks; i++)
487 : : {
488 [ - + ]: 41 : if (rf->relative_block_numbers[i] >= RELSEG_SIZE)
10 peter@eisentraut.org 489 :UBC 0 : pg_fatal("file \"%s\" has block number %u in excess of segment size %u",
490 : : filename, rf->relative_block_numbers[i], RELSEG_SIZE);
491 : : }
492 : : }
493 : :
494 : : /* Remember length of header. */
1051 rhaas@postgresql.org 495 :CBC 10449 : rf->header_length = sizeof(magic) + sizeof(rf->num_blocks) +
496 : 10449 : sizeof(rf->truncation_block_length) +
497 : 10449 : sizeof(BlockNumber) * rf->num_blocks;
498 : :
499 : : /*
500 : : * Round header length to a multiple of BLCKSZ, so that blocks contents
501 : : * are properly aligned. Only do this when the file actually has data for
502 : : * some blocks.
503 : : */
944 tomas.vondra@postgre 504 [ + + + - ]: 10449 : if ((rf->num_blocks > 0) && ((rf->header_length % BLCKSZ) != 0))
505 : 29 : rf->header_length += (BLCKSZ - (rf->header_length % BLCKSZ));
506 : :
1051 rhaas@postgresql.org 507 : 10449 : return rf;
508 : : }
509 : :
510 : : /*
511 : : * Allocate and perform basic initialization of an rfile.
512 : : */
513 : : static rfile *
115 peter@eisentraut.org 514 :GNC 20897 : make_rfile(const char *filename, bool missing_ok)
515 : : {
516 : : rfile *rf;
517 : :
251 michael@paquier.xyz 518 :CBC 20897 : rf = pg_malloc0_object(rfile);
1051 rhaas@postgresql.org 519 : 20897 : rf->filename = pstrdup(filename);
520 [ + + ]: 20897 : if ((rf->fd = open(filename, O_RDONLY | PG_BINARY, 0)) < 0)
521 : : {
522 [ + - + - ]: 1332 : if (missing_ok && errno == ENOENT)
523 : : {
127 peter@eisentraut.org 524 :GNC 1332 : pfree(rf->filename);
1051 rhaas@postgresql.org 525 :CBC 1332 : pg_free(rf);
526 : 1332 : return NULL;
527 : : }
1051 rhaas@postgresql.org 528 :UBC 0 : pg_fatal("could not open file \"%s\": %m", filename);
529 : : }
530 : :
1051 rhaas@postgresql.org 531 :CBC 19565 : return rf;
532 : : }
533 : :
534 : : /*
535 : : * Read the indicated number of bytes from an rfile into the buffer.
536 : : */
537 : : static void
113 peter@eisentraut.org 538 :GNC 31376 : read_bytes(const rfile *rf, void *buffer, size_t length)
539 : : {
540 : : ssize_t rb;
541 : :
542 : 31376 : rb = read(rf->fd, buffer, length);
543 : :
1051 rhaas@postgresql.org 544 [ - + ]:CBC 31376 : if (rb != length)
545 : : {
1051 rhaas@postgresql.org 546 [ # # ]:UBC 0 : if (rb < 0)
547 : 0 : pg_fatal("could not read file \"%s\": %m", rf->filename);
548 : : else
113 peter@eisentraut.org 549 :UNC 0 : pg_fatal("could not read file \"%s\": read %zd of %zu",
550 : : rf->filename, rb, length);
551 : : }
1051 rhaas@postgresql.org 552 :CBC 31376 : }
553 : :
554 : : /*
555 : : * Write out a reconstructed file.
556 : : */
557 : : static void
115 peter@eisentraut.org 558 :GNC 31 : write_reconstructed_file(const char *input_filename,
559 : : const char *output_filename,
560 : : unsigned block_length,
561 : : rfile **sourcemap,
562 : : const off_t *offsetmap,
563 : : pg_checksum_context *checksum_ctx,
564 : : CopyMethod copy_method,
565 : : bool debug,
566 : : bool dry_run)
567 : : {
1051 rhaas@postgresql.org 568 :CBC 31 : int wfd = -1;
569 : : unsigned i;
570 : 31 : unsigned zero_blocks = 0;
571 : :
572 : : /* Debugging output. */
573 [ + - ]: 31 : if (debug)
574 : : {
575 : : StringInfoData debug_buf;
576 : 31 : unsigned start_of_range = 0;
577 : 31 : unsigned current_block = 0;
578 : :
579 : : /* Basic information about the output file to be produced. */
580 [ - + ]: 31 : if (dry_run)
1051 rhaas@postgresql.org 581 [ # # ]:UBC 0 : pg_log_debug("would reconstruct \"%s\" (%u blocks, checksum %s)",
582 : : output_filename, block_length,
583 : : pg_checksum_type_name(checksum_ctx->type));
584 : : else
1051 rhaas@postgresql.org 585 [ + - ]:CBC 31 : pg_log_debug("reconstructing \"%s\" (%u blocks, checksum %s)",
586 : : output_filename, block_length,
587 : : pg_checksum_type_name(checksum_ctx->type));
588 : :
589 : : /* Print out the plan for reconstructing this file. */
590 : 31 : initStringInfo(&debug_buf);
591 [ + + ]: 439 : while (current_block < block_length)
592 : : {
593 : 408 : rfile *s = sourcemap[current_block];
594 : :
595 : : /* Extend range, if possible. */
596 [ + + ]: 408 : if (current_block + 1 < block_length &&
597 [ + + ]: 377 : s == sourcemap[current_block + 1])
598 : : {
599 : 335 : ++current_block;
600 : 335 : continue;
601 : : }
602 : :
603 : : /* Add details about this range. */
604 [ - + ]: 73 : if (s == NULL)
605 : : {
1051 rhaas@postgresql.org 606 [ # # ]:UBC 0 : if (current_block == start_of_range)
607 : 0 : appendStringInfo(&debug_buf, " %u:zero", current_block);
608 : : else
609 : 0 : appendStringInfo(&debug_buf, " %u-%u:zero",
610 : : start_of_range, current_block);
611 : : }
612 : : else
613 : : {
1051 rhaas@postgresql.org 614 [ + + ]:CBC 73 : if (current_block == start_of_range)
615 : 46 : appendStringInfo(&debug_buf, " %u:%s@" UINT64_FORMAT,
616 : : current_block, s->filename,
617 : 46 : (uint64) offsetmap[current_block]);
618 : : else
619 : 27 : appendStringInfo(&debug_buf, " %u-%u:%s@" UINT64_FORMAT,
620 : : start_of_range, current_block,
621 : : s->filename,
622 : 27 : (uint64) offsetmap[current_block]);
623 : : }
624 : :
625 : : /* Begin new range. */
626 : 73 : start_of_range = ++current_block;
627 : :
628 : : /* If the output is very long or we are done, dump it now. */
629 [ + + - + ]: 73 : if (current_block == block_length || debug_buf.len > 1024)
630 : : {
631 [ + - ]: 31 : pg_log_debug("reconstruction plan:%s", debug_buf.data);
632 : 31 : resetStringInfo(&debug_buf);
633 : : }
634 : : }
635 : :
636 : : /* Free memory. */
637 : 31 : pfree(debug_buf.data);
638 : : }
639 : :
640 : : /* Open the output file, except in dry_run mode. */
641 [ + - - + ]: 62 : if (!dry_run &&
642 : 31 : (wfd = open(output_filename,
643 : : O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
644 : : pg_file_create_mode)) < 0)
1051 rhaas@postgresql.org 645 :UBC 0 : pg_fatal("could not open file \"%s\": %m", output_filename);
646 : :
647 : : /* Read and write the blocks as required. */
1051 rhaas@postgresql.org 648 [ + + ]:CBC 439 : for (i = 0; i < block_length; ++i)
649 : : {
650 : : uint8 buffer[BLCKSZ];
651 : 408 : rfile *s = sourcemap[i];
652 : :
653 : : /* Update accounting information. */
654 [ - + ]: 408 : if (s == NULL)
1051 rhaas@postgresql.org 655 :UBC 0 : ++zero_blocks;
656 : : else
657 : : {
1051 rhaas@postgresql.org 658 :CBC 408 : s->num_blocks_read++;
659 : 408 : s->highest_offset_read = Max(s->highest_offset_read,
660 : : offsetmap[i] + BLCKSZ);
661 : : }
662 : :
663 : : /* Skip the rest of this in dry-run mode. */
664 [ - + ]: 408 : if (dry_run)
1051 rhaas@postgresql.org 665 :UBC 0 : continue;
666 : :
667 : : /* Read or zero-fill the block as appropriate. */
1051 rhaas@postgresql.org 668 [ - + ]:CBC 408 : if (s == NULL)
669 : : {
670 : : /*
671 : : * New block not mentioned in the WAL summary. Should have been an
672 : : * uninitialized block, so just zero-fill it.
673 : : */
1051 rhaas@postgresql.org 674 :UBC 0 : memset(buffer, 0, BLCKSZ);
675 : :
676 : : /* Write out the block, update the checksum if needed. */
944 tomas.vondra@postgre 677 : 0 : write_block(wfd, output_filename, buffer, checksum_ctx);
678 : :
679 : : /* Nothing else to do for zero-filled blocks. */
680 : 0 : continue;
681 : : }
682 : :
683 : : /* Copy the block using the appropriate copy method. */
944 tomas.vondra@postgre 684 [ + - ]:CBC 408 : if (copy_method != COPY_METHOD_COPY_FILE_RANGE)
685 : : {
686 : : /*
687 : : * Read the block from the correct source file, and then write it
688 : : * out, possibly with a checksum update.
689 : : */
690 : 408 : read_block(s, offsetmap[i], buffer);
691 : 408 : write_block(wfd, output_filename, buffer, checksum_ctx);
692 : : }
693 : : else /* use copy_file_range */
694 : : {
695 : : #if defined(HAVE_COPY_FILE_RANGE)
696 : : /* copy_file_range modifies the offset, so use a local copy */
944 tomas.vondra@postgre 697 :UBC 0 : off_t off = offsetmap[i];
698 : 0 : size_t nwritten = 0;
699 : :
700 : : /*
701 : : * Retry until we've written all the bytes (the offset is updated
702 : : * by copy_file_range, and so is the wfd file offset).
703 : : */
704 : : do
705 : : {
706 : : ssize_t wb;
707 : :
708 : 0 : wb = copy_file_range(s->fd, &off, wfd, NULL, BLCKSZ - nwritten, 0);
709 : :
710 [ # # ]: 0 : if (wb < 0)
711 : 0 : pg_fatal("error while copying file range from \"%s\" to \"%s\": %m",
712 : : input_filename, output_filename);
129 peter@eisentraut.org 713 [ # # ]: 0 : else if (wb == 0)
714 : 0 : pg_fatal("unexpected end of file while copying file range from \"%s\" to \"%s\"",
715 : : input_filename, output_filename);
716 : :
944 tomas.vondra@postgre 717 : 0 : nwritten += wb;
718 : :
719 [ # # ]: 0 : } while (BLCKSZ > nwritten);
720 : :
721 : : /*
722 : : * When checksum calculation not needed, we're done, otherwise
723 : : * read the block and pass it to the checksum calculation.
724 : : */
725 [ # # ]: 0 : if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
726 : 0 : continue;
727 : :
728 : 0 : read_block(s, offsetmap[i], buffer);
729 : :
730 [ # # ]: 0 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
731 : 0 : pg_fatal("could not update checksum of file \"%s\"",
732 : : output_filename);
733 : : #else
734 : : pg_fatal("copy_file_range not supported on this platform");
735 : : #endif
736 : : }
737 : : }
738 : :
739 : : /* Debugging output. */
1051 rhaas@postgresql.org 740 [ - + ]:CBC 31 : if (zero_blocks > 0)
741 : : {
1051 rhaas@postgresql.org 742 [ # # ]:UBC 0 : if (dry_run)
743 [ # # ]: 0 : pg_log_debug("would have zero-filled %u blocks", zero_blocks);
744 : : else
745 [ # # ]: 0 : pg_log_debug("zero-filled %u blocks", zero_blocks);
746 : : }
747 : :
748 : : /* Close the output file. */
1051 rhaas@postgresql.org 749 [ + - - + ]:CBC 31 : if (wfd >= 0 && close(wfd) != 0)
867 peter@eisentraut.org 750 :UBC 0 : pg_fatal("could not close file \"%s\": %m", output_filename);
1051 rhaas@postgresql.org 751 :CBC 31 : }
752 : :
753 : : /*
754 : : * Write the block into the file (using the file descriptor), and
755 : : * if needed update the checksum calculation.
756 : : *
757 : : * The buffer is expected to contain BLCKSZ bytes. The filename is
758 : : * provided only for the error message.
759 : : */
760 : : static void
115 peter@eisentraut.org 761 :GNC 408 : write_block(int fd, const char *output_filename,
762 : : const uint8 *buffer, pg_checksum_context *checksum_ctx)
763 : : {
764 : : ssize_t wb;
765 : :
944 tomas.vondra@postgre 766 [ - + ]:CBC 408 : if ((wb = write(fd, buffer, BLCKSZ)) != BLCKSZ)
767 : : {
944 tomas.vondra@postgre 768 [ # # ]:UBC 0 : if (wb < 0)
769 : 0 : pg_fatal("could not write file \"%s\": %m", output_filename);
770 : : else
113 peter@eisentraut.org 771 :UNC 0 : pg_fatal("could not write file \"%s\": wrote %zd of %zu",
772 : : output_filename, wb, (size_t) BLCKSZ);
773 : : }
774 : :
775 : : /* Update the checksum computation. */
944 tomas.vondra@postgre 776 [ - + ]:CBC 408 : if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
944 tomas.vondra@postgre 777 :UBC 0 : pg_fatal("could not update checksum of file \"%s\"",
778 : : output_filename);
944 tomas.vondra@postgre 779 :CBC 408 : }
780 : :
781 : : /*
782 : : * Read a block of data (BLCKSZ bytes) into the buffer.
783 : : */
784 : : static void
115 peter@eisentraut.org 785 :GNC 408 : read_block(const rfile *s, off_t off, uint8 *buffer)
786 : : {
787 : : ssize_t rb;
788 : :
789 : : /* Read the block from the correct source, except if dry-run. */
944 tomas.vondra@postgre 790 :CBC 408 : rb = pg_pread(s->fd, buffer, BLCKSZ, off);
791 [ - + ]: 408 : if (rb != BLCKSZ)
792 : : {
944 tomas.vondra@postgre 793 [ # # ]:UBC 0 : if (rb < 0)
867 peter@eisentraut.org 794 : 0 : pg_fatal("could not read from file \"%s\": %m", s->filename);
795 : : else
113 peter@eisentraut.org 796 :UNC 0 : pg_fatal("could not read from file \"%s\", offset %lld: read %zd of %zu",
797 : : s->filename, (long long) off, rb, (size_t) BLCKSZ);
798 : : }
944 tomas.vondra@postgre 799 :CBC 408 : }
|
