Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_backup_directory.c
4 : *
5 : * A directory format dump is a directory, which contains a "toc.dat" file
6 : * for the TOC, and a separate file for each data entry, named "<oid>.dat".
7 : * Large objects are stored in separate files named "blob_<oid>.dat",
8 : * and there's a plain-text TOC file for each BLOBS TOC entry named
9 : * "blobs_<dumpID>.toc" (or just "blobs.toc" in archive versions before 16).
10 : *
11 : * If compression is used, each data file is individually compressed and the
12 : * ".gz" suffix is added to the filenames. The TOC files are never
13 : * compressed by pg_dump, however they are accepted with the .gz suffix too,
14 : * in case the user has manually compressed them with 'gzip'.
15 : *
16 : * NOTE: This format is identical to the files written in the tar file in
17 : * the 'tar' format, except that we don't write the restore.sql file (TODO),
18 : * and the tar format doesn't support compression. Please keep the formats in
19 : * sync.
20 : *
21 : *
22 : * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
23 : * Portions Copyright (c) 1994, Regents of the University of California
24 : * Portions Copyright (c) 2000, Philip Warner
25 : *
26 : * Rights are granted to use this software in any way so long
27 : * as this notice is not removed.
28 : *
29 : * The author is not responsible for loss or damages that may
30 : * result from its use.
31 : *
32 : * IDENTIFICATION
33 : * src/bin/pg_dump/pg_backup_directory.c
34 : *
35 : *-------------------------------------------------------------------------
36 : */
37 : #include "postgres_fe.h"
38 :
39 : #include <dirent.h>
40 : #include <sys/stat.h>
41 :
42 : #include "common/file_utils.h"
43 : #include "compress_io.h"
44 : #include "parallel.h"
45 : #include "pg_backup_utils.h"
46 :
47 : typedef struct
48 : {
49 : /*
50 : * Our archive location. This is basically what the user specified as his
51 : * backup file but of course here it is a directory.
52 : */
53 : char *directory;
54 :
55 : CompressFileHandle *dataFH; /* currently open data file */
56 : CompressFileHandle *LOsTocFH; /* file handle for blobs_NNN.toc */
57 : ParallelState *pstate; /* for parallel backup / restore */
58 : } lclContext;
59 :
60 : typedef struct
61 : {
62 : char *filename; /* filename excluding the directory (basename) */
63 : } lclTocEntry;
64 :
65 : /* prototypes for private functions */
66 : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
67 : static void _StartData(ArchiveHandle *AH, TocEntry *te);
68 : static void _EndData(ArchiveHandle *AH, TocEntry *te);
69 : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
70 : static int _WriteByte(ArchiveHandle *AH, const int i);
71 : static int _ReadByte(ArchiveHandle *AH);
72 : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
73 : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
74 : static void _CloseArchive(ArchiveHandle *AH);
75 : static void _ReopenArchive(ArchiveHandle *AH);
76 : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
77 :
78 : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
79 : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
80 : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
81 :
82 : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
83 : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
84 : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
85 : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
86 : static void _LoadLOs(ArchiveHandle *AH, TocEntry *te);
87 :
88 : static void _PrepParallelRestore(ArchiveHandle *AH);
89 : static void _Clone(ArchiveHandle *AH);
90 : static void _DeClone(ArchiveHandle *AH);
91 :
92 : static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
93 : static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
94 :
95 : static void setFilePath(ArchiveHandle *AH, char *buf,
96 : const char *relativeFilename);
97 :
98 : /*
99 : * Init routine required by ALL formats. This is a global routine
100 : * and should be declared in pg_backup_archiver.h
101 : *
102 : * Its task is to create any extra archive context (using AH->formatData),
103 : * and to initialize the supported function pointers.
104 : *
105 : * It should also prepare whatever its input source is for reading/writing,
106 : * and in the case of a read mode connection, it should load the Header & TOC.
107 : */
108 : void
109 44 : InitArchiveFmt_Directory(ArchiveHandle *AH)
110 : {
111 : lclContext *ctx;
112 :
113 : /* Assuming static functions, this can be copied for each format. */
114 44 : AH->ArchiveEntryPtr = _ArchiveEntry;
115 44 : AH->StartDataPtr = _StartData;
116 44 : AH->WriteDataPtr = _WriteData;
117 44 : AH->EndDataPtr = _EndData;
118 44 : AH->WriteBytePtr = _WriteByte;
119 44 : AH->ReadBytePtr = _ReadByte;
120 44 : AH->WriteBufPtr = _WriteBuf;
121 44 : AH->ReadBufPtr = _ReadBuf;
122 44 : AH->ClosePtr = _CloseArchive;
123 44 : AH->ReopenPtr = _ReopenArchive;
124 44 : AH->PrintTocDataPtr = _PrintTocData;
125 44 : AH->ReadExtraTocPtr = _ReadExtraToc;
126 44 : AH->WriteExtraTocPtr = _WriteExtraToc;
127 44 : AH->PrintExtraTocPtr = _PrintExtraToc;
128 :
129 44 : AH->StartLOsPtr = _StartLOs;
130 44 : AH->StartLOPtr = _StartLO;
131 44 : AH->EndLOPtr = _EndLO;
132 44 : AH->EndLOsPtr = _EndLOs;
133 :
134 44 : AH->PrepParallelRestorePtr = _PrepParallelRestore;
135 44 : AH->ClonePtr = _Clone;
136 44 : AH->DeClonePtr = _DeClone;
137 :
138 44 : AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
139 44 : AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
140 :
141 : /* Set up our private context */
142 44 : ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
143 44 : AH->formatData = ctx;
144 :
145 44 : ctx->dataFH = NULL;
146 44 : ctx->LOsTocFH = NULL;
147 :
148 : /*
149 : * Now open the TOC file
150 : */
151 :
152 44 : if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
153 0 : pg_fatal("no output directory specified");
154 :
155 44 : ctx->directory = AH->fSpec;
156 :
157 44 : if (AH->mode == archModeWrite)
158 : {
159 : struct stat st;
160 20 : bool is_empty = false;
161 :
162 : /* we accept an empty existing directory */
163 20 : if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
164 : {
165 0 : DIR *dir = opendir(ctx->directory);
166 :
167 0 : if (dir)
168 : {
169 : struct dirent *d;
170 :
171 0 : is_empty = true;
172 0 : while (errno = 0, (d = readdir(dir)))
173 : {
174 0 : if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
175 : {
176 0 : is_empty = false;
177 0 : break;
178 : }
179 : }
180 :
181 0 : if (errno)
182 0 : pg_fatal("could not read directory \"%s\": %m",
183 : ctx->directory);
184 :
185 0 : if (closedir(dir))
186 0 : pg_fatal("could not close directory \"%s\": %m",
187 : ctx->directory);
188 : }
189 : }
190 :
191 20 : if (!is_empty && mkdir(ctx->directory, 0700) < 0)
192 0 : pg_fatal("could not create directory \"%s\": %m",
193 : ctx->directory);
194 : }
195 : else
196 : { /* Read Mode */
197 : char fname[MAXPGPATH];
198 : CompressFileHandle *tocFH;
199 :
200 24 : setFilePath(AH, fname, "toc.dat");
201 :
202 24 : tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
203 24 : if (tocFH == NULL)
204 0 : pg_fatal("could not open input file \"%s\": %m", fname);
205 :
206 24 : ctx->dataFH = tocFH;
207 :
208 : /*
209 : * The TOC of a directory format dump shares the format code of the
210 : * tar format.
211 : */
212 24 : AH->format = archTar;
213 24 : ReadHead(AH);
214 24 : AH->format = archDirectory;
215 24 : ReadToc(AH);
216 :
217 : /* Nothing else in the file, so close it again... */
218 24 : if (!EndCompressFileHandle(tocFH))
219 0 : pg_fatal("could not close TOC file: %m");
220 24 : ctx->dataFH = NULL;
221 : }
222 44 : }
223 :
224 : /*
225 : * Called by the Archiver when the dumper creates a new TOC entry.
226 : *
227 : * We determine the filename for this entry.
228 : */
229 : static void
230 3292 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
231 : {
232 : lclTocEntry *tctx;
233 : char fn[MAXPGPATH];
234 :
235 3292 : tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
236 3292 : if (strcmp(te->desc, "BLOBS") == 0)
237 : {
238 16 : snprintf(fn, MAXPGPATH, "blobs_%d.toc", te->dumpId);
239 16 : tctx->filename = pg_strdup(fn);
240 : }
241 3276 : else if (te->dataDumper)
242 : {
243 298 : snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
244 298 : tctx->filename = pg_strdup(fn);
245 : }
246 : else
247 2978 : tctx->filename = NULL;
248 :
249 3292 : te->formatData = tctx;
250 3292 : }
251 :
252 : /*
253 : * Called by the Archiver to save any extra format-related TOC entry
254 : * data.
255 : *
256 : * Use the Archiver routines to write data - they are non-endian, and
257 : * maintain other important file information.
258 : */
259 : static void
260 3292 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
261 : {
262 3292 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
263 :
264 : /*
265 : * A dumpable object has set tctx->filename, any other object has not.
266 : * (see _ArchiveEntry).
267 : */
268 3292 : if (tctx->filename)
269 314 : WriteStr(AH, tctx->filename);
270 : else
271 2978 : WriteStr(AH, "");
272 3292 : }
273 :
274 : /*
275 : * Called by the Archiver to read any extra format-related TOC data.
276 : *
277 : * Needs to match the order defined in _WriteExtraToc, and should also
278 : * use the Archiver input routines.
279 : */
280 : static void
281 4012 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
282 : {
283 4012 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
284 :
285 4012 : if (tctx == NULL)
286 : {
287 4012 : tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
288 4012 : te->formatData = tctx;
289 : }
290 :
291 4012 : tctx->filename = ReadStr(AH);
292 4012 : if (strlen(tctx->filename) == 0)
293 : {
294 3630 : free(tctx->filename);
295 3630 : tctx->filename = NULL;
296 : }
297 4012 : }
298 :
299 : /*
300 : * Called by the Archiver when restoring an archive to output a comment
301 : * that includes useful information about the TOC entry.
302 : */
303 : static void
304 2968 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
305 : {
306 2968 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
307 :
308 2968 : if (AH->public.verbose && tctx->filename)
309 0 : ahprintf(AH, "-- File: %s\n", tctx->filename);
310 2968 : }
311 :
312 : /*
313 : * Called by the archiver when saving TABLE DATA (not schema). This routine
314 : * should save whatever format-specific information is needed to read
315 : * the archive back.
316 : *
317 : * It is called just prior to the dumper's 'DataDumper' routine being called.
318 : *
319 : * We create the data file for writing.
320 : */
321 : static void
322 298 : _StartData(ArchiveHandle *AH, TocEntry *te)
323 : {
324 298 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
325 298 : lclContext *ctx = (lclContext *) AH->formatData;
326 : char fname[MAXPGPATH];
327 :
328 298 : setFilePath(AH, fname, tctx->filename);
329 :
330 298 : ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
331 :
332 298 : if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
333 0 : pg_fatal("could not open output file \"%s\": %m", fname);
334 298 : }
335 :
336 : /*
337 : * Called by archiver when dumper calls WriteData. This routine is
338 : * called for both LO and table data; it is the responsibility of
339 : * the format to manage each kind of data using StartLO/StartData.
340 : *
341 : * It should only be called from within a DataDumper routine.
342 : *
343 : * We write the data to the open data file.
344 : */
345 : static void
346 49364 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
347 : {
348 49364 : lclContext *ctx = (lclContext *) AH->formatData;
349 49364 : CompressFileHandle *CFH = ctx->dataFH;
350 :
351 49364 : errno = 0;
352 49364 : if (dLen > 0 && !CFH->write_func(data, dLen, CFH))
353 : {
354 : /* if write didn't set errno, assume problem is no disk space */
355 0 : if (errno == 0)
356 0 : errno = ENOSPC;
357 0 : pg_fatal("could not write to output file: %s",
358 : CFH->get_error_func(CFH));
359 : }
360 49364 : }
361 :
362 : /*
363 : * Called by the archiver when a dumper's 'DataDumper' routine has
364 : * finished.
365 : *
366 : * We close the data file.
367 : */
368 : static void
369 298 : _EndData(ArchiveHandle *AH, TocEntry *te)
370 : {
371 298 : lclContext *ctx = (lclContext *) AH->formatData;
372 :
373 : /* Close the file */
374 298 : if (!EndCompressFileHandle(ctx->dataFH))
375 0 : pg_fatal("could not close data file: %m");
376 :
377 298 : ctx->dataFH = NULL;
378 298 : }
379 :
380 : /*
381 : * Print data for a given file (can be a LO as well)
382 : */
383 : static void
384 316 : _PrintFileData(ArchiveHandle *AH, char *filename)
385 : {
386 316 : size_t cnt = 0;
387 : char *buf;
388 : size_t buflen;
389 : CompressFileHandle *CFH;
390 :
391 316 : if (!filename)
392 0 : return;
393 :
394 316 : CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
395 316 : if (!CFH)
396 0 : pg_fatal("could not open input file \"%s\": %m", filename);
397 :
398 316 : buflen = DEFAULT_IO_BUFFER_SIZE;
399 316 : buf = pg_malloc(buflen);
400 :
401 710 : while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
402 : {
403 394 : ahwrite(buf, 1, cnt, AH);
404 : }
405 :
406 316 : free(buf);
407 316 : if (!EndCompressFileHandle(CFH))
408 0 : pg_fatal("could not close data file \"%s\": %m", filename);
409 : }
410 :
411 : /*
412 : * Print data for a given TOC entry
413 : */
414 : static void
415 316 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
416 : {
417 316 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
418 :
419 316 : if (!tctx->filename)
420 0 : return;
421 :
422 316 : if (strcmp(te->desc, "BLOBS") == 0)
423 16 : _LoadLOs(AH, te);
424 : else
425 : {
426 : char fname[MAXPGPATH];
427 :
428 300 : setFilePath(AH, fname, tctx->filename);
429 300 : _PrintFileData(AH, fname);
430 : }
431 : }
432 :
433 : static void
434 16 : _LoadLOs(ArchiveHandle *AH, TocEntry *te)
435 : {
436 : Oid oid;
437 16 : lclContext *ctx = (lclContext *) AH->formatData;
438 16 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
439 : CompressFileHandle *CFH;
440 : char tocfname[MAXPGPATH];
441 : char line[MAXPGPATH];
442 :
443 16 : StartRestoreLOs(AH);
444 :
445 : /*
446 : * Note: before archive v16, there was always only one BLOBS TOC entry,
447 : * now there can be multiple. We don't need to worry what version we are
448 : * reading though, because tctx->filename should be correct either way.
449 : */
450 16 : setFilePath(AH, tocfname, tctx->filename);
451 :
452 16 : CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
453 :
454 16 : if (ctx->LOsTocFH == NULL)
455 0 : pg_fatal("could not open large object TOC file \"%s\" for input: %m",
456 : tocfname);
457 :
458 : /* Read the LOs TOC file line-by-line, and process each LO */
459 32 : while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
460 : {
461 : char lofname[MAXPGPATH + 1];
462 : char path[MAXPGPATH];
463 :
464 : /* Can't overflow because line and lofname are the same length */
465 16 : if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, lofname) != 2)
466 0 : pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
467 : tocfname, line);
468 :
469 16 : StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
470 16 : snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, lofname);
471 16 : _PrintFileData(AH, path);
472 16 : EndRestoreLO(AH, oid);
473 : }
474 16 : if (!CFH->eof_func(CFH))
475 0 : pg_fatal("error reading large object TOC file \"%s\"",
476 : tocfname);
477 :
478 16 : if (!EndCompressFileHandle(ctx->LOsTocFH))
479 0 : pg_fatal("could not close large object TOC file \"%s\": %m",
480 : tocfname);
481 :
482 16 : ctx->LOsTocFH = NULL;
483 :
484 16 : EndRestoreLOs(AH);
485 16 : }
486 :
487 :
488 : /*
489 : * Write a byte of data to the archive.
490 : * Called by the archiver to do integer & byte output to the archive.
491 : * These routines are only used to read & write the headers & TOC.
492 : */
493 : static int
494 326530 : _WriteByte(ArchiveHandle *AH, const int i)
495 : {
496 326530 : unsigned char c = (unsigned char) i;
497 326530 : lclContext *ctx = (lclContext *) AH->formatData;
498 326530 : CompressFileHandle *CFH = ctx->dataFH;
499 :
500 326530 : errno = 0;
501 326530 : if (!CFH->write_func(&c, 1, CFH))
502 : {
503 : /* if write didn't set errno, assume problem is no disk space */
504 0 : if (errno == 0)
505 0 : errno = ENOSPC;
506 0 : pg_fatal("could not write to output file: %s",
507 : CFH->get_error_func(CFH));
508 : }
509 :
510 326530 : return 1;
511 : }
512 :
513 : /*
514 : * Read a byte of data from the archive.
515 : * Called by the archiver to read bytes & integers from the archive.
516 : * These routines are only used to read & write headers & TOC.
517 : * EOF should be treated as a fatal error.
518 : */
519 : static int
520 397768 : _ReadByte(ArchiveHandle *AH)
521 : {
522 397768 : lclContext *ctx = (lclContext *) AH->formatData;
523 397768 : CompressFileHandle *CFH = ctx->dataFH;
524 :
525 397768 : return CFH->getc_func(CFH);
526 : }
527 :
528 : /*
529 : * Write a buffer of data to the archive.
530 : * Called by the archiver to write a block of bytes to the TOC or a data file.
531 : */
532 : static void
533 36624 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
534 : {
535 36624 : lclContext *ctx = (lclContext *) AH->formatData;
536 36624 : CompressFileHandle *CFH = ctx->dataFH;
537 :
538 36624 : errno = 0;
539 36624 : if (!CFH->write_func(buf, len, CFH))
540 : {
541 : /* if write didn't set errno, assume problem is no disk space */
542 0 : if (errno == 0)
543 0 : errno = ENOSPC;
544 0 : pg_fatal("could not write to output file: %s",
545 : CFH->get_error_func(CFH));
546 : }
547 36624 : }
548 :
549 : /*
550 : * Read a block of bytes from the archive.
551 : *
552 : * Called by the archiver to read a block of bytes from the archive
553 : */
554 : static void
555 44610 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
556 : {
557 44610 : lclContext *ctx = (lclContext *) AH->formatData;
558 44610 : CompressFileHandle *CFH = ctx->dataFH;
559 :
560 : /*
561 : * If there was an I/O error, we already exited in readF(), so here we
562 : * exit on short reads.
563 : */
564 44610 : if (!CFH->read_func(buf, len, NULL, CFH))
565 0 : pg_fatal("could not read from input file: end of file");
566 44610 : }
567 :
568 : /*
569 : * Close the archive.
570 : *
571 : * When writing the archive, this is the routine that actually starts
572 : * the process of saving it to files. No data should be written prior
573 : * to this point, since the user could sort the TOC after creating it.
574 : *
575 : * If an archive is to be written, this routine must call:
576 : * WriteHead to save the archive header
577 : * WriteToc to save the TOC entries
578 : * WriteDataChunks to save all data & LOs.
579 : */
580 : static void
581 44 : _CloseArchive(ArchiveHandle *AH)
582 : {
583 44 : lclContext *ctx = (lclContext *) AH->formatData;
584 :
585 44 : if (AH->mode == archModeWrite)
586 : {
587 : CompressFileHandle *tocFH;
588 20 : pg_compress_specification compression_spec = {0};
589 : char fname[MAXPGPATH];
590 :
591 20 : setFilePath(AH, fname, "toc.dat");
592 :
593 : /* this will actually fork the processes for a parallel backup */
594 20 : ctx->pstate = ParallelBackupStart(AH);
595 :
596 : /* The TOC is always created uncompressed */
597 20 : compression_spec.algorithm = PG_COMPRESSION_NONE;
598 20 : tocFH = InitCompressFileHandle(compression_spec);
599 20 : if (!tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
600 0 : pg_fatal("could not open output file \"%s\": %m", fname);
601 20 : ctx->dataFH = tocFH;
602 :
603 : /*
604 : * Write 'tar' in the format field of the toc.dat file. The directory
605 : * is compatible with 'tar', so there's no point having a different
606 : * format code for it.
607 : */
608 20 : AH->format = archTar;
609 20 : WriteHead(AH);
610 20 : AH->format = archDirectory;
611 20 : WriteToc(AH);
612 20 : if (!EndCompressFileHandle(tocFH))
613 0 : pg_fatal("could not close TOC file: %m");
614 20 : WriteDataChunks(AH, ctx->pstate);
615 :
616 20 : ParallelBackupEnd(AH, ctx->pstate);
617 :
618 : /*
619 : * In directory mode, there is no need to sync all the entries
620 : * individually. Just recurse once through all the files generated.
621 : */
622 20 : if (AH->dosync)
623 8 : sync_dir_recurse(ctx->directory, AH->sync_method);
624 : }
625 44 : AH->FH = NULL;
626 44 : }
627 :
628 : /*
629 : * Reopen the archive's file handle.
630 : */
631 : static void
632 28 : _ReopenArchive(ArchiveHandle *AH)
633 : {
634 : /*
635 : * Our TOC is in memory, our data files are opened by each child anyway as
636 : * they are separate. We support reopening the archive by just doing
637 : * nothing.
638 : */
639 28 : }
640 :
641 : /*
642 : * LO support
643 : */
644 :
645 : /*
646 : * Called by the archiver when starting to save BLOB DATA (not schema).
647 : * It is called just prior to the dumper's DataDumper routine.
648 : *
649 : * We open the large object TOC file here, so that we can append a line to
650 : * it for each LO.
651 : */
652 : static void
653 16 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
654 : {
655 16 : lclContext *ctx = (lclContext *) AH->formatData;
656 16 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
657 16 : pg_compress_specification compression_spec = {0};
658 : char fname[MAXPGPATH];
659 :
660 16 : setFilePath(AH, fname, tctx->filename);
661 :
662 : /* The LO TOC file is never compressed */
663 16 : compression_spec.algorithm = PG_COMPRESSION_NONE;
664 16 : ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
665 16 : if (!ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
666 0 : pg_fatal("could not open output file \"%s\": %m", fname);
667 16 : }
668 :
669 : /*
670 : * Called by the archiver when we're about to start dumping a LO.
671 : *
672 : * We create a file to write the LO to.
673 : */
674 : static void
675 16 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
676 : {
677 16 : lclContext *ctx = (lclContext *) AH->formatData;
678 : char fname[MAXPGPATH];
679 :
680 16 : snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
681 :
682 16 : ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
683 16 : if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
684 0 : pg_fatal("could not open output file \"%s\": %m", fname);
685 16 : }
686 :
687 : /*
688 : * Called by the archiver when the dumper is finished writing a LO.
689 : *
690 : * We close the LO file and write an entry to the LO TOC file for it.
691 : */
692 : static void
693 16 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
694 : {
695 16 : lclContext *ctx = (lclContext *) AH->formatData;
696 16 : CompressFileHandle *CFH = ctx->LOsTocFH;
697 : char buf[50];
698 : int len;
699 :
700 : /* Close the BLOB data file itself */
701 16 : if (!EndCompressFileHandle(ctx->dataFH))
702 0 : pg_fatal("could not close LO data file: %m");
703 16 : ctx->dataFH = NULL;
704 :
705 : /* register the LO in blobs_NNN.toc */
706 16 : len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
707 16 : if (!CFH->write_func(buf, len, CFH))
708 : {
709 : /* if write didn't set errno, assume problem is no disk space */
710 0 : if (errno == 0)
711 0 : errno = ENOSPC;
712 0 : pg_fatal("could not write to LOs TOC file: %s",
713 : CFH->get_error_func(CFH));
714 : }
715 16 : }
716 :
717 : /*
718 : * Called by the archiver when finishing saving BLOB DATA.
719 : *
720 : * We close the LOs TOC file.
721 : */
722 : static void
723 16 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
724 : {
725 16 : lclContext *ctx = (lclContext *) AH->formatData;
726 :
727 16 : if (!EndCompressFileHandle(ctx->LOsTocFH))
728 0 : pg_fatal("could not close LOs TOC file: %m");
729 16 : ctx->LOsTocFH = NULL;
730 16 : }
731 :
732 : /*
733 : * Gets a relative file name and prepends the output directory, writing the
734 : * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
735 : * big. Can't use a static char[MAXPGPATH] inside the function because we run
736 : * multithreaded on Windows.
737 : */
738 : static void
739 706 : setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
740 : {
741 706 : lclContext *ctx = (lclContext *) AH->formatData;
742 : char *dname;
743 :
744 706 : dname = ctx->directory;
745 :
746 706 : if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
747 0 : pg_fatal("file name too long: \"%s\"", dname);
748 :
749 706 : strcpy(buf, dname);
750 706 : strcat(buf, "/");
751 706 : strcat(buf, relativeFilename);
752 706 : }
753 :
754 : /*
755 : * Prepare for parallel restore.
756 : *
757 : * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
758 : * TOC entries' dataLength fields with appropriate values to guide the
759 : * ordering of restore jobs. The source of said data is format-dependent,
760 : * as is the exact meaning of the values.
761 : *
762 : * A format module might also choose to do other setup here.
763 : */
764 : static void
765 8 : _PrepParallelRestore(ArchiveHandle *AH)
766 : {
767 : TocEntry *te;
768 :
769 276 : for (te = AH->toc->next; te != AH->toc; te = te->next)
770 : {
771 268 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
772 : char fname[MAXPGPATH];
773 : struct stat st;
774 :
775 : /*
776 : * A dumpable object has set tctx->filename, any other object has not.
777 : * (see _ArchiveEntry).
778 : */
779 268 : if (tctx->filename == NULL)
780 236 : continue;
781 :
782 : /* We may ignore items not due to be restored */
783 32 : if ((te->reqs & (REQ_DATA | REQ_STATS)) == 0)
784 0 : continue;
785 :
786 : /*
787 : * Stat the file and, if successful, put its size in dataLength. When
788 : * using compression, the physical file size might not be a very good
789 : * guide to the amount of work involved in restoring the file, but we
790 : * only need an approximate indicator of that.
791 : */
792 32 : setFilePath(AH, fname, tctx->filename);
793 :
794 32 : if (stat(fname, &st) == 0)
795 0 : te->dataLength = st.st_size;
796 32 : else if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
797 : {
798 32 : if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
799 32 : strlcat(fname, ".gz", sizeof(fname));
800 0 : else if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
801 0 : strlcat(fname, ".lz4", sizeof(fname));
802 0 : else if (AH->compression_spec.algorithm == PG_COMPRESSION_ZSTD)
803 0 : strlcat(fname, ".zst", sizeof(fname));
804 :
805 32 : if (stat(fname, &st) == 0)
806 32 : te->dataLength = st.st_size;
807 : }
808 :
809 : /*
810 : * If this is a BLOBS entry, what we stat'd was blobs_NNN.toc, which
811 : * most likely is a lot smaller than the actual blob data. We don't
812 : * have a cheap way to estimate how much smaller, but fortunately it
813 : * doesn't matter too much as long as we get the LOs processed
814 : * reasonably early. Arbitrarily scale up by a factor of 1K.
815 : */
816 32 : if (strcmp(te->desc, "BLOBS") == 0)
817 0 : te->dataLength *= 1024;
818 : }
819 8 : }
820 :
821 : /*
822 : * Clone format-specific fields during parallel restoration.
823 : */
824 : static void
825 52 : _Clone(ArchiveHandle *AH)
826 : {
827 52 : lclContext *ctx = (lclContext *) AH->formatData;
828 :
829 52 : AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
830 52 : memcpy(AH->formatData, ctx, sizeof(lclContext));
831 52 : ctx = (lclContext *) AH->formatData;
832 :
833 : /*
834 : * TOC-entry-local state isn't an issue because any one TOC entry is
835 : * touched by just one worker child.
836 : */
837 :
838 : /*
839 : * We also don't copy the ParallelState pointer (pstate), only the leader
840 : * process ever writes to it.
841 : */
842 52 : }
843 :
844 : static void
845 52 : _DeClone(ArchiveHandle *AH)
846 : {
847 52 : lclContext *ctx = (lclContext *) AH->formatData;
848 :
849 52 : free(ctx);
850 52 : }
851 :
852 : /*
853 : * This function is executed in the child of a parallel backup for a
854 : * directory-format archive and dumps the actual data for one TOC entry.
855 : */
856 : static int
857 242 : _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
858 : {
859 : /*
860 : * This function returns void. We either fail and die horribly or
861 : * succeed... A failure will be detected by the parent when the child dies
862 : * unexpectedly.
863 : */
864 242 : WriteDataChunksForTocEntry(AH, te);
865 :
866 242 : return 0;
867 : }
868 :
869 : /*
870 : * This function is executed in the child of a parallel restore from a
871 : * directory-format archive and restores the actual data for one TOC entry.
872 : */
873 : static int
874 168 : _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
875 : {
876 168 : return parallel_restore(AH, te);
877 : }
|
