Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_backup_directory.c
4 : *
5 : * A directory format dump is a directory, which contains a "toc.dat" file
6 : * for the TOC, and a separate file for each data entry, named "<oid>.dat".
7 : * Large objects are stored in separate files named "blob_<oid>.dat",
8 : * and there's a plain-text TOC file for each BLOBS TOC entry named
9 : * "blobs_<dumpID>.toc" (or just "blobs.toc" in archive versions before 16).
10 : *
11 : * If compression is used, each data file is individually compressed and the
12 : * ".gz" suffix is added to the filenames. The TOC files are never
13 : * compressed by pg_dump, however they are accepted with the .gz suffix too,
14 : * in case the user has manually compressed them with 'gzip'.
15 : *
16 : * NOTE: This format is identical to the files written in the tar file in
17 : * the 'tar' format, except that we don't write the restore.sql file (TODO),
18 : * and the tar format doesn't support compression. Please keep the formats in
19 : * sync.
20 : *
21 : *
22 : * Portions Copyright (c) 1996-2025, PostgreSQL Global Development Group
23 : * Portions Copyright (c) 1994, Regents of the University of California
24 : * Portions Copyright (c) 2000, Philip Warner
25 : *
26 : * Rights are granted to use this software in any way so long
27 : * as this notice is not removed.
28 : *
29 : * The author is not responsible for loss or damages that may
30 : * result from its use.
31 : *
32 : * IDENTIFICATION
33 : * src/bin/pg_dump/pg_backup_directory.c
34 : *
35 : *-------------------------------------------------------------------------
36 : */
37 : #include "postgres_fe.h"
38 :
39 : #include <dirent.h>
40 : #include <sys/stat.h>
41 :
42 : #include "common/file_utils.h"
43 : #include "compress_io.h"
44 : #include "dumputils.h"
45 : #include "parallel.h"
46 : #include "pg_backup_utils.h"
47 :
48 : typedef struct
49 : {
50 : /*
51 : * Our archive location. This is basically what the user specified as his
52 : * backup file but of course here it is a directory.
53 : */
54 : char *directory;
55 :
56 : CompressFileHandle *dataFH; /* currently open data file */
57 : CompressFileHandle *LOsTocFH; /* file handle for blobs_NNN.toc */
58 : ParallelState *pstate; /* for parallel backup / restore */
59 : } lclContext;
60 :
61 : typedef struct
62 : {
63 : char *filename; /* filename excluding the directory (basename) */
64 : } lclTocEntry;
65 :
66 : /* prototypes for private functions */
67 : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
68 : static void _StartData(ArchiveHandle *AH, TocEntry *te);
69 : static void _EndData(ArchiveHandle *AH, TocEntry *te);
70 : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
71 : static int _WriteByte(ArchiveHandle *AH, const int i);
72 : static int _ReadByte(ArchiveHandle *AH);
73 : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
74 : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
75 : static void _CloseArchive(ArchiveHandle *AH);
76 : static void _ReopenArchive(ArchiveHandle *AH);
77 : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
78 :
79 : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
80 : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
81 : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
82 :
83 : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
84 : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
85 : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
86 : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
87 : static void _LoadLOs(ArchiveHandle *AH, TocEntry *te);
88 :
89 : static void _PrepParallelRestore(ArchiveHandle *AH);
90 : static void _Clone(ArchiveHandle *AH);
91 : static void _DeClone(ArchiveHandle *AH);
92 :
93 : static int _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
94 : static int _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
95 :
96 : static void setFilePath(ArchiveHandle *AH, char *buf,
97 : const char *relativeFilename);
98 :
99 : /*
100 : * Init routine required by ALL formats. This is a global routine
101 : * and should be declared in pg_backup_archiver.h
102 : *
103 : * Its task is to create any extra archive context (using AH->formatData),
104 : * and to initialize the supported function pointers.
105 : *
106 : * It should also prepare whatever its input source is for reading/writing,
107 : * and in the case of a read mode connection, it should load the Header & TOC.
108 : */
109 : void
110 198 : InitArchiveFmt_Directory(ArchiveHandle *AH)
111 : {
112 : lclContext *ctx;
113 :
114 : /* Assuming static functions, this can be copied for each format. */
115 198 : AH->ArchiveEntryPtr = _ArchiveEntry;
116 198 : AH->StartDataPtr = _StartData;
117 198 : AH->WriteDataPtr = _WriteData;
118 198 : AH->EndDataPtr = _EndData;
119 198 : AH->WriteBytePtr = _WriteByte;
120 198 : AH->ReadBytePtr = _ReadByte;
121 198 : AH->WriteBufPtr = _WriteBuf;
122 198 : AH->ReadBufPtr = _ReadBuf;
123 198 : AH->ClosePtr = _CloseArchive;
124 198 : AH->ReopenPtr = _ReopenArchive;
125 198 : AH->PrintTocDataPtr = _PrintTocData;
126 198 : AH->ReadExtraTocPtr = _ReadExtraToc;
127 198 : AH->WriteExtraTocPtr = _WriteExtraToc;
128 198 : AH->PrintExtraTocPtr = _PrintExtraToc;
129 :
130 198 : AH->StartLOsPtr = _StartLOs;
131 198 : AH->StartLOPtr = _StartLO;
132 198 : AH->EndLOPtr = _EndLO;
133 198 : AH->EndLOsPtr = _EndLOs;
134 :
135 198 : AH->PrepParallelRestorePtr = _PrepParallelRestore;
136 198 : AH->ClonePtr = _Clone;
137 198 : AH->DeClonePtr = _DeClone;
138 :
139 198 : AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
140 198 : AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
141 :
142 : /* Set up our private context */
143 198 : ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
144 198 : AH->formatData = ctx;
145 :
146 198 : ctx->dataFH = NULL;
147 198 : ctx->LOsTocFH = NULL;
148 :
149 : /*
150 : * Now open the TOC file
151 : */
152 :
153 198 : if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
154 0 : pg_fatal("no output directory specified");
155 :
156 198 : ctx->directory = AH->fSpec;
157 :
158 198 : if (AH->mode == archModeWrite)
159 : {
160 : /* we accept an empty existing directory */
161 98 : create_or_open_dir(ctx->directory);
162 : }
163 : else
164 : { /* Read Mode */
165 : char fname[MAXPGPATH];
166 : CompressFileHandle *tocFH;
167 :
168 100 : setFilePath(AH, fname, "toc.dat");
169 :
170 100 : tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
171 100 : if (tocFH == NULL)
172 0 : pg_fatal("could not open input file \"%s\": %m", fname);
173 :
174 100 : ctx->dataFH = tocFH;
175 :
176 : /*
177 : * The TOC of a directory format dump shares the format code of the
178 : * tar format.
179 : */
180 100 : AH->format = archTar;
181 100 : ReadHead(AH);
182 100 : AH->format = archDirectory;
183 100 : ReadToc(AH);
184 :
185 : /* Nothing else in the file, so close it again... */
186 100 : if (!EndCompressFileHandle(tocFH))
187 0 : pg_fatal("could not close TOC file: %m");
188 100 : ctx->dataFH = NULL;
189 : }
190 198 : }
191 :
192 : /*
193 : * Called by the Archiver when the dumper creates a new TOC entry.
194 : *
195 : * We determine the filename for this entry.
196 : */
197 : static void
198 9516 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
199 : {
200 : lclTocEntry *tctx;
201 : char fn[MAXPGPATH];
202 :
203 9516 : tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
204 9516 : if (strcmp(te->desc, "BLOBS") == 0)
205 : {
206 18 : snprintf(fn, MAXPGPATH, "blobs_%d.toc", te->dumpId);
207 18 : tctx->filename = pg_strdup(fn);
208 : }
209 9498 : else if (te->dataDumper)
210 : {
211 1744 : snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
212 1744 : tctx->filename = pg_strdup(fn);
213 : }
214 : else
215 7754 : tctx->filename = NULL;
216 :
217 9516 : te->formatData = tctx;
218 9516 : }
219 :
220 : /*
221 : * Called by the Archiver to save any extra format-related TOC entry
222 : * data.
223 : *
224 : * Use the Archiver routines to write data - they are non-endian, and
225 : * maintain other important file information.
226 : */
227 : static void
228 9516 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
229 : {
230 9516 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
231 :
232 : /*
233 : * A dumpable object has set tctx->filename, any other object has not.
234 : * (see _ArchiveEntry).
235 : */
236 9516 : if (tctx->filename)
237 1762 : WriteStr(AH, tctx->filename);
238 : else
239 7754 : WriteStr(AH, "");
240 9516 : }
241 :
242 : /*
243 : * Called by the Archiver to read any extra format-related TOC data.
244 : *
245 : * Needs to match the order defined in _WriteExtraToc, and should also
246 : * use the Archiver input routines.
247 : */
248 : static void
249 10256 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
250 : {
251 10256 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
252 :
253 10256 : if (tctx == NULL)
254 : {
255 10256 : tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
256 10256 : te->formatData = tctx;
257 : }
258 :
259 10256 : tctx->filename = ReadStr(AH);
260 10256 : if (strlen(tctx->filename) == 0)
261 : {
262 8420 : free(tctx->filename);
263 8420 : tctx->filename = NULL;
264 : }
265 10256 : }
266 :
267 : /*
268 : * Called by the Archiver when restoring an archive to output a comment
269 : * that includes useful information about the TOC entry.
270 : */
271 : static void
272 3584 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
273 : {
274 3584 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
275 :
276 3584 : if (AH->public.verbose && tctx->filename)
277 0 : ahprintf(AH, "-- File: %s\n", tctx->filename);
278 3584 : }
279 :
280 : /*
281 : * Called by the archiver when saving TABLE DATA (not schema). This routine
282 : * should save whatever format-specific information is needed to read
283 : * the archive back.
284 : *
285 : * It is called just prior to the dumper's 'DataDumper' routine being called.
286 : *
287 : * We create the data file for writing.
288 : */
289 : static void
290 1744 : _StartData(ArchiveHandle *AH, TocEntry *te)
291 : {
292 1744 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
293 1744 : lclContext *ctx = (lclContext *) AH->formatData;
294 : char fname[MAXPGPATH];
295 :
296 1744 : setFilePath(AH, fname, tctx->filename);
297 :
298 1744 : ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
299 :
300 1744 : if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
301 0 : pg_fatal("could not open output file \"%s\": %m", fname);
302 1744 : }
303 :
304 : /*
305 : * Called by archiver when dumper calls WriteData. This routine is
306 : * called for both LO and table data; it is the responsibility of
307 : * the format to manage each kind of data using StartLO/StartData.
308 : *
309 : * It should only be called from within a DataDumper routine.
310 : *
311 : * We write the data to the open data file.
312 : */
313 : static void
314 888442 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
315 : {
316 888442 : lclContext *ctx = (lclContext *) AH->formatData;
317 888442 : CompressFileHandle *CFH = ctx->dataFH;
318 :
319 888442 : errno = 0;
320 888442 : if (dLen > 0 && !CFH->write_func(data, dLen, CFH))
321 : {
322 : /* if write didn't set errno, assume problem is no disk space */
323 0 : if (errno == 0)
324 0 : errno = ENOSPC;
325 0 : pg_fatal("could not write to output file: %s",
326 : CFH->get_error_func(CFH));
327 : }
328 888442 : }
329 :
330 : /*
331 : * Called by the archiver when a dumper's 'DataDumper' routine has
332 : * finished.
333 : *
334 : * We close the data file.
335 : */
336 : static void
337 1744 : _EndData(ArchiveHandle *AH, TocEntry *te)
338 : {
339 1744 : lclContext *ctx = (lclContext *) AH->formatData;
340 :
341 : /* Close the file */
342 1744 : if (!EndCompressFileHandle(ctx->dataFH))
343 0 : pg_fatal("could not close data file: %m");
344 :
345 1744 : ctx->dataFH = NULL;
346 1744 : }
347 :
348 : /*
349 : * Print data for a given file (can be a LO as well)
350 : */
351 : static void
352 1764 : _PrintFileData(ArchiveHandle *AH, char *filename)
353 : {
354 1764 : size_t cnt = 0;
355 : char *buf;
356 : size_t buflen;
357 : CompressFileHandle *CFH;
358 :
359 1764 : if (!filename)
360 0 : return;
361 :
362 1764 : CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
363 1764 : if (!CFH)
364 0 : pg_fatal("could not open input file \"%s\": %m", filename);
365 :
366 1764 : buflen = DEFAULT_IO_BUFFER_SIZE;
367 1764 : buf = pg_malloc(buflen);
368 :
369 9436 : while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
370 : {
371 7672 : ahwrite(buf, 1, cnt, AH);
372 : }
373 :
374 1764 : free(buf);
375 1764 : if (!EndCompressFileHandle(CFH))
376 0 : pg_fatal("could not close data file \"%s\": %m", filename);
377 : }
378 :
379 : /*
380 : * Print data for a given TOC entry
381 : */
382 : static void
383 1760 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
384 : {
385 1760 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
386 :
387 1760 : if (!tctx->filename)
388 0 : return;
389 :
390 1760 : if (strcmp(te->desc, "BLOBS") == 0)
391 18 : _LoadLOs(AH, te);
392 : else
393 : {
394 : char fname[MAXPGPATH];
395 :
396 1742 : setFilePath(AH, fname, tctx->filename);
397 1742 : _PrintFileData(AH, fname);
398 : }
399 : }
400 :
401 : static void
402 18 : _LoadLOs(ArchiveHandle *AH, TocEntry *te)
403 : {
404 : Oid oid;
405 18 : lclContext *ctx = (lclContext *) AH->formatData;
406 18 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
407 : CompressFileHandle *CFH;
408 : char tocfname[MAXPGPATH];
409 : char line[MAXPGPATH];
410 :
411 18 : StartRestoreLOs(AH);
412 :
413 : /*
414 : * Note: before archive v16, there was always only one BLOBS TOC entry,
415 : * now there can be multiple. Furthermore, although the actual filename
416 : * was always "blobs.toc" before v16, the value of tctx->filename did not
417 : * match that before commit 548e50976 fixed it. For simplicity we assume
418 : * it must be "blobs.toc" in all archives before v16.
419 : */
420 18 : if (AH->version < K_VERS_1_16)
421 0 : setFilePath(AH, tocfname, "blobs.toc");
422 : else
423 18 : setFilePath(AH, tocfname, tctx->filename);
424 :
425 18 : CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
426 :
427 18 : if (ctx->LOsTocFH == NULL)
428 0 : pg_fatal("could not open large object TOC file \"%s\" for input: %m",
429 : tocfname);
430 :
431 : /* Read the LOs TOC file line-by-line, and process each LO */
432 40 : while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
433 : {
434 : char lofname[MAXPGPATH + 1];
435 : char path[MAXPGPATH];
436 :
437 : /* Can't overflow because line and lofname are the same length */
438 22 : if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, lofname) != 2)
439 0 : pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
440 : tocfname, line);
441 :
442 22 : StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
443 22 : snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, lofname);
444 22 : _PrintFileData(AH, path);
445 22 : EndRestoreLO(AH, oid);
446 : }
447 18 : if (!CFH->eof_func(CFH))
448 0 : pg_fatal("error reading large object TOC file \"%s\"",
449 : tocfname);
450 :
451 18 : if (!EndCompressFileHandle(ctx->LOsTocFH))
452 0 : pg_fatal("could not close large object TOC file \"%s\": %m",
453 : tocfname);
454 :
455 18 : ctx->LOsTocFH = NULL;
456 :
457 18 : EndRestoreLOs(AH);
458 18 : }
459 :
460 :
461 : /*
462 : * Write a byte of data to the archive.
463 : * Called by the archiver to do integer & byte output to the archive.
464 : * These routines are only used to read & write the headers & TOC.
465 : */
466 : static int
467 926876 : _WriteByte(ArchiveHandle *AH, const int i)
468 : {
469 926876 : unsigned char c = (unsigned char) i;
470 926876 : lclContext *ctx = (lclContext *) AH->formatData;
471 926876 : CompressFileHandle *CFH = ctx->dataFH;
472 :
473 926876 : errno = 0;
474 926876 : if (!CFH->write_func(&c, 1, CFH))
475 : {
476 : /* if write didn't set errno, assume problem is no disk space */
477 0 : if (errno == 0)
478 0 : errno = ENOSPC;
479 0 : pg_fatal("could not write to output file: %s",
480 : CFH->get_error_func(CFH));
481 : }
482 :
483 926876 : return 1;
484 : }
485 :
486 : /*
487 : * Read a byte of data from the archive.
488 : * Called by the archiver to read bytes & integers from the archive.
489 : * These routines are only used to read & write headers & TOC.
490 : * EOF should be treated as a fatal error.
491 : */
492 : static int
493 1000020 : _ReadByte(ArchiveHandle *AH)
494 : {
495 1000020 : lclContext *ctx = (lclContext *) AH->formatData;
496 1000020 : CompressFileHandle *CFH = ctx->dataFH;
497 :
498 1000020 : return CFH->getc_func(CFH);
499 : }
500 :
501 : /*
502 : * Write a buffer of data to the archive.
503 : * Called by the archiver to write a block of bytes to the TOC or a data file.
504 : */
505 : static void
506 106774 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
507 : {
508 106774 : lclContext *ctx = (lclContext *) AH->formatData;
509 106774 : CompressFileHandle *CFH = ctx->dataFH;
510 :
511 106774 : errno = 0;
512 106774 : if (!CFH->write_func(buf, len, CFH))
513 : {
514 : /* if write didn't set errno, assume problem is no disk space */
515 0 : if (errno == 0)
516 0 : errno = ENOSPC;
517 0 : pg_fatal("could not write to output file: %s",
518 : CFH->get_error_func(CFH));
519 : }
520 106774 : }
521 :
522 : /*
523 : * Read a block of bytes from the archive.
524 : *
525 : * Called by the archiver to read a block of bytes from the archive
526 : */
527 : static void
528 115000 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
529 : {
530 115000 : lclContext *ctx = (lclContext *) AH->formatData;
531 115000 : CompressFileHandle *CFH = ctx->dataFH;
532 :
533 : /*
534 : * If there was an I/O error, we already exited in readF(), so here we
535 : * exit on short reads.
536 : */
537 115000 : if (!CFH->read_func(buf, len, NULL, CFH))
538 0 : pg_fatal("could not read from input file: end of file");
539 115000 : }
540 :
541 : /*
542 : * Close the archive.
543 : *
544 : * When writing the archive, this is the routine that actually starts
545 : * the process of saving it to files. No data should be written prior
546 : * to this point, since the user could sort the TOC after creating it.
547 : *
548 : * If an archive is to be written, this routine must call:
549 : * WriteHead to save the archive header
550 : * WriteToc to save the TOC entries
551 : * WriteDataChunks to save all data & LOs.
552 : */
553 : static void
554 198 : _CloseArchive(ArchiveHandle *AH)
555 : {
556 198 : lclContext *ctx = (lclContext *) AH->formatData;
557 :
558 198 : if (AH->mode == archModeWrite)
559 : {
560 : CompressFileHandle *tocFH;
561 98 : pg_compress_specification compression_spec = {0};
562 : char fname[MAXPGPATH];
563 :
564 98 : setFilePath(AH, fname, "toc.dat");
565 :
566 : /* this will actually fork the processes for a parallel backup */
567 98 : ctx->pstate = ParallelBackupStart(AH);
568 :
569 : /* The TOC is always created uncompressed */
570 98 : compression_spec.algorithm = PG_COMPRESSION_NONE;
571 98 : tocFH = InitCompressFileHandle(compression_spec);
572 98 : if (!tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
573 0 : pg_fatal("could not open output file \"%s\": %m", fname);
574 98 : ctx->dataFH = tocFH;
575 :
576 : /*
577 : * Write 'tar' in the format field of the toc.dat file. The directory
578 : * is compatible with 'tar', so there's no point having a different
579 : * format code for it.
580 : */
581 98 : AH->format = archTar;
582 98 : WriteHead(AH);
583 98 : AH->format = archDirectory;
584 98 : WriteToc(AH);
585 98 : if (!EndCompressFileHandle(tocFH))
586 0 : pg_fatal("could not close TOC file: %m");
587 98 : WriteDataChunks(AH, ctx->pstate);
588 :
589 98 : ParallelBackupEnd(AH, ctx->pstate);
590 :
591 : /*
592 : * In directory mode, there is no need to sync all the entries
593 : * individually. Just recurse once through all the files generated.
594 : */
595 98 : if (AH->dosync)
596 84 : sync_dir_recurse(ctx->directory, AH->sync_method);
597 : }
598 198 : AH->FH = NULL;
599 198 : }
600 :
601 : /*
602 : * Reopen the archive's file handle.
603 : */
604 : static void
605 34 : _ReopenArchive(ArchiveHandle *AH)
606 : {
607 : /*
608 : * Our TOC is in memory, our data files are opened by each child anyway as
609 : * they are separate. We support reopening the archive by just doing
610 : * nothing.
611 : */
612 34 : }
613 :
614 : /*
615 : * LO support
616 : */
617 :
618 : /*
619 : * Called by the archiver when starting to save BLOB DATA (not schema).
620 : * It is called just prior to the dumper's DataDumper routine.
621 : *
622 : * We open the large object TOC file here, so that we can append a line to
623 : * it for each LO.
624 : */
625 : static void
626 18 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
627 : {
628 18 : lclContext *ctx = (lclContext *) AH->formatData;
629 18 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
630 18 : pg_compress_specification compression_spec = {0};
631 : char fname[MAXPGPATH];
632 :
633 18 : setFilePath(AH, fname, tctx->filename);
634 :
635 : /* The LO TOC file is never compressed */
636 18 : compression_spec.algorithm = PG_COMPRESSION_NONE;
637 18 : ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
638 18 : if (!ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
639 0 : pg_fatal("could not open output file \"%s\": %m", fname);
640 18 : }
641 :
642 : /*
643 : * Called by the archiver when we're about to start dumping a LO.
644 : *
645 : * We create a file to write the LO to.
646 : */
647 : static void
648 22 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
649 : {
650 22 : lclContext *ctx = (lclContext *) AH->formatData;
651 : char fname[MAXPGPATH];
652 :
653 22 : snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
654 :
655 22 : ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
656 22 : if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
657 0 : pg_fatal("could not open output file \"%s\": %m", fname);
658 22 : }
659 :
660 : /*
661 : * Called by the archiver when the dumper is finished writing a LO.
662 : *
663 : * We close the LO file and write an entry to the LO TOC file for it.
664 : */
665 : static void
666 22 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
667 : {
668 22 : lclContext *ctx = (lclContext *) AH->formatData;
669 22 : CompressFileHandle *CFH = ctx->LOsTocFH;
670 : char buf[50];
671 : int len;
672 :
673 : /* Close the BLOB data file itself */
674 22 : if (!EndCompressFileHandle(ctx->dataFH))
675 0 : pg_fatal("could not close LO data file: %m");
676 22 : ctx->dataFH = NULL;
677 :
678 : /* register the LO in blobs_NNN.toc */
679 22 : len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
680 22 : if (!CFH->write_func(buf, len, CFH))
681 : {
682 : /* if write didn't set errno, assume problem is no disk space */
683 0 : if (errno == 0)
684 0 : errno = ENOSPC;
685 0 : pg_fatal("could not write to LOs TOC file: %s",
686 : CFH->get_error_func(CFH));
687 : }
688 22 : }
689 :
690 : /*
691 : * Called by the archiver when finishing saving BLOB DATA.
692 : *
693 : * We close the LOs TOC file.
694 : */
695 : static void
696 18 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
697 : {
698 18 : lclContext *ctx = (lclContext *) AH->formatData;
699 :
700 18 : if (!EndCompressFileHandle(ctx->LOsTocFH))
701 0 : pg_fatal("could not close LOs TOC file: %m");
702 18 : ctx->LOsTocFH = NULL;
703 18 : }
704 :
705 : /*
706 : * Gets a relative file name and prepends the output directory, writing the
707 : * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
708 : * big. Can't use a static char[MAXPGPATH] inside the function because we run
709 : * multithreaded on Windows.
710 : */
711 : static void
712 5018 : setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
713 : {
714 5018 : lclContext *ctx = (lclContext *) AH->formatData;
715 : char *dname;
716 :
717 5018 : dname = ctx->directory;
718 :
719 5018 : if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
720 0 : pg_fatal("file name too long: \"%s\"", dname);
721 :
722 5018 : strcpy(buf, dname);
723 5018 : strcat(buf, "/");
724 5018 : strcat(buf, relativeFilename);
725 5018 : }
726 :
727 : /*
728 : * Prepare for parallel restore.
729 : *
730 : * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
731 : * TOC entries' dataLength fields with appropriate values to guide the
732 : * ordering of restore jobs. The source of said data is format-dependent,
733 : * as is the exact meaning of the values.
734 : *
735 : * A format module might also choose to do other setup here.
736 : */
737 : static void
738 10 : _PrepParallelRestore(ArchiveHandle *AH)
739 : {
740 : TocEntry *te;
741 :
742 5646 : for (te = AH->toc->next; te != AH->toc; te = te->next)
743 : {
744 5636 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
745 : char fname[MAXPGPATH];
746 : struct stat st;
747 :
748 : /*
749 : * A dumpable object has set tctx->filename, any other object has not.
750 : * (see _ArchiveEntry).
751 : */
752 5636 : if (tctx->filename == NULL)
753 4338 : continue;
754 :
755 : /* We may ignore items not due to be restored */
756 1298 : if ((te->reqs & (REQ_DATA | REQ_STATS)) == 0)
757 0 : continue;
758 :
759 : /*
760 : * Stat the file and, if successful, put its size in dataLength. When
761 : * using compression, the physical file size might not be a very good
762 : * guide to the amount of work involved in restoring the file, but we
763 : * only need an approximate indicator of that.
764 : */
765 1298 : setFilePath(AH, fname, tctx->filename);
766 :
767 1298 : if (stat(fname, &st) == 0)
768 2 : te->dataLength = st.st_size;
769 1296 : else if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
770 : {
771 1296 : if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
772 1296 : strlcat(fname, ".gz", sizeof(fname));
773 0 : else if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
774 0 : strlcat(fname, ".lz4", sizeof(fname));
775 0 : else if (AH->compression_spec.algorithm == PG_COMPRESSION_ZSTD)
776 0 : strlcat(fname, ".zst", sizeof(fname));
777 :
778 1296 : if (stat(fname, &st) == 0)
779 1296 : te->dataLength = st.st_size;
780 : }
781 :
782 : /*
783 : * If this is a BLOBS entry, what we stat'd was blobs_NNN.toc, which
784 : * most likely is a lot smaller than the actual blob data. We don't
785 : * have a cheap way to estimate how much smaller, but fortunately it
786 : * doesn't matter too much as long as we get the LOs processed
787 : * reasonably early. Arbitrarily scale up by a factor of 1K.
788 : */
789 1298 : if (strcmp(te->desc, "BLOBS") == 0)
790 2 : te->dataLength *= 1024;
791 : }
792 10 : }
793 :
794 : /*
795 : * Clone format-specific fields during parallel restoration.
796 : */
797 : static void
798 60 : _Clone(ArchiveHandle *AH)
799 : {
800 60 : lclContext *ctx = (lclContext *) AH->formatData;
801 :
802 60 : AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
803 60 : memcpy(AH->formatData, ctx, sizeof(lclContext));
804 60 : ctx = (lclContext *) AH->formatData;
805 :
806 : /*
807 : * TOC-entry-local state isn't an issue because any one TOC entry is
808 : * touched by just one worker child.
809 : */
810 :
811 : /*
812 : * We also don't copy the ParallelState pointer (pstate), only the leader
813 : * process ever writes to it.
814 : */
815 60 : }
816 :
817 : static void
818 60 : _DeClone(ArchiveHandle *AH)
819 : {
820 60 : lclContext *ctx = (lclContext *) AH->formatData;
821 :
822 60 : free(ctx);
823 60 : }
824 :
825 : /*
826 : * This function is executed in the child of a parallel backup for a
827 : * directory-format archive and dumps the actual data for one TOC entry.
828 : */
829 : static int
830 1538 : _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
831 : {
832 : /*
833 : * This function returns void. We either fail and die horribly or
834 : * succeed... A failure will be detected by the parent when the child dies
835 : * unexpectedly.
836 : */
837 1538 : WriteDataChunksForTocEntry(AH, te);
838 :
839 1538 : return 0;
840 : }
841 :
842 : /*
843 : * This function is executed in the child of a parallel restore from a
844 : * directory-format archive and restores the actual data for one TOC entry.
845 : */
846 : static int
847 2494 : _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
848 : {
849 2494 : return parallel_restore(AH, te);
850 : }
|
