LCOV - code coverage report
Current view: top level - src/bin/pg_dump - pg_backup_directory.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 240 293 81.9 %
Date: 2024-03-28 14:11:12 Functions: 27 27 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_backup_directory.c
       4             :  *
       5             :  *  A directory format dump is a directory, which contains a "toc.dat" file
       6             :  *  for the TOC, and a separate file for each data entry, named "<oid>.dat".
       7             :  *  Large objects are stored in separate files named "blob_<oid>.dat",
       8             :  *  and there's a plain-text TOC file for them called "blobs.toc". If
       9             :  *  compression is used, each data file is individually compressed and the
      10             :  *  ".gz" suffix is added to the filenames. The TOC files are never
      11             :  *  compressed by pg_dump, however they are accepted with the .gz suffix too,
      12             :  *  in case the user has manually compressed them with 'gzip'.
      13             :  *
      14             :  *  NOTE: This format is identical to the files written in the tar file in
      15             :  *  the 'tar' format, except that we don't write the restore.sql file (TODO),
      16             :  *  and the tar format doesn't support compression. Please keep the formats in
      17             :  *  sync.
      18             :  *
      19             :  *
      20             :  *  Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
      21             :  *  Portions Copyright (c) 1994, Regents of the University of California
      22             :  *  Portions Copyright (c) 2000, Philip Warner
      23             :  *
      24             :  *  Rights are granted to use this software in any way so long
      25             :  *  as this notice is not removed.
      26             :  *
      27             :  *  The author is not responsible for loss or damages that may
      28             :  *  result from its use.
      29             :  *
      30             :  * IDENTIFICATION
      31             :  *      src/bin/pg_dump/pg_backup_directory.c
      32             :  *
      33             :  *-------------------------------------------------------------------------
      34             :  */
      35             : #include "postgres_fe.h"
      36             : 
      37             : #include <dirent.h>
      38             : #include <sys/stat.h>
      39             : 
      40             : #include "common/file_utils.h"
      41             : #include "compress_io.h"
      42             : #include "parallel.h"
      43             : #include "pg_backup_utils.h"
      44             : 
      45             : typedef struct
      46             : {
      47             :     /*
      48             :      * Our archive location. This is basically what the user specified as his
      49             :      * backup file but of course here it is a directory.
      50             :      */
      51             :     char       *directory;
      52             : 
      53             :     CompressFileHandle *dataFH; /* currently open data file */
      54             :     CompressFileHandle *LOsTocFH;   /* file handle for blobs.toc */
      55             :     ParallelState *pstate;      /* for parallel backup / restore */
      56             : } lclContext;
      57             : 
      58             : typedef struct
      59             : {
      60             :     char       *filename;       /* filename excluding the directory (basename) */
      61             : } lclTocEntry;
      62             : 
      63             : /* prototypes for private functions */
      64             : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
      65             : static void _StartData(ArchiveHandle *AH, TocEntry *te);
      66             : static void _EndData(ArchiveHandle *AH, TocEntry *te);
      67             : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
      68             : static int  _WriteByte(ArchiveHandle *AH, const int i);
      69             : static int  _ReadByte(ArchiveHandle *AH);
      70             : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
      71             : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
      72             : static void _CloseArchive(ArchiveHandle *AH);
      73             : static void _ReopenArchive(ArchiveHandle *AH);
      74             : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
      75             : 
      76             : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
      77             : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
      78             : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
      79             : 
      80             : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
      81             : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
      82             : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
      83             : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
      84             : static void _LoadLOs(ArchiveHandle *AH);
      85             : 
      86             : static void _PrepParallelRestore(ArchiveHandle *AH);
      87             : static void _Clone(ArchiveHandle *AH);
      88             : static void _DeClone(ArchiveHandle *AH);
      89             : 
      90             : static int  _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
      91             : static int  _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
      92             : 
      93             : static void setFilePath(ArchiveHandle *AH, char *buf,
      94             :                         const char *relativeFilename);
      95             : 
      96             : /*
      97             :  *  Init routine required by ALL formats. This is a global routine
      98             :  *  and should be declared in pg_backup_archiver.h
      99             :  *
     100             :  *  Its task is to create any extra archive context (using AH->formatData),
     101             :  *  and to initialize the supported function pointers.
     102             :  *
     103             :  *  It should also prepare whatever its input source is for reading/writing,
     104             :  *  and in the case of a read mode connection, it should load the Header & TOC.
     105             :  */
     106             : void
     107          42 : InitArchiveFmt_Directory(ArchiveHandle *AH)
     108             : {
     109             :     lclContext *ctx;
     110             : 
     111             :     /* Assuming static functions, this can be copied for each format. */
     112          42 :     AH->ArchiveEntryPtr = _ArchiveEntry;
     113          42 :     AH->StartDataPtr = _StartData;
     114          42 :     AH->WriteDataPtr = _WriteData;
     115          42 :     AH->EndDataPtr = _EndData;
     116          42 :     AH->WriteBytePtr = _WriteByte;
     117          42 :     AH->ReadBytePtr = _ReadByte;
     118          42 :     AH->WriteBufPtr = _WriteBuf;
     119          42 :     AH->ReadBufPtr = _ReadBuf;
     120          42 :     AH->ClosePtr = _CloseArchive;
     121          42 :     AH->ReopenPtr = _ReopenArchive;
     122          42 :     AH->PrintTocDataPtr = _PrintTocData;
     123          42 :     AH->ReadExtraTocPtr = _ReadExtraToc;
     124          42 :     AH->WriteExtraTocPtr = _WriteExtraToc;
     125          42 :     AH->PrintExtraTocPtr = _PrintExtraToc;
     126             : 
     127          42 :     AH->StartLOsPtr = _StartLOs;
     128          42 :     AH->StartLOPtr = _StartLO;
     129          42 :     AH->EndLOPtr = _EndLO;
     130          42 :     AH->EndLOsPtr = _EndLOs;
     131             : 
     132          42 :     AH->PrepParallelRestorePtr = _PrepParallelRestore;
     133          42 :     AH->ClonePtr = _Clone;
     134          42 :     AH->DeClonePtr = _DeClone;
     135             : 
     136          42 :     AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
     137          42 :     AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
     138             : 
     139             :     /* Set up our private context */
     140          42 :     ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
     141          42 :     AH->formatData = (void *) ctx;
     142             : 
     143          42 :     ctx->dataFH = NULL;
     144          42 :     ctx->LOsTocFH = NULL;
     145             : 
     146             :     /* Initialize LO buffering */
     147          42 :     AH->lo_buf_size = LOBBUFSIZE;
     148          42 :     AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
     149             : 
     150             :     /*
     151             :      * Now open the TOC file
     152             :      */
     153             : 
     154          42 :     if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
     155           0 :         pg_fatal("no output directory specified");
     156             : 
     157          42 :     ctx->directory = AH->fSpec;
     158             : 
     159          42 :     if (AH->mode == archModeWrite)
     160             :     {
     161             :         struct stat st;
     162          20 :         bool        is_empty = false;
     163             : 
     164             :         /* we accept an empty existing directory */
     165          20 :         if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
     166             :         {
     167           0 :             DIR        *dir = opendir(ctx->directory);
     168             : 
     169           0 :             if (dir)
     170             :             {
     171             :                 struct dirent *d;
     172             : 
     173           0 :                 is_empty = true;
     174           0 :                 while (errno = 0, (d = readdir(dir)))
     175             :                 {
     176           0 :                     if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
     177             :                     {
     178           0 :                         is_empty = false;
     179           0 :                         break;
     180             :                     }
     181             :                 }
     182             : 
     183           0 :                 if (errno)
     184           0 :                     pg_fatal("could not read directory \"%s\": %m",
     185             :                              ctx->directory);
     186             : 
     187           0 :                 if (closedir(dir))
     188           0 :                     pg_fatal("could not close directory \"%s\": %m",
     189             :                              ctx->directory);
     190             :             }
     191             :         }
     192             : 
     193          20 :         if (!is_empty && mkdir(ctx->directory, 0700) < 0)
     194           0 :             pg_fatal("could not create directory \"%s\": %m",
     195             :                      ctx->directory);
     196             :     }
     197             :     else
     198             :     {                           /* Read Mode */
     199             :         char        fname[MAXPGPATH];
     200             :         CompressFileHandle *tocFH;
     201             : 
     202          22 :         setFilePath(AH, fname, "toc.dat");
     203             : 
     204          22 :         tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
     205          22 :         if (tocFH == NULL)
     206           0 :             pg_fatal("could not open input file \"%s\": %m", fname);
     207             : 
     208          22 :         ctx->dataFH = tocFH;
     209             : 
     210             :         /*
     211             :          * The TOC of a directory format dump shares the format code of the
     212             :          * tar format.
     213             :          */
     214          22 :         AH->format = archTar;
     215          22 :         ReadHead(AH);
     216          22 :         AH->format = archDirectory;
     217          22 :         ReadToc(AH);
     218             : 
     219             :         /* Nothing else in the file, so close it again... */
     220          22 :         if (!EndCompressFileHandle(tocFH))
     221           0 :             pg_fatal("could not close TOC file: %m");
     222          22 :         ctx->dataFH = NULL;
     223             :     }
     224          42 : }
     225             : 
     226             : /*
     227             :  * Called by the Archiver when the dumper creates a new TOC entry.
     228             :  *
     229             :  * We determine the filename for this entry.
     230             : */
     231             : static void
     232        2614 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
     233             : {
     234             :     lclTocEntry *tctx;
     235             :     char        fn[MAXPGPATH];
     236             : 
     237        2614 :     tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     238        2614 :     if (strcmp(te->desc, "BLOBS") == 0)
     239           8 :         tctx->filename = pg_strdup("blobs.toc");
     240        2606 :     else if (te->dataDumper)
     241             :     {
     242         290 :         snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
     243         290 :         tctx->filename = pg_strdup(fn);
     244             :     }
     245             :     else
     246        2316 :         tctx->filename = NULL;
     247             : 
     248        2614 :     te->formatData = (void *) tctx;
     249        2614 : }
     250             : 
     251             : /*
     252             :  * Called by the Archiver to save any extra format-related TOC entry
     253             :  * data.
     254             :  *
     255             :  * Use the Archiver routines to write data - they are non-endian, and
     256             :  * maintain other important file information.
     257             :  */
     258             : static void
     259        2614 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
     260             : {
     261        2614 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     262             : 
     263             :     /*
     264             :      * A dumpable object has set tctx->filename, any other object has not.
     265             :      * (see _ArchiveEntry).
     266             :      */
     267        2614 :     if (tctx->filename)
     268         298 :         WriteStr(AH, tctx->filename);
     269             :     else
     270        2316 :         WriteStr(AH, "");
     271        2614 : }
     272             : 
     273             : /*
     274             :  * Called by the Archiver to read any extra format-related TOC data.
     275             :  *
     276             :  * Needs to match the order defined in _WriteExtraToc, and should also
     277             :  * use the Archiver input routines.
     278             :  */
     279             : static void
     280        2626 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
     281             : {
     282        2626 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     283             : 
     284        2626 :     if (tctx == NULL)
     285             :     {
     286        2626 :         tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     287        2626 :         te->formatData = (void *) tctx;
     288             :     }
     289             : 
     290        2626 :     tctx->filename = ReadStr(AH);
     291        2626 :     if (strlen(tctx->filename) == 0)
     292             :     {
     293        2326 :         free(tctx->filename);
     294        2326 :         tctx->filename = NULL;
     295             :     }
     296        2626 : }
     297             : 
     298             : /*
     299             :  * Called by the Archiver when restoring an archive to output a comment
     300             :  * that includes useful information about the TOC entry.
     301             :  */
     302             : static void
     303        1808 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
     304             : {
     305        1808 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     306             : 
     307        1808 :     if (AH->public.verbose && tctx->filename)
     308           0 :         ahprintf(AH, "-- File: %s\n", tctx->filename);
     309        1808 : }
     310             : 
     311             : /*
     312             :  * Called by the archiver when saving TABLE DATA (not schema). This routine
     313             :  * should save whatever format-specific information is needed to read
     314             :  * the archive back.
     315             :  *
     316             :  * It is called just prior to the dumper's 'DataDumper' routine being called.
     317             :  *
     318             :  * We create the data file for writing.
     319             :  */
     320             : static void
     321         290 : _StartData(ArchiveHandle *AH, TocEntry *te)
     322             : {
     323         290 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     324         290 :     lclContext *ctx = (lclContext *) AH->formatData;
     325             :     char        fname[MAXPGPATH];
     326             : 
     327         290 :     setFilePath(AH, fname, tctx->filename);
     328             : 
     329         290 :     ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
     330             : 
     331         290 :     if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
     332           0 :         pg_fatal("could not open output file \"%s\": %m", fname);
     333         290 : }
     334             : 
     335             : /*
     336             :  * Called by archiver when dumper calls WriteData. This routine is
     337             :  * called for both LO and table data; it is the responsibility of
     338             :  * the format to manage each kind of data using StartLO/StartData.
     339             :  *
     340             :  * It should only be called from within a DataDumper routine.
     341             :  *
     342             :  * We write the data to the open data file.
     343             :  */
     344             : static void
     345       48556 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
     346             : {
     347       48556 :     lclContext *ctx = (lclContext *) AH->formatData;
     348       48556 :     CompressFileHandle *CFH = ctx->dataFH;
     349             : 
     350       48556 :     errno = 0;
     351       48556 :     if (dLen > 0 && !CFH->write_func(data, dLen, CFH))
     352             :     {
     353             :         /* if write didn't set errno, assume problem is no disk space */
     354           0 :         if (errno == 0)
     355           0 :             errno = ENOSPC;
     356           0 :         pg_fatal("could not write to output file: %s",
     357             :                  CFH->get_error_func(CFH));
     358             :     }
     359       48556 : }
     360             : 
     361             : /*
     362             :  * Called by the archiver when a dumper's 'DataDumper' routine has
     363             :  * finished.
     364             :  *
     365             :  * We close the data file.
     366             :  */
     367             : static void
     368         290 : _EndData(ArchiveHandle *AH, TocEntry *te)
     369             : {
     370         290 :     lclContext *ctx = (lclContext *) AH->formatData;
     371             : 
     372             :     /* Close the file */
     373         290 :     if (!EndCompressFileHandle(ctx->dataFH))
     374           0 :         pg_fatal("could not close data file: %m");
     375             : 
     376         290 :     ctx->dataFH = NULL;
     377         290 : }
     378             : 
     379             : /*
     380             :  * Print data for a given file (can be a LO as well)
     381             :  */
     382             : static void
     383         244 : _PrintFileData(ArchiveHandle *AH, char *filename)
     384             : {
     385         244 :     size_t      cnt = 0;
     386             :     char       *buf;
     387             :     size_t      buflen;
     388             :     CompressFileHandle *CFH;
     389             : 
     390         244 :     if (!filename)
     391           0 :         return;
     392             : 
     393         244 :     CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
     394         244 :     if (!CFH)
     395           0 :         pg_fatal("could not open input file \"%s\": %m", filename);
     396             : 
     397         244 :     buflen = DEFAULT_IO_BUFFER_SIZE;
     398         244 :     buf = pg_malloc(buflen);
     399             : 
     400         562 :     while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
     401             :     {
     402         318 :         ahwrite(buf, 1, cnt, AH);
     403             :     }
     404             : 
     405         244 :     free(buf);
     406         244 :     if (!EndCompressFileHandle(CFH))
     407           0 :         pg_fatal("could not close data file \"%s\": %m", filename);
     408             : }
     409             : 
     410             : /*
     411             :  * Print data for a given TOC entry
     412             : */
     413             : static void
     414         238 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
     415             : {
     416         238 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     417             : 
     418         238 :     if (!tctx->filename)
     419           0 :         return;
     420             : 
     421         238 :     if (strcmp(te->desc, "BLOBS") == 0)
     422           6 :         _LoadLOs(AH);
     423             :     else
     424             :     {
     425             :         char        fname[MAXPGPATH];
     426             : 
     427         232 :         setFilePath(AH, fname, tctx->filename);
     428         232 :         _PrintFileData(AH, fname);
     429             :     }
     430             : }
     431             : 
     432             : static void
     433           6 : _LoadLOs(ArchiveHandle *AH)
     434             : {
     435             :     Oid         oid;
     436           6 :     lclContext *ctx = (lclContext *) AH->formatData;
     437             :     CompressFileHandle *CFH;
     438             :     char        tocfname[MAXPGPATH];
     439             :     char        line[MAXPGPATH];
     440             : 
     441           6 :     StartRestoreLOs(AH);
     442             : 
     443           6 :     setFilePath(AH, tocfname, "blobs.toc");
     444             : 
     445           6 :     CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
     446             : 
     447           6 :     if (ctx->LOsTocFH == NULL)
     448           0 :         pg_fatal("could not open large object TOC file \"%s\" for input: %m",
     449             :                  tocfname);
     450             : 
     451             :     /* Read the LOs TOC file line-by-line, and process each LO */
     452          18 :     while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
     453             :     {
     454             :         char        lofname[MAXPGPATH + 1];
     455             :         char        path[MAXPGPATH];
     456             : 
     457             :         /* Can't overflow because line and lofname are the same length */
     458          12 :         if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, lofname) != 2)
     459           0 :             pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
     460             :                      tocfname, line);
     461             : 
     462          12 :         StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
     463          12 :         snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, lofname);
     464          12 :         _PrintFileData(AH, path);
     465          12 :         EndRestoreLO(AH, oid);
     466             :     }
     467           6 :     if (!CFH->eof_func(CFH))
     468           0 :         pg_fatal("error reading large object TOC file \"%s\"",
     469             :                  tocfname);
     470             : 
     471           6 :     if (!EndCompressFileHandle(ctx->LOsTocFH))
     472           0 :         pg_fatal("could not close large object TOC file \"%s\": %m",
     473             :                  tocfname);
     474             : 
     475           6 :     ctx->LOsTocFH = NULL;
     476             : 
     477           6 :     EndRestoreLOs(AH);
     478           6 : }
     479             : 
     480             : 
     481             : /*
     482             :  * Write a byte of data to the archive.
     483             :  * Called by the archiver to do integer & byte output to the archive.
     484             :  * These routines are only used to read & write the headers & TOC.
     485             :  */
     486             : static int
     487      244570 : _WriteByte(ArchiveHandle *AH, const int i)
     488             : {
     489      244570 :     unsigned char c = (unsigned char) i;
     490      244570 :     lclContext *ctx = (lclContext *) AH->formatData;
     491      244570 :     CompressFileHandle *CFH = ctx->dataFH;
     492             : 
     493      244570 :     errno = 0;
     494      244570 :     if (!CFH->write_func(&c, 1, CFH))
     495             :     {
     496             :         /* if write didn't set errno, assume problem is no disk space */
     497           0 :         if (errno == 0)
     498           0 :             errno = ENOSPC;
     499           0 :         pg_fatal("could not write to output file: %s",
     500             :                  CFH->get_error_func(CFH));
     501             :     }
     502             : 
     503      244570 :     return 1;
     504             : }
     505             : 
     506             : /*
     507             :  * Read a byte of data from the archive.
     508             :  * Called by the archiver to read bytes & integers from the archive.
     509             :  * These routines are only used to read & write headers & TOC.
     510             :  * EOF should be treated as a fatal error.
     511             :  */
     512             : static int
     513      245724 : _ReadByte(ArchiveHandle *AH)
     514             : {
     515      245724 :     lclContext *ctx = (lclContext *) AH->formatData;
     516      245724 :     CompressFileHandle *CFH = ctx->dataFH;
     517             : 
     518      245724 :     return CFH->getc_func(CFH);
     519             : }
     520             : 
     521             : /*
     522             :  * Write a buffer of data to the archive.
     523             :  * Called by the archiver to write a block of bytes to the TOC or a data file.
     524             :  */
     525             : static void
     526       29514 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
     527             : {
     528       29514 :     lclContext *ctx = (lclContext *) AH->formatData;
     529       29514 :     CompressFileHandle *CFH = ctx->dataFH;
     530             : 
     531       29514 :     errno = 0;
     532       29514 :     if (!CFH->write_func(buf, len, CFH))
     533             :     {
     534             :         /* if write didn't set errno, assume problem is no disk space */
     535           0 :         if (errno == 0)
     536           0 :             errno = ENOSPC;
     537           0 :         pg_fatal("could not write to output file: %s",
     538             :                  CFH->get_error_func(CFH));
     539             :     }
     540       29514 : }
     541             : 
     542             : /*
     543             :  * Read a block of bytes from the archive.
     544             :  *
     545             :  * Called by the archiver to read a block of bytes from the archive
     546             :  */
     547             : static void
     548       29626 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
     549             : {
     550       29626 :     lclContext *ctx = (lclContext *) AH->formatData;
     551       29626 :     CompressFileHandle *CFH = ctx->dataFH;
     552             : 
     553             :     /*
     554             :      * If there was an I/O error, we already exited in readF(), so here we
     555             :      * exit on short reads.
     556             :      */
     557       29626 :     if (!CFH->read_func(buf, len, NULL, CFH))
     558           0 :         pg_fatal("could not read from input file: end of file");
     559       29626 : }
     560             : 
     561             : /*
     562             :  * Close the archive.
     563             :  *
     564             :  * When writing the archive, this is the routine that actually starts
     565             :  * the process of saving it to files. No data should be written prior
     566             :  * to this point, since the user could sort the TOC after creating it.
     567             :  *
     568             :  * If an archive is to be written, this routine must call:
     569             :  *      WriteHead           to save the archive header
     570             :  *      WriteToc            to save the TOC entries
     571             :  *      WriteDataChunks     to save all data & LOs.
     572             :  */
     573             : static void
     574          42 : _CloseArchive(ArchiveHandle *AH)
     575             : {
     576          42 :     lclContext *ctx = (lclContext *) AH->formatData;
     577             : 
     578          42 :     if (AH->mode == archModeWrite)
     579             :     {
     580             :         CompressFileHandle *tocFH;
     581          20 :         pg_compress_specification compression_spec = {0};
     582             :         char        fname[MAXPGPATH];
     583             : 
     584          20 :         setFilePath(AH, fname, "toc.dat");
     585             : 
     586             :         /* this will actually fork the processes for a parallel backup */
     587          20 :         ctx->pstate = ParallelBackupStart(AH);
     588             : 
     589             :         /* The TOC is always created uncompressed */
     590          20 :         compression_spec.algorithm = PG_COMPRESSION_NONE;
     591          20 :         tocFH = InitCompressFileHandle(compression_spec);
     592          20 :         if (!tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
     593           0 :             pg_fatal("could not open output file \"%s\": %m", fname);
     594          20 :         ctx->dataFH = tocFH;
     595             : 
     596             :         /*
     597             :          * Write 'tar' in the format field of the toc.dat file. The directory
     598             :          * is compatible with 'tar', so there's no point having a different
     599             :          * format code for it.
     600             :          */
     601          20 :         AH->format = archTar;
     602          20 :         WriteHead(AH);
     603          20 :         AH->format = archDirectory;
     604          20 :         WriteToc(AH);
     605          20 :         if (!EndCompressFileHandle(tocFH))
     606           0 :             pg_fatal("could not close TOC file: %m");
     607          20 :         WriteDataChunks(AH, ctx->pstate);
     608             : 
     609          20 :         ParallelBackupEnd(AH, ctx->pstate);
     610             : 
     611             :         /*
     612             :          * In directory mode, there is no need to sync all the entries
     613             :          * individually. Just recurse once through all the files generated.
     614             :          */
     615          20 :         if (AH->dosync)
     616           8 :             sync_dir_recurse(ctx->directory, AH->sync_method);
     617             :     }
     618          42 :     AH->FH = NULL;
     619          42 : }
     620             : 
     621             : /*
     622             :  * Reopen the archive's file handle.
     623             :  */
     624             : static void
     625          28 : _ReopenArchive(ArchiveHandle *AH)
     626             : {
     627             :     /*
     628             :      * Our TOC is in memory, our data files are opened by each child anyway as
     629             :      * they are separate. We support reopening the archive by just doing
     630             :      * nothing.
     631             :      */
     632          28 : }
     633             : 
     634             : /*
     635             :  * LO support
     636             :  */
     637             : 
     638             : /*
     639             :  * Called by the archiver when starting to save all BLOB DATA (not schema).
     640             :  * It is called just prior to the dumper's DataDumper routine.
     641             :  *
     642             :  * We open the large object TOC file here, so that we can append a line to
     643             :  * it for each LO.
     644             :  */
     645             : static void
     646           8 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
     647             : {
     648           8 :     lclContext *ctx = (lclContext *) AH->formatData;
     649           8 :     pg_compress_specification compression_spec = {0};
     650             :     char        fname[MAXPGPATH];
     651             : 
     652           8 :     setFilePath(AH, fname, "blobs.toc");
     653             : 
     654             :     /* The LO TOC file is never compressed */
     655           8 :     compression_spec.algorithm = PG_COMPRESSION_NONE;
     656           8 :     ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
     657           8 :     if (!ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
     658           0 :         pg_fatal("could not open output file \"%s\": %m", fname);
     659           8 : }
     660             : 
     661             : /*
     662             :  * Called by the archiver when we're about to start dumping a LO.
     663             :  *
     664             :  * We create a file to write the LO to.
     665             :  */
     666             : static void
     667          16 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
     668             : {
     669          16 :     lclContext *ctx = (lclContext *) AH->formatData;
     670             :     char        fname[MAXPGPATH];
     671             : 
     672          16 :     snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
     673             : 
     674          16 :     ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
     675          16 :     if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
     676           0 :         pg_fatal("could not open output file \"%s\": %m", fname);
     677          16 : }
     678             : 
     679             : /*
     680             :  * Called by the archiver when the dumper is finished writing a LO.
     681             :  *
     682             :  * We close the LO file and write an entry to the LO TOC file for it.
     683             :  */
     684             : static void
     685          16 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
     686             : {
     687          16 :     lclContext *ctx = (lclContext *) AH->formatData;
     688          16 :     CompressFileHandle *CFH = ctx->LOsTocFH;
     689             :     char        buf[50];
     690             :     int         len;
     691             : 
     692             :     /* Close the BLOB data file itself */
     693          16 :     if (!EndCompressFileHandle(ctx->dataFH))
     694           0 :         pg_fatal("could not close LO data file: %m");
     695          16 :     ctx->dataFH = NULL;
     696             : 
     697             :     /* register the LO in blobs.toc */
     698          16 :     len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
     699          16 :     if (!CFH->write_func(buf, len, CFH))
     700             :     {
     701             :         /* if write didn't set errno, assume problem is no disk space */
     702           0 :         if (errno == 0)
     703           0 :             errno = ENOSPC;
     704           0 :         pg_fatal("could not write to LOs TOC file: %s",
     705             :                  CFH->get_error_func(CFH));
     706             :     }
     707          16 : }
     708             : 
     709             : /*
     710             :  * Called by the archiver when finishing saving all BLOB DATA.
     711             :  *
     712             :  * We close the LOs TOC file.
     713             :  */
     714             : static void
     715           8 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
     716             : {
     717           8 :     lclContext *ctx = (lclContext *) AH->formatData;
     718             : 
     719           8 :     if (!EndCompressFileHandle(ctx->LOsTocFH))
     720           0 :         pg_fatal("could not close LOs TOC file: %m");
     721           8 :     ctx->LOsTocFH = NULL;
     722           8 : }
     723             : 
     724             : /*
     725             :  * Gets a relative file name and prepends the output directory, writing the
     726             :  * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
     727             :  * big. Can't use a static char[MAXPGPATH] inside the function because we run
     728             :  * multithreaded on Windows.
     729             :  */
     730             : static void
     731         610 : setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
     732             : {
     733         610 :     lclContext *ctx = (lclContext *) AH->formatData;
     734             :     char       *dname;
     735             : 
     736         610 :     dname = ctx->directory;
     737             : 
     738         610 :     if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
     739           0 :         pg_fatal("file name too long: \"%s\"", dname);
     740             : 
     741         610 :     strcpy(buf, dname);
     742         610 :     strcat(buf, "/");
     743         610 :     strcat(buf, relativeFilename);
     744         610 : }
     745             : 
     746             : /*
     747             :  * Prepare for parallel restore.
     748             :  *
     749             :  * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
     750             :  * TOC entries' dataLength fields with appropriate values to guide the
     751             :  * ordering of restore jobs.  The source of said data is format-dependent,
     752             :  * as is the exact meaning of the values.
     753             :  *
     754             :  * A format module might also choose to do other setup here.
     755             :  */
     756             : static void
     757           8 : _PrepParallelRestore(ArchiveHandle *AH)
     758             : {
     759             :     TocEntry   *te;
     760             : 
     761         200 :     for (te = AH->toc->next; te != AH->toc; te = te->next)
     762             :     {
     763         192 :         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     764             :         char        fname[MAXPGPATH];
     765             :         struct stat st;
     766             : 
     767             :         /*
     768             :          * A dumpable object has set tctx->filename, any other object has not.
     769             :          * (see _ArchiveEntry).
     770             :          */
     771         192 :         if (tctx->filename == NULL)
     772         160 :             continue;
     773             : 
     774             :         /* We may ignore items not due to be restored */
     775          32 :         if ((te->reqs & REQ_DATA) == 0)
     776           0 :             continue;
     777             : 
     778             :         /*
     779             :          * Stat the file and, if successful, put its size in dataLength.  When
     780             :          * using compression, the physical file size might not be a very good
     781             :          * guide to the amount of work involved in restoring the file, but we
     782             :          * only need an approximate indicator of that.
     783             :          */
     784          32 :         setFilePath(AH, fname, tctx->filename);
     785             : 
     786          32 :         if (stat(fname, &st) == 0)
     787           0 :             te->dataLength = st.st_size;
     788          32 :         else if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
     789             :         {
     790          32 :             if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
     791          32 :                 strlcat(fname, ".gz", sizeof(fname));
     792           0 :             else if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
     793           0 :                 strlcat(fname, ".lz4", sizeof(fname));
     794           0 :             else if (AH->compression_spec.algorithm == PG_COMPRESSION_ZSTD)
     795           0 :                 strlcat(fname, ".zst", sizeof(fname));
     796             : 
     797          32 :             if (stat(fname, &st) == 0)
     798          32 :                 te->dataLength = st.st_size;
     799             :         }
     800             : 
     801             :         /*
     802             :          * If this is the BLOBS entry, what we stat'd was blobs.toc, which
     803             :          * most likely is a lot smaller than the actual blob data.  We don't
     804             :          * have a cheap way to estimate how much smaller, but fortunately it
     805             :          * doesn't matter too much as long as we get the LOs processed
     806             :          * reasonably early.  Arbitrarily scale up by a factor of 1K.
     807             :          */
     808          32 :         if (strcmp(te->desc, "BLOBS") == 0)
     809           0 :             te->dataLength *= 1024;
     810             :     }
     811           8 : }
     812             : 
     813             : /*
     814             :  * Clone format-specific fields during parallel restoration.
     815             :  */
     816             : static void
     817          52 : _Clone(ArchiveHandle *AH)
     818             : {
     819          52 :     lclContext *ctx = (lclContext *) AH->formatData;
     820             : 
     821          52 :     AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
     822          52 :     memcpy(AH->formatData, ctx, sizeof(lclContext));
     823          52 :     ctx = (lclContext *) AH->formatData;
     824             : 
     825             :     /*
     826             :      * Note: we do not make a local lo_buf because we expect at most one BLOBS
     827             :      * entry per archive, so no parallelism is possible.  Likewise,
     828             :      * TOC-entry-local state isn't an issue because any one TOC entry is
     829             :      * touched by just one worker child.
     830             :      */
     831             : 
     832             :     /*
     833             :      * We also don't copy the ParallelState pointer (pstate), only the leader
     834             :      * process ever writes to it.
     835             :      */
     836          52 : }
     837             : 
     838             : static void
     839          52 : _DeClone(ArchiveHandle *AH)
     840             : {
     841          52 :     lclContext *ctx = (lclContext *) AH->formatData;
     842             : 
     843          52 :     free(ctx);
     844          52 : }
     845             : 
     846             : /*
     847             :  * This function is executed in the child of a parallel backup for a
     848             :  * directory-format archive and dumps the actual data for one TOC entry.
     849             :  */
     850             : static int
     851         230 : _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
     852             : {
     853             :     /*
     854             :      * This function returns void. We either fail and die horribly or
     855             :      * succeed... A failure will be detected by the parent when the child dies
     856             :      * unexpectedly.
     857             :      */
     858         230 :     WriteDataChunksForTocEntry(AH, te);
     859             : 
     860         230 :     return 0;
     861             : }
     862             : 
     863             : /*
     864             :  * This function is executed in the child of a parallel restore from a
     865             :  * directory-format archive and restores the actual data for one TOC entry.
     866             :  */
     867             : static int
     868          92 : _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
     869             : {
     870          92 :     return parallel_restore(AH, te);
     871             : }

Generated by: LCOV version 1.14