LCOV - code coverage report
Current view: top level - src/bin/pg_dump - pg_backup_directory.c (source / functions) Hit Total Coverage
Test: PostgreSQL 18devel Lines: 241 294 82.0 %
Date: 2024-10-10 04:14:55 Functions: 27 27 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_backup_directory.c
       4             :  *
       5             :  *  A directory format dump is a directory, which contains a "toc.dat" file
       6             :  *  for the TOC, and a separate file for each data entry, named "<oid>.dat".
       7             :  *  Large objects are stored in separate files named "blob_<oid>.dat",
       8             :  *  and there's a plain-text TOC file for each BLOBS TOC entry named
       9             :  *  "blobs_<dumpID>.toc" (or just "blobs.toc" in archive versions before 16).
      10             :  *
      11             :  *  If compression is used, each data file is individually compressed and the
      12             :  *  ".gz" suffix is added to the filenames. The TOC files are never
      13             :  *  compressed by pg_dump, however they are accepted with the .gz suffix too,
      14             :  *  in case the user has manually compressed them with 'gzip'.
      15             :  *
      16             :  *  NOTE: This format is identical to the files written in the tar file in
      17             :  *  the 'tar' format, except that we don't write the restore.sql file (TODO),
      18             :  *  and the tar format doesn't support compression. Please keep the formats in
      19             :  *  sync.
      20             :  *
      21             :  *
      22             :  *  Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
      23             :  *  Portions Copyright (c) 1994, Regents of the University of California
      24             :  *  Portions Copyright (c) 2000, Philip Warner
      25             :  *
      26             :  *  Rights are granted to use this software in any way so long
      27             :  *  as this notice is not removed.
      28             :  *
      29             :  *  The author is not responsible for loss or damages that may
      30             :  *  result from its use.
      31             :  *
      32             :  * IDENTIFICATION
      33             :  *      src/bin/pg_dump/pg_backup_directory.c
      34             :  *
      35             :  *-------------------------------------------------------------------------
      36             :  */
      37             : #include "postgres_fe.h"
      38             : 
      39             : #include <dirent.h>
      40             : #include <sys/stat.h>
      41             : 
      42             : #include "common/file_utils.h"
      43             : #include "compress_io.h"
      44             : #include "parallel.h"
      45             : #include "pg_backup_utils.h"
      46             : 
      47             : typedef struct
      48             : {
      49             :     /*
      50             :      * Our archive location. This is basically what the user specified as his
      51             :      * backup file but of course here it is a directory.
      52             :      */
      53             :     char       *directory;
      54             : 
      55             :     CompressFileHandle *dataFH; /* currently open data file */
      56             :     CompressFileHandle *LOsTocFH;   /* file handle for blobs_NNN.toc */
      57             :     ParallelState *pstate;      /* for parallel backup / restore */
      58             : } lclContext;
      59             : 
      60             : typedef struct
      61             : {
      62             :     char       *filename;       /* filename excluding the directory (basename) */
      63             : } lclTocEntry;
      64             : 
      65             : /* prototypes for private functions */
      66             : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
      67             : static void _StartData(ArchiveHandle *AH, TocEntry *te);
      68             : static void _EndData(ArchiveHandle *AH, TocEntry *te);
      69             : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
      70             : static int  _WriteByte(ArchiveHandle *AH, const int i);
      71             : static int  _ReadByte(ArchiveHandle *AH);
      72             : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
      73             : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
      74             : static void _CloseArchive(ArchiveHandle *AH);
      75             : static void _ReopenArchive(ArchiveHandle *AH);
      76             : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
      77             : 
      78             : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
      79             : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
      80             : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
      81             : 
      82             : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
      83             : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
      84             : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
      85             : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
      86             : static void _LoadLOs(ArchiveHandle *AH, TocEntry *te);
      87             : 
      88             : static void _PrepParallelRestore(ArchiveHandle *AH);
      89             : static void _Clone(ArchiveHandle *AH);
      90             : static void _DeClone(ArchiveHandle *AH);
      91             : 
      92             : static int  _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
      93             : static int  _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
      94             : 
      95             : static void setFilePath(ArchiveHandle *AH, char *buf,
      96             :                         const char *relativeFilename);
      97             : 
      98             : /*
      99             :  *  Init routine required by ALL formats. This is a global routine
     100             :  *  and should be declared in pg_backup_archiver.h
     101             :  *
     102             :  *  Its task is to create any extra archive context (using AH->formatData),
     103             :  *  and to initialize the supported function pointers.
     104             :  *
     105             :  *  It should also prepare whatever its input source is for reading/writing,
     106             :  *  and in the case of a read mode connection, it should load the Header & TOC.
     107             :  */
     108             : void
     109          44 : InitArchiveFmt_Directory(ArchiveHandle *AH)
     110             : {
     111             :     lclContext *ctx;
     112             : 
     113             :     /* Assuming static functions, this can be copied for each format. */
     114          44 :     AH->ArchiveEntryPtr = _ArchiveEntry;
     115          44 :     AH->StartDataPtr = _StartData;
     116          44 :     AH->WriteDataPtr = _WriteData;
     117          44 :     AH->EndDataPtr = _EndData;
     118          44 :     AH->WriteBytePtr = _WriteByte;
     119          44 :     AH->ReadBytePtr = _ReadByte;
     120          44 :     AH->WriteBufPtr = _WriteBuf;
     121          44 :     AH->ReadBufPtr = _ReadBuf;
     122          44 :     AH->ClosePtr = _CloseArchive;
     123          44 :     AH->ReopenPtr = _ReopenArchive;
     124          44 :     AH->PrintTocDataPtr = _PrintTocData;
     125          44 :     AH->ReadExtraTocPtr = _ReadExtraToc;
     126          44 :     AH->WriteExtraTocPtr = _WriteExtraToc;
     127          44 :     AH->PrintExtraTocPtr = _PrintExtraToc;
     128             : 
     129          44 :     AH->StartLOsPtr = _StartLOs;
     130          44 :     AH->StartLOPtr = _StartLO;
     131          44 :     AH->EndLOPtr = _EndLO;
     132          44 :     AH->EndLOsPtr = _EndLOs;
     133             : 
     134          44 :     AH->PrepParallelRestorePtr = _PrepParallelRestore;
     135          44 :     AH->ClonePtr = _Clone;
     136          44 :     AH->DeClonePtr = _DeClone;
     137             : 
     138          44 :     AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
     139          44 :     AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
     140             : 
     141             :     /* Set up our private context */
     142          44 :     ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
     143          44 :     AH->formatData = (void *) ctx;
     144             : 
     145          44 :     ctx->dataFH = NULL;
     146          44 :     ctx->LOsTocFH = NULL;
     147             : 
     148             :     /*
     149             :      * Now open the TOC file
     150             :      */
     151             : 
     152          44 :     if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
     153           0 :         pg_fatal("no output directory specified");
     154             : 
     155          44 :     ctx->directory = AH->fSpec;
     156             : 
     157          44 :     if (AH->mode == archModeWrite)
     158             :     {
     159             :         struct stat st;
     160          20 :         bool        is_empty = false;
     161             : 
     162             :         /* we accept an empty existing directory */
     163          20 :         if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
     164             :         {
     165           0 :             DIR        *dir = opendir(ctx->directory);
     166             : 
     167           0 :             if (dir)
     168             :             {
     169             :                 struct dirent *d;
     170             : 
     171           0 :                 is_empty = true;
     172           0 :                 while (errno = 0, (d = readdir(dir)))
     173             :                 {
     174           0 :                     if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
     175             :                     {
     176           0 :                         is_empty = false;
     177           0 :                         break;
     178             :                     }
     179             :                 }
     180             : 
     181           0 :                 if (errno)
     182           0 :                     pg_fatal("could not read directory \"%s\": %m",
     183             :                              ctx->directory);
     184             : 
     185           0 :                 if (closedir(dir))
     186           0 :                     pg_fatal("could not close directory \"%s\": %m",
     187             :                              ctx->directory);
     188             :             }
     189             :         }
     190             : 
     191          20 :         if (!is_empty && mkdir(ctx->directory, 0700) < 0)
     192           0 :             pg_fatal("could not create directory \"%s\": %m",
     193             :                      ctx->directory);
     194             :     }
     195             :     else
     196             :     {                           /* Read Mode */
     197             :         char        fname[MAXPGPATH];
     198             :         CompressFileHandle *tocFH;
     199             : 
     200          24 :         setFilePath(AH, fname, "toc.dat");
     201             : 
     202          24 :         tocFH = InitDiscoverCompressFileHandle(fname, PG_BINARY_R);
     203          24 :         if (tocFH == NULL)
     204           0 :             pg_fatal("could not open input file \"%s\": %m", fname);
     205             : 
     206          24 :         ctx->dataFH = tocFH;
     207             : 
     208             :         /*
     209             :          * The TOC of a directory format dump shares the format code of the
     210             :          * tar format.
     211             :          */
     212          24 :         AH->format = archTar;
     213          24 :         ReadHead(AH);
     214          24 :         AH->format = archDirectory;
     215          24 :         ReadToc(AH);
     216             : 
     217             :         /* Nothing else in the file, so close it again... */
     218          24 :         if (!EndCompressFileHandle(tocFH))
     219           0 :             pg_fatal("could not close TOC file: %m");
     220          24 :         ctx->dataFH = NULL;
     221             :     }
     222          44 : }
     223             : 
     224             : /*
     225             :  * Called by the Archiver when the dumper creates a new TOC entry.
     226             :  *
     227             :  * We determine the filename for this entry.
     228             : */
     229             : static void
     230        2622 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
     231             : {
     232             :     lclTocEntry *tctx;
     233             :     char        fn[MAXPGPATH];
     234             : 
     235        2622 :     tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     236        2622 :     if (strcmp(te->desc, "BLOBS") == 0)
     237             :     {
     238          16 :         snprintf(fn, MAXPGPATH, "blobs_%d.toc", te->dumpId);
     239          16 :         tctx->filename = pg_strdup(fn);
     240             :     }
     241        2606 :     else if (te->dataDumper)
     242             :     {
     243         290 :         snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
     244         290 :         tctx->filename = pg_strdup(fn);
     245             :     }
     246             :     else
     247        2316 :         tctx->filename = NULL;
     248             : 
     249        2622 :     te->formatData = (void *) tctx;
     250        2622 : }
     251             : 
     252             : /*
     253             :  * Called by the Archiver to save any extra format-related TOC entry
     254             :  * data.
     255             :  *
     256             :  * Use the Archiver routines to write data - they are non-endian, and
     257             :  * maintain other important file information.
     258             :  */
     259             : static void
     260        2622 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
     261             : {
     262        2622 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     263             : 
     264             :     /*
     265             :      * A dumpable object has set tctx->filename, any other object has not.
     266             :      * (see _ArchiveEntry).
     267             :      */
     268        2622 :     if (tctx->filename)
     269         306 :         WriteStr(AH, tctx->filename);
     270             :     else
     271        2316 :         WriteStr(AH, "");
     272        2622 : }
     273             : 
     274             : /*
     275             :  * Called by the Archiver to read any extra format-related TOC data.
     276             :  *
     277             :  * Needs to match the order defined in _WriteExtraToc, and should also
     278             :  * use the Archiver input routines.
     279             :  */
     280             : static void
     281        3202 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
     282             : {
     283        3202 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     284             : 
     285        3202 :     if (tctx == NULL)
     286             :     {
     287        3202 :         tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     288        3202 :         te->formatData = (void *) tctx;
     289             :     }
     290             : 
     291        3202 :     tctx->filename = ReadStr(AH);
     292        3202 :     if (strlen(tctx->filename) == 0)
     293             :     {
     294        2830 :         free(tctx->filename);
     295        2830 :         tctx->filename = NULL;
     296             :     }
     297        3202 : }
     298             : 
     299             : /*
     300             :  * Called by the Archiver when restoring an archive to output a comment
     301             :  * that includes useful information about the TOC entry.
     302             :  */
     303             : static void
     304        2372 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
     305             : {
     306        2372 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     307             : 
     308        2372 :     if (AH->public.verbose && tctx->filename)
     309           0 :         ahprintf(AH, "-- File: %s\n", tctx->filename);
     310        2372 : }
     311             : 
     312             : /*
     313             :  * Called by the archiver when saving TABLE DATA (not schema). This routine
     314             :  * should save whatever format-specific information is needed to read
     315             :  * the archive back.
     316             :  *
     317             :  * It is called just prior to the dumper's 'DataDumper' routine being called.
     318             :  *
     319             :  * We create the data file for writing.
     320             :  */
     321             : static void
     322         290 : _StartData(ArchiveHandle *AH, TocEntry *te)
     323             : {
     324         290 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     325         290 :     lclContext *ctx = (lclContext *) AH->formatData;
     326             :     char        fname[MAXPGPATH];
     327             : 
     328         290 :     setFilePath(AH, fname, tctx->filename);
     329             : 
     330         290 :     ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
     331             : 
     332         290 :     if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
     333           0 :         pg_fatal("could not open output file \"%s\": %m", fname);
     334         290 : }
     335             : 
     336             : /*
     337             :  * Called by archiver when dumper calls WriteData. This routine is
     338             :  * called for both LO and table data; it is the responsibility of
     339             :  * the format to manage each kind of data using StartLO/StartData.
     340             :  *
     341             :  * It should only be called from within a DataDumper routine.
     342             :  *
     343             :  * We write the data to the open data file.
     344             :  */
     345             : static void
     346       48556 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
     347             : {
     348       48556 :     lclContext *ctx = (lclContext *) AH->formatData;
     349       48556 :     CompressFileHandle *CFH = ctx->dataFH;
     350             : 
     351       48556 :     errno = 0;
     352       48556 :     if (dLen > 0 && !CFH->write_func(data, dLen, CFH))
     353             :     {
     354             :         /* if write didn't set errno, assume problem is no disk space */
     355           0 :         if (errno == 0)
     356           0 :             errno = ENOSPC;
     357           0 :         pg_fatal("could not write to output file: %s",
     358             :                  CFH->get_error_func(CFH));
     359             :     }
     360       48556 : }
     361             : 
     362             : /*
     363             :  * Called by the archiver when a dumper's 'DataDumper' routine has
     364             :  * finished.
     365             :  *
     366             :  * We close the data file.
     367             :  */
     368             : static void
     369         290 : _EndData(ArchiveHandle *AH, TocEntry *te)
     370             : {
     371         290 :     lclContext *ctx = (lclContext *) AH->formatData;
     372             : 
     373             :     /* Close the file */
     374         290 :     if (!EndCompressFileHandle(ctx->dataFH))
     375           0 :         pg_fatal("could not close data file: %m");
     376             : 
     377         290 :     ctx->dataFH = NULL;
     378         290 : }
     379             : 
     380             : /*
     381             :  * Print data for a given file (can be a LO as well)
     382             :  */
     383             : static void
     384         308 : _PrintFileData(ArchiveHandle *AH, char *filename)
     385             : {
     386         308 :     size_t      cnt = 0;
     387             :     char       *buf;
     388             :     size_t      buflen;
     389             :     CompressFileHandle *CFH;
     390             : 
     391         308 :     if (!filename)
     392           0 :         return;
     393             : 
     394         308 :     CFH = InitDiscoverCompressFileHandle(filename, PG_BINARY_R);
     395         308 :     if (!CFH)
     396           0 :         pg_fatal("could not open input file \"%s\": %m", filename);
     397             : 
     398         308 :     buflen = DEFAULT_IO_BUFFER_SIZE;
     399         308 :     buf = pg_malloc(buflen);
     400             : 
     401         694 :     while (CFH->read_func(buf, buflen, &cnt, CFH) && cnt > 0)
     402             :     {
     403         386 :         ahwrite(buf, 1, cnt, AH);
     404             :     }
     405             : 
     406         308 :     free(buf);
     407         308 :     if (!EndCompressFileHandle(CFH))
     408           0 :         pg_fatal("could not close data file \"%s\": %m", filename);
     409             : }
     410             : 
     411             : /*
     412             :  * Print data for a given TOC entry
     413             : */
     414             : static void
     415         308 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
     416             : {
     417         308 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     418             : 
     419         308 :     if (!tctx->filename)
     420           0 :         return;
     421             : 
     422         308 :     if (strcmp(te->desc, "BLOBS") == 0)
     423          16 :         _LoadLOs(AH, te);
     424             :     else
     425             :     {
     426             :         char        fname[MAXPGPATH];
     427             : 
     428         292 :         setFilePath(AH, fname, tctx->filename);
     429         292 :         _PrintFileData(AH, fname);
     430             :     }
     431             : }
     432             : 
     433             : static void
     434          16 : _LoadLOs(ArchiveHandle *AH, TocEntry *te)
     435             : {
     436             :     Oid         oid;
     437          16 :     lclContext *ctx = (lclContext *) AH->formatData;
     438          16 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     439             :     CompressFileHandle *CFH;
     440             :     char        tocfname[MAXPGPATH];
     441             :     char        line[MAXPGPATH];
     442             : 
     443          16 :     StartRestoreLOs(AH);
     444             : 
     445             :     /*
     446             :      * Note: before archive v16, there was always only one BLOBS TOC entry,
     447             :      * now there can be multiple.  We don't need to worry what version we are
     448             :      * reading though, because tctx->filename should be correct either way.
     449             :      */
     450          16 :     setFilePath(AH, tocfname, tctx->filename);
     451             : 
     452          16 :     CFH = ctx->LOsTocFH = InitDiscoverCompressFileHandle(tocfname, PG_BINARY_R);
     453             : 
     454          16 :     if (ctx->LOsTocFH == NULL)
     455           0 :         pg_fatal("could not open large object TOC file \"%s\" for input: %m",
     456             :                  tocfname);
     457             : 
     458             :     /* Read the LOs TOC file line-by-line, and process each LO */
     459          32 :     while ((CFH->gets_func(line, MAXPGPATH, CFH)) != NULL)
     460             :     {
     461             :         char        lofname[MAXPGPATH + 1];
     462             :         char        path[MAXPGPATH];
     463             : 
     464             :         /* Can't overflow because line and lofname are the same length */
     465          16 :         if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, lofname) != 2)
     466           0 :             pg_fatal("invalid line in large object TOC file \"%s\": \"%s\"",
     467             :                      tocfname, line);
     468             : 
     469          16 :         StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
     470          16 :         snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, lofname);
     471          16 :         _PrintFileData(AH, path);
     472          16 :         EndRestoreLO(AH, oid);
     473             :     }
     474          16 :     if (!CFH->eof_func(CFH))
     475           0 :         pg_fatal("error reading large object TOC file \"%s\"",
     476             :                  tocfname);
     477             : 
     478          16 :     if (!EndCompressFileHandle(ctx->LOsTocFH))
     479           0 :         pg_fatal("could not close large object TOC file \"%s\": %m",
     480             :                  tocfname);
     481             : 
     482          16 :     ctx->LOsTocFH = NULL;
     483             : 
     484          16 :     EndRestoreLOs(AH);
     485          16 : }
     486             : 
     487             : 
     488             : /*
     489             :  * Write a byte of data to the archive.
     490             :  * Called by the archiver to do integer & byte output to the archive.
     491             :  * These routines are only used to read & write the headers & TOC.
     492             :  */
     493             : static int
     494      258520 : _WriteByte(ArchiveHandle *AH, const int i)
     495             : {
     496      258520 :     unsigned char c = (unsigned char) i;
     497      258520 :     lclContext *ctx = (lclContext *) AH->formatData;
     498      258520 :     CompressFileHandle *CFH = ctx->dataFH;
     499             : 
     500      258520 :     errno = 0;
     501      258520 :     if (!CFH->write_func(&c, 1, CFH))
     502             :     {
     503             :         /* if write didn't set errno, assume problem is no disk space */
     504           0 :         if (errno == 0)
     505           0 :             errno = ENOSPC;
     506           0 :         pg_fatal("could not write to output file: %s",
     507             :                  CFH->get_error_func(CFH));
     508             :     }
     509             : 
     510      258520 :     return 1;
     511             : }
     512             : 
     513             : /*
     514             :  * Read a byte of data from the archive.
     515             :  * Called by the archiver to read bytes & integers from the archive.
     516             :  * These routines are only used to read & write headers & TOC.
     517             :  * EOF should be treated as a fatal error.
     518             :  */
     519             : static int
     520      315568 : _ReadByte(ArchiveHandle *AH)
     521             : {
     522      315568 :     lclContext *ctx = (lclContext *) AH->formatData;
     523      315568 :     CompressFileHandle *CFH = ctx->dataFH;
     524             : 
     525      315568 :     return CFH->getc_func(CFH);
     526             : }
     527             : 
     528             : /*
     529             :  * Write a buffer of data to the archive.
     530             :  * Called by the archiver to write a block of bytes to the TOC or a data file.
     531             :  */
     532             : static void
     533       29610 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
     534             : {
     535       29610 :     lclContext *ctx = (lclContext *) AH->formatData;
     536       29610 :     CompressFileHandle *CFH = ctx->dataFH;
     537             : 
     538       29610 :     errno = 0;
     539       29610 :     if (!CFH->write_func(buf, len, CFH))
     540             :     {
     541             :         /* if write didn't set errno, assume problem is no disk space */
     542           0 :         if (errno == 0)
     543           0 :             errno = ENOSPC;
     544           0 :         pg_fatal("could not write to output file: %s",
     545             :                  CFH->get_error_func(CFH));
     546             :     }
     547       29610 : }
     548             : 
     549             : /*
     550             :  * Read a block of bytes from the archive.
     551             :  *
     552             :  * Called by the archiver to read a block of bytes from the archive
     553             :  */
     554             : static void
     555       36130 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
     556             : {
     557       36130 :     lclContext *ctx = (lclContext *) AH->formatData;
     558       36130 :     CompressFileHandle *CFH = ctx->dataFH;
     559             : 
     560             :     /*
     561             :      * If there was an I/O error, we already exited in readF(), so here we
     562             :      * exit on short reads.
     563             :      */
     564       36130 :     if (!CFH->read_func(buf, len, NULL, CFH))
     565           0 :         pg_fatal("could not read from input file: end of file");
     566       36130 : }
     567             : 
     568             : /*
     569             :  * Close the archive.
     570             :  *
     571             :  * When writing the archive, this is the routine that actually starts
     572             :  * the process of saving it to files. No data should be written prior
     573             :  * to this point, since the user could sort the TOC after creating it.
     574             :  *
     575             :  * If an archive is to be written, this routine must call:
     576             :  *      WriteHead           to save the archive header
     577             :  *      WriteToc            to save the TOC entries
     578             :  *      WriteDataChunks     to save all data & LOs.
     579             :  */
     580             : static void
     581          44 : _CloseArchive(ArchiveHandle *AH)
     582             : {
     583          44 :     lclContext *ctx = (lclContext *) AH->formatData;
     584             : 
     585          44 :     if (AH->mode == archModeWrite)
     586             :     {
     587             :         CompressFileHandle *tocFH;
     588          20 :         pg_compress_specification compression_spec = {0};
     589             :         char        fname[MAXPGPATH];
     590             : 
     591          20 :         setFilePath(AH, fname, "toc.dat");
     592             : 
     593             :         /* this will actually fork the processes for a parallel backup */
     594          20 :         ctx->pstate = ParallelBackupStart(AH);
     595             : 
     596             :         /* The TOC is always created uncompressed */
     597          20 :         compression_spec.algorithm = PG_COMPRESSION_NONE;
     598          20 :         tocFH = InitCompressFileHandle(compression_spec);
     599          20 :         if (!tocFH->open_write_func(fname, PG_BINARY_W, tocFH))
     600           0 :             pg_fatal("could not open output file \"%s\": %m", fname);
     601          20 :         ctx->dataFH = tocFH;
     602             : 
     603             :         /*
     604             :          * Write 'tar' in the format field of the toc.dat file. The directory
     605             :          * is compatible with 'tar', so there's no point having a different
     606             :          * format code for it.
     607             :          */
     608          20 :         AH->format = archTar;
     609          20 :         WriteHead(AH);
     610          20 :         AH->format = archDirectory;
     611          20 :         WriteToc(AH);
     612          20 :         if (!EndCompressFileHandle(tocFH))
     613           0 :             pg_fatal("could not close TOC file: %m");
     614          20 :         WriteDataChunks(AH, ctx->pstate);
     615             : 
     616          20 :         ParallelBackupEnd(AH, ctx->pstate);
     617             : 
     618             :         /*
     619             :          * In directory mode, there is no need to sync all the entries
     620             :          * individually. Just recurse once through all the files generated.
     621             :          */
     622          20 :         if (AH->dosync)
     623           8 :             sync_dir_recurse(ctx->directory, AH->sync_method);
     624             :     }
     625          44 :     AH->FH = NULL;
     626          44 : }
     627             : 
     628             : /*
     629             :  * Reopen the archive's file handle.
     630             :  */
     631             : static void
     632          28 : _ReopenArchive(ArchiveHandle *AH)
     633             : {
     634             :     /*
     635             :      * Our TOC is in memory, our data files are opened by each child anyway as
     636             :      * they are separate. We support reopening the archive by just doing
     637             :      * nothing.
     638             :      */
     639          28 : }
     640             : 
     641             : /*
     642             :  * LO support
     643             :  */
     644             : 
     645             : /*
     646             :  * Called by the archiver when starting to save BLOB DATA (not schema).
     647             :  * It is called just prior to the dumper's DataDumper routine.
     648             :  *
     649             :  * We open the large object TOC file here, so that we can append a line to
     650             :  * it for each LO.
     651             :  */
     652             : static void
     653          16 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
     654             : {
     655          16 :     lclContext *ctx = (lclContext *) AH->formatData;
     656          16 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     657          16 :     pg_compress_specification compression_spec = {0};
     658             :     char        fname[MAXPGPATH];
     659             : 
     660          16 :     setFilePath(AH, fname, tctx->filename);
     661             : 
     662             :     /* The LO TOC file is never compressed */
     663          16 :     compression_spec.algorithm = PG_COMPRESSION_NONE;
     664          16 :     ctx->LOsTocFH = InitCompressFileHandle(compression_spec);
     665          16 :     if (!ctx->LOsTocFH->open_write_func(fname, "ab", ctx->LOsTocFH))
     666           0 :         pg_fatal("could not open output file \"%s\": %m", fname);
     667          16 : }
     668             : 
     669             : /*
     670             :  * Called by the archiver when we're about to start dumping a LO.
     671             :  *
     672             :  * We create a file to write the LO to.
     673             :  */
     674             : static void
     675          16 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
     676             : {
     677          16 :     lclContext *ctx = (lclContext *) AH->formatData;
     678             :     char        fname[MAXPGPATH];
     679             : 
     680          16 :     snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
     681             : 
     682          16 :     ctx->dataFH = InitCompressFileHandle(AH->compression_spec);
     683          16 :     if (!ctx->dataFH->open_write_func(fname, PG_BINARY_W, ctx->dataFH))
     684           0 :         pg_fatal("could not open output file \"%s\": %m", fname);
     685          16 : }
     686             : 
     687             : /*
     688             :  * Called by the archiver when the dumper is finished writing a LO.
     689             :  *
     690             :  * We close the LO file and write an entry to the LO TOC file for it.
     691             :  */
     692             : static void
     693          16 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
     694             : {
     695          16 :     lclContext *ctx = (lclContext *) AH->formatData;
     696          16 :     CompressFileHandle *CFH = ctx->LOsTocFH;
     697             :     char        buf[50];
     698             :     int         len;
     699             : 
     700             :     /* Close the BLOB data file itself */
     701          16 :     if (!EndCompressFileHandle(ctx->dataFH))
     702           0 :         pg_fatal("could not close LO data file: %m");
     703          16 :     ctx->dataFH = NULL;
     704             : 
     705             :     /* register the LO in blobs_NNN.toc */
     706          16 :     len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
     707          16 :     if (!CFH->write_func(buf, len, CFH))
     708             :     {
     709             :         /* if write didn't set errno, assume problem is no disk space */
     710           0 :         if (errno == 0)
     711           0 :             errno = ENOSPC;
     712           0 :         pg_fatal("could not write to LOs TOC file: %s",
     713             :                  CFH->get_error_func(CFH));
     714             :     }
     715          16 : }
     716             : 
     717             : /*
     718             :  * Called by the archiver when finishing saving BLOB DATA.
     719             :  *
     720             :  * We close the LOs TOC file.
     721             :  */
     722             : static void
     723          16 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
     724             : {
     725          16 :     lclContext *ctx = (lclContext *) AH->formatData;
     726             : 
     727          16 :     if (!EndCompressFileHandle(ctx->LOsTocFH))
     728           0 :         pg_fatal("could not close LOs TOC file: %m");
     729          16 :     ctx->LOsTocFH = NULL;
     730          16 : }
     731             : 
     732             : /*
     733             :  * Gets a relative file name and prepends the output directory, writing the
     734             :  * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
     735             :  * big. Can't use a static char[MAXPGPATH] inside the function because we run
     736             :  * multithreaded on Windows.
     737             :  */
     738             : static void
     739         690 : setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
     740             : {
     741         690 :     lclContext *ctx = (lclContext *) AH->formatData;
     742             :     char       *dname;
     743             : 
     744         690 :     dname = ctx->directory;
     745             : 
     746         690 :     if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
     747           0 :         pg_fatal("file name too long: \"%s\"", dname);
     748             : 
     749         690 :     strcpy(buf, dname);
     750         690 :     strcat(buf, "/");
     751         690 :     strcat(buf, relativeFilename);
     752         690 : }
     753             : 
     754             : /*
     755             :  * Prepare for parallel restore.
     756             :  *
     757             :  * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
     758             :  * TOC entries' dataLength fields with appropriate values to guide the
     759             :  * ordering of restore jobs.  The source of said data is format-dependent,
     760             :  * as is the exact meaning of the values.
     761             :  *
     762             :  * A format module might also choose to do other setup here.
     763             :  */
     764             : static void
     765           8 : _PrepParallelRestore(ArchiveHandle *AH)
     766             : {
     767             :     TocEntry   *te;
     768             : 
     769         200 :     for (te = AH->toc->next; te != AH->toc; te = te->next)
     770             :     {
     771         192 :         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     772             :         char        fname[MAXPGPATH];
     773             :         struct stat st;
     774             : 
     775             :         /*
     776             :          * A dumpable object has set tctx->filename, any other object has not.
     777             :          * (see _ArchiveEntry).
     778             :          */
     779         192 :         if (tctx->filename == NULL)
     780         160 :             continue;
     781             : 
     782             :         /* We may ignore items not due to be restored */
     783          32 :         if ((te->reqs & REQ_DATA) == 0)
     784           0 :             continue;
     785             : 
     786             :         /*
     787             :          * Stat the file and, if successful, put its size in dataLength.  When
     788             :          * using compression, the physical file size might not be a very good
     789             :          * guide to the amount of work involved in restoring the file, but we
     790             :          * only need an approximate indicator of that.
     791             :          */
     792          32 :         setFilePath(AH, fname, tctx->filename);
     793             : 
     794          32 :         if (stat(fname, &st) == 0)
     795           0 :             te->dataLength = st.st_size;
     796          32 :         else if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
     797             :         {
     798          32 :             if (AH->compression_spec.algorithm == PG_COMPRESSION_GZIP)
     799          32 :                 strlcat(fname, ".gz", sizeof(fname));
     800           0 :             else if (AH->compression_spec.algorithm == PG_COMPRESSION_LZ4)
     801           0 :                 strlcat(fname, ".lz4", sizeof(fname));
     802           0 :             else if (AH->compression_spec.algorithm == PG_COMPRESSION_ZSTD)
     803           0 :                 strlcat(fname, ".zst", sizeof(fname));
     804             : 
     805          32 :             if (stat(fname, &st) == 0)
     806          32 :                 te->dataLength = st.st_size;
     807             :         }
     808             : 
     809             :         /*
     810             :          * If this is a BLOBS entry, what we stat'd was blobs_NNN.toc, which
     811             :          * most likely is a lot smaller than the actual blob data.  We don't
     812             :          * have a cheap way to estimate how much smaller, but fortunately it
     813             :          * doesn't matter too much as long as we get the LOs processed
     814             :          * reasonably early.  Arbitrarily scale up by a factor of 1K.
     815             :          */
     816          32 :         if (strcmp(te->desc, "BLOBS") == 0)
     817           0 :             te->dataLength *= 1024;
     818             :     }
     819           8 : }
     820             : 
     821             : /*
     822             :  * Clone format-specific fields during parallel restoration.
     823             :  */
     824             : static void
     825          52 : _Clone(ArchiveHandle *AH)
     826             : {
     827          52 :     lclContext *ctx = (lclContext *) AH->formatData;
     828             : 
     829          52 :     AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
     830          52 :     memcpy(AH->formatData, ctx, sizeof(lclContext));
     831          52 :     ctx = (lclContext *) AH->formatData;
     832             : 
     833             :     /*
     834             :      * TOC-entry-local state isn't an issue because any one TOC entry is
     835             :      * touched by just one worker child.
     836             :      */
     837             : 
     838             :     /*
     839             :      * We also don't copy the ParallelState pointer (pstate), only the leader
     840             :      * process ever writes to it.
     841             :      */
     842          52 : }
     843             : 
     844             : static void
     845          52 : _DeClone(ArchiveHandle *AH)
     846             : {
     847          52 :     lclContext *ctx = (lclContext *) AH->formatData;
     848             : 
     849          52 :     free(ctx);
     850          52 : }
     851             : 
     852             : /*
     853             :  * This function is executed in the child of a parallel backup for a
     854             :  * directory-format archive and dumps the actual data for one TOC entry.
     855             :  */
     856             : static int
     857         236 : _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
     858             : {
     859             :     /*
     860             :      * This function returns void. We either fail and die horribly or
     861             :      * succeed... A failure will be detected by the parent when the child dies
     862             :      * unexpectedly.
     863             :      */
     864         236 :     WriteDataChunksForTocEntry(AH, te);
     865             : 
     866         236 :     return 0;
     867             : }
     868             : 
     869             : /*
     870             :  * This function is executed in the child of a parallel restore from a
     871             :  * directory-format archive and restores the actual data for one TOC entry.
     872             :  */
     873             : static int
     874          92 : _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
     875             : {
     876          92 :     return parallel_restore(AH, te);
     877             : }

Generated by: LCOV version 1.14