LCOV - code coverage report
Current view: top level - src/bin/pg_dump - pg_backup_directory.c (source / functions) Hit Total Coverage
Test: PostgreSQL 13devel Lines: 224 262 85.5 %
Date: 2019-11-15 23:07:02 Functions: 27 27 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_backup_directory.c
       4             :  *
       5             :  *  A directory format dump is a directory, which contains a "toc.dat" file
       6             :  *  for the TOC, and a separate file for each data entry, named "<oid>.dat".
       7             :  *  Large objects (BLOBs) are stored in separate files named "blob_<uid>.dat",
       8             :  *  and there's a plain-text TOC file for them called "blobs.toc". If
       9             :  *  compression is used, each data file is individually compressed and the
      10             :  *  ".gz" suffix is added to the filenames. The TOC files are never
      11             :  *  compressed by pg_dump, however they are accepted with the .gz suffix too,
      12             :  *  in case the user has manually compressed them with 'gzip'.
      13             :  *
      14             :  *  NOTE: This format is identical to the files written in the tar file in
      15             :  *  the 'tar' format, except that we don't write the restore.sql file (TODO),
      16             :  *  and the tar format doesn't support compression. Please keep the formats in
      17             :  *  sync.
      18             :  *
      19             :  *
      20             :  *  Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
      21             :  *  Portions Copyright (c) 1994, Regents of the University of California
      22             :  *  Portions Copyright (c) 2000, Philip Warner
      23             :  *
      24             :  *  Rights are granted to use this software in any way so long
      25             :  *  as this notice is not removed.
      26             :  *
      27             :  *  The author is not responsible for loss or damages that may
      28             :  *  result from its use.
      29             :  *
      30             :  * IDENTIFICATION
      31             :  *      src/bin/pg_dump/pg_backup_directory.c
      32             :  *
      33             :  *-------------------------------------------------------------------------
      34             :  */
      35             : #include "postgres_fe.h"
      36             : 
      37             : #include <dirent.h>
      38             : #include <sys/stat.h>
      39             : 
      40             : #include "common/file_utils.h"
      41             : #include "compress_io.h"
      42             : #include "parallel.h"
      43             : #include "pg_backup_utils.h"
      44             : 
      45             : typedef struct
      46             : {
      47             :     /*
      48             :      * Our archive location. This is basically what the user specified as his
      49             :      * backup file but of course here it is a directory.
      50             :      */
      51             :     char       *directory;
      52             : 
      53             :     cfp        *dataFH;         /* currently open data file */
      54             : 
      55             :     cfp        *blobsTocFH;     /* file handle for blobs.toc */
      56             :     ParallelState *pstate;      /* for parallel backup / restore */
      57             : } lclContext;
      58             : 
      59             : typedef struct
      60             : {
      61             :     char       *filename;       /* filename excluding the directory (basename) */
      62             : } lclTocEntry;
      63             : 
      64             : /* prototypes for private functions */
      65             : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
      66             : static void _StartData(ArchiveHandle *AH, TocEntry *te);
      67             : static void _EndData(ArchiveHandle *AH, TocEntry *te);
      68             : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
      69             : static int  _WriteByte(ArchiveHandle *AH, const int i);
      70             : static int  _ReadByte(ArchiveHandle *);
      71             : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
      72             : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
      73             : static void _CloseArchive(ArchiveHandle *AH);
      74             : static void _ReopenArchive(ArchiveHandle *AH);
      75             : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
      76             : 
      77             : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
      78             : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
      79             : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
      80             : 
      81             : static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
      82             : static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
      83             : static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
      84             : static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
      85             : static void _LoadBlobs(ArchiveHandle *AH);
      86             : 
      87             : static void _PrepParallelRestore(ArchiveHandle *AH);
      88             : static void _Clone(ArchiveHandle *AH);
      89             : static void _DeClone(ArchiveHandle *AH);
      90             : 
      91             : static int  _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
      92             : static int  _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
      93             : 
      94             : static void setFilePath(ArchiveHandle *AH, char *buf,
      95             :                         const char *relativeFilename);
      96             : 
      97             : /*
      98             :  *  Init routine required by ALL formats. This is a global routine
      99             :  *  and should be declared in pg_backup_archiver.h
     100             :  *
     101             :  *  Its task is to create any extra archive context (using AH->formatData),
     102             :  *  and to initialize the supported function pointers.
     103             :  *
     104             :  *  It should also prepare whatever its input source is for reading/writing,
     105             :  *  and in the case of a read mode connection, it should load the Header & TOC.
     106             :  */
     107             : void
     108          26 : InitArchiveFmt_Directory(ArchiveHandle *AH)
     109             : {
     110             :     lclContext *ctx;
     111             : 
     112             :     /* Assuming static functions, this can be copied for each format. */
     113          26 :     AH->ArchiveEntryPtr = _ArchiveEntry;
     114          26 :     AH->StartDataPtr = _StartData;
     115          26 :     AH->WriteDataPtr = _WriteData;
     116          26 :     AH->EndDataPtr = _EndData;
     117          26 :     AH->WriteBytePtr = _WriteByte;
     118          26 :     AH->ReadBytePtr = _ReadByte;
     119          26 :     AH->WriteBufPtr = _WriteBuf;
     120          26 :     AH->ReadBufPtr = _ReadBuf;
     121          26 :     AH->ClosePtr = _CloseArchive;
     122          26 :     AH->ReopenPtr = _ReopenArchive;
     123          26 :     AH->PrintTocDataPtr = _PrintTocData;
     124          26 :     AH->ReadExtraTocPtr = _ReadExtraToc;
     125          26 :     AH->WriteExtraTocPtr = _WriteExtraToc;
     126          26 :     AH->PrintExtraTocPtr = _PrintExtraToc;
     127             : 
     128          26 :     AH->StartBlobsPtr = _StartBlobs;
     129          26 :     AH->StartBlobPtr = _StartBlob;
     130          26 :     AH->EndBlobPtr = _EndBlob;
     131          26 :     AH->EndBlobsPtr = _EndBlobs;
     132             : 
     133          26 :     AH->PrepParallelRestorePtr = _PrepParallelRestore;
     134          26 :     AH->ClonePtr = _Clone;
     135          26 :     AH->DeClonePtr = _DeClone;
     136             : 
     137          26 :     AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
     138          26 :     AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
     139             : 
     140             :     /* Set up our private context */
     141          26 :     ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
     142          26 :     AH->formatData = (void *) ctx;
     143             : 
     144          26 :     ctx->dataFH = NULL;
     145          26 :     ctx->blobsTocFH = NULL;
     146             : 
     147             :     /* Initialize LO buffering */
     148          26 :     AH->lo_buf_size = LOBBUFSIZE;
     149          26 :     AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
     150             : 
     151             :     /*
     152             :      * Now open the TOC file
     153             :      */
     154             : 
     155          26 :     if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
     156           0 :         fatal("no output directory specified");
     157             : 
     158          26 :     ctx->directory = AH->fSpec;
     159             : 
     160          26 :     if (AH->mode == archModeWrite)
     161             :     {
     162             :         struct stat st;
     163          12 :         bool        is_empty = false;
     164             : 
     165             :         /* we accept an empty existing directory */
     166          12 :         if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
     167             :         {
     168           0 :             DIR        *dir = opendir(ctx->directory);
     169             : 
     170           0 :             if (dir)
     171             :             {
     172             :                 struct dirent *d;
     173             : 
     174           0 :                 is_empty = true;
     175           0 :                 while (errno = 0, (d = readdir(dir)))
     176             :                 {
     177           0 :                     if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
     178             :                     {
     179           0 :                         is_empty = false;
     180           0 :                         break;
     181             :                     }
     182             :                 }
     183             : 
     184           0 :                 if (errno)
     185           0 :                     fatal("could not read directory \"%s\": %m",
     186             :                           ctx->directory);
     187             : 
     188           0 :                 if (closedir(dir))
     189           0 :                     fatal("could not close directory \"%s\": %m",
     190             :                           ctx->directory);
     191             :             }
     192             :         }
     193             : 
     194          12 :         if (!is_empty && mkdir(ctx->directory, 0700) < 0)
     195           0 :             fatal("could not create directory \"%s\": %m",
     196             :                   ctx->directory);
     197             :     }
     198             :     else
     199             :     {                           /* Read Mode */
     200             :         char        fname[MAXPGPATH];
     201             :         cfp        *tocFH;
     202             : 
     203          14 :         setFilePath(AH, fname, "toc.dat");
     204             : 
     205          14 :         tocFH = cfopen_read(fname, PG_BINARY_R);
     206          14 :         if (tocFH == NULL)
     207           0 :             fatal("could not open input file \"%s\": %m", fname);
     208             : 
     209          14 :         ctx->dataFH = tocFH;
     210             : 
     211             :         /*
     212             :          * The TOC of a directory format dump shares the format code of the
     213             :          * tar format.
     214             :          */
     215          14 :         AH->format = archTar;
     216          14 :         ReadHead(AH);
     217          14 :         AH->format = archDirectory;
     218          14 :         ReadToc(AH);
     219             : 
     220             :         /* Nothing else in the file, so close it again... */
     221          14 :         if (cfclose(tocFH) != 0)
     222           0 :             fatal("could not close TOC file: %m");
     223          14 :         ctx->dataFH = NULL;
     224             :     }
     225          26 : }
     226             : 
     227             : /*
     228             :  * Called by the Archiver when the dumper creates a new TOC entry.
     229             :  *
     230             :  * We determine the filename for this entry.
     231             : */
     232             : static void
     233         890 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
     234             : {
     235             :     lclTocEntry *tctx;
     236             :     char        fn[MAXPGPATH];
     237             : 
     238         890 :     tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     239         890 :     if (strcmp(te->desc, "BLOBS") == 0)
     240           4 :         tctx->filename = pg_strdup("blobs.toc");
     241         886 :     else if (te->dataDumper)
     242             :     {
     243          68 :         snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
     244          68 :         tctx->filename = pg_strdup(fn);
     245             :     }
     246             :     else
     247         818 :         tctx->filename = NULL;
     248             : 
     249         890 :     te->formatData = (void *) tctx;
     250         890 : }
     251             : 
     252             : /*
     253             :  * Called by the Archiver to save any extra format-related TOC entry
     254             :  * data.
     255             :  *
     256             :  * Use the Archiver routines to write data - they are non-endian, and
     257             :  * maintain other important file information.
     258             :  */
     259             : static void
     260         890 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
     261             : {
     262         890 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     263             : 
     264             :     /*
     265             :      * A dumpable object has set tctx->filename, any other object has not.
     266             :      * (see _ArchiveEntry).
     267             :      */
     268         890 :     if (tctx->filename)
     269          72 :         WriteStr(AH, tctx->filename);
     270             :     else
     271         818 :         WriteStr(AH, "");
     272         890 : }
     273             : 
     274             : /*
     275             :  * Called by the Archiver to read any extra format-related TOC data.
     276             :  *
     277             :  * Needs to match the order defined in _WriteExtraToc, and should also
     278             :  * use the Archiver input routines.
     279             :  */
     280             : static void
     281         902 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
     282             : {
     283         902 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     284             : 
     285         902 :     if (tctx == NULL)
     286             :     {
     287         902 :         tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     288         902 :         te->formatData = (void *) tctx;
     289             :     }
     290             : 
     291         902 :     tctx->filename = ReadStr(AH);
     292         902 :     if (strlen(tctx->filename) == 0)
     293             :     {
     294         828 :         free(tctx->filename);
     295         828 :         tctx->filename = NULL;
     296             :     }
     297         902 : }
     298             : 
     299             : /*
     300             :  * Called by the Archiver when restoring an archive to output a comment
     301             :  * that includes useful information about the TOC entry.
     302             :  */
     303             : static void
     304         828 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
     305             : {
     306         828 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     307             : 
     308         828 :     if (AH->public.verbose && tctx->filename)
     309           0 :         ahprintf(AH, "-- File: %s\n", tctx->filename);
     310         828 : }
     311             : 
     312             : /*
     313             :  * Called by the archiver when saving TABLE DATA (not schema). This routine
     314             :  * should save whatever format-specific information is needed to read
     315             :  * the archive back.
     316             :  *
     317             :  * It is called just prior to the dumper's 'DataDumper' routine being called.
     318             :  *
     319             :  * We create the data file for writing.
     320             :  */
     321             : static void
     322          68 : _StartData(ArchiveHandle *AH, TocEntry *te)
     323             : {
     324          68 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     325          68 :     lclContext *ctx = (lclContext *) AH->formatData;
     326             :     char        fname[MAXPGPATH];
     327             : 
     328          68 :     setFilePath(AH, fname, tctx->filename);
     329             : 
     330          68 :     ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
     331          68 :     if (ctx->dataFH == NULL)
     332           0 :         fatal("could not open output file \"%s\": %m", fname);
     333          68 : }
     334             : 
     335             : /*
     336             :  * Called by archiver when dumper calls WriteData. This routine is
     337             :  * called for both BLOB and TABLE data; it is the responsibility of
     338             :  * the format to manage each kind of data using StartBlob/StartData.
     339             :  *
     340             :  * It should only be called from within a DataDumper routine.
     341             :  *
     342             :  * We write the data to the open data file.
     343             :  */
     344             : static void
     345         184 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
     346             : {
     347         184 :     lclContext *ctx = (lclContext *) AH->formatData;
     348             : 
     349         184 :     if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
     350           0 :         fatal("could not write to output file: %s",
     351             :               get_cfp_error(ctx->dataFH));
     352             : 
     353             : 
     354         184 :     return;
     355             : }
     356             : 
     357             : /*
     358             :  * Called by the archiver when a dumper's 'DataDumper' routine has
     359             :  * finished.
     360             :  *
     361             :  * We close the data file.
     362             :  */
     363             : static void
     364          68 : _EndData(ArchiveHandle *AH, TocEntry *te)
     365             : {
     366          68 :     lclContext *ctx = (lclContext *) AH->formatData;
     367             : 
     368             :     /* Close the file */
     369          68 :     cfclose(ctx->dataFH);
     370             : 
     371          68 :     ctx->dataFH = NULL;
     372          68 : }
     373             : 
     374             : /*
     375             :  * Print data for a given file (can be a BLOB as well)
     376             :  */
     377             : static void
     378          74 : _PrintFileData(ArchiveHandle *AH, char *filename)
     379             : {
     380             :     size_t      cnt;
     381             :     char       *buf;
     382             :     size_t      buflen;
     383             :     cfp        *cfp;
     384             : 
     385          74 :     if (!filename)
     386           0 :         return;
     387             : 
     388          74 :     cfp = cfopen_read(filename, PG_BINARY_R);
     389             : 
     390          74 :     if (!cfp)
     391           0 :         fatal("could not open input file \"%s\": %m", filename);
     392             : 
     393          74 :     buf = pg_malloc(ZLIB_OUT_SIZE);
     394          74 :     buflen = ZLIB_OUT_SIZE;
     395             : 
     396         222 :     while ((cnt = cfread(buf, buflen, cfp)))
     397             :     {
     398          74 :         ahwrite(buf, 1, cnt, AH);
     399             :     }
     400             : 
     401          74 :     free(buf);
     402          74 :     if (cfclose(cfp) !=0)
     403           0 :         fatal("could not close data file: %m");
     404             : }
     405             : 
     406             : /*
     407             :  * Print data for a given TOC entry
     408             : */
     409             : static void
     410          74 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
     411             : {
     412          74 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     413             : 
     414          74 :     if (!tctx->filename)
     415           0 :         return;
     416             : 
     417          74 :     if (strcmp(te->desc, "BLOBS") == 0)
     418           4 :         _LoadBlobs(AH);
     419             :     else
     420             :     {
     421             :         char        fname[MAXPGPATH];
     422             : 
     423          70 :         setFilePath(AH, fname, tctx->filename);
     424          70 :         _PrintFileData(AH, fname);
     425             :     }
     426             : }
     427             : 
     428             : static void
     429           4 : _LoadBlobs(ArchiveHandle *AH)
     430             : {
     431             :     Oid         oid;
     432           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     433             :     char        fname[MAXPGPATH];
     434             :     char        line[MAXPGPATH];
     435             : 
     436           4 :     StartRestoreBlobs(AH);
     437             : 
     438           4 :     setFilePath(AH, fname, "blobs.toc");
     439             : 
     440           4 :     ctx->blobsTocFH = cfopen_read(fname, PG_BINARY_R);
     441             : 
     442           4 :     if (ctx->blobsTocFH == NULL)
     443           0 :         fatal("could not open large object TOC file \"%s\" for input: %m",
     444             :               fname);
     445             : 
     446             :     /* Read the blobs TOC file line-by-line, and process each blob */
     447          12 :     while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
     448             :     {
     449             :         char        fname[MAXPGPATH];
     450             :         char        path[MAXPGPATH];
     451             : 
     452             :         /* Can't overflow because line and fname are the same length. */
     453           4 :         if (sscanf(line, "%u %s\n", &oid, fname) != 2)
     454           0 :             fatal("invalid line in large object TOC file \"%s\": \"%s\"",
     455             :                   fname, line);
     456             : 
     457           4 :         StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
     458           4 :         snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, fname);
     459           4 :         _PrintFileData(AH, path);
     460           4 :         EndRestoreBlob(AH, oid);
     461             :     }
     462           4 :     if (!cfeof(ctx->blobsTocFH))
     463           0 :         fatal("error reading large object TOC file \"%s\"",
     464             :               fname);
     465             : 
     466           4 :     if (cfclose(ctx->blobsTocFH) != 0)
     467           0 :         fatal("could not close large object TOC file \"%s\": %m",
     468             :               fname);
     469             : 
     470           4 :     ctx->blobsTocFH = NULL;
     471             : 
     472           4 :     EndRestoreBlobs(AH);
     473           4 : }
     474             : 
     475             : 
     476             : /*
     477             :  * Write a byte of data to the archive.
     478             :  * Called by the archiver to do integer & byte output to the archive.
     479             :  * These routines are only used to read & write the headers & TOC.
     480             :  */
     481             : static int
     482       81702 : _WriteByte(ArchiveHandle *AH, const int i)
     483             : {
     484       81702 :     unsigned char c = (unsigned char) i;
     485       81702 :     lclContext *ctx = (lclContext *) AH->formatData;
     486             : 
     487       81702 :     if (cfwrite(&c, 1, ctx->dataFH) != 1)
     488           0 :         fatal("could not write to output file: %s",
     489             :               get_cfp_error(ctx->dataFH));
     490             : 
     491       81702 :     return 1;
     492             : }
     493             : 
     494             : /*
     495             :  * Read a byte of data from the archive.
     496             :  * Called by the archiver to read bytes & integers from the archive.
     497             :  * These routines are only used to read & write headers & TOC.
     498             :  * EOF should be treated as a fatal error.
     499             :  */
     500             : static int
     501       82864 : _ReadByte(ArchiveHandle *AH)
     502             : {
     503       82864 :     lclContext *ctx = (lclContext *) AH->formatData;
     504             : 
     505       82864 :     return cfgetc(ctx->dataFH);
     506             : }
     507             : 
     508             : /*
     509             :  * Write a buffer of data to the archive.
     510             :  * Called by the archiver to write a block of bytes to the TOC or a data file.
     511             :  */
     512             : static void
     513        9488 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
     514             : {
     515        9488 :     lclContext *ctx = (lclContext *) AH->formatData;
     516             : 
     517        9488 :     if (cfwrite(buf, len, ctx->dataFH) != len)
     518           0 :         fatal("could not write to output file: %s",
     519             :               get_cfp_error(ctx->dataFH));
     520             : 
     521        9488 :     return;
     522             : }
     523             : 
     524             : /*
     525             :  * Read a block of bytes from the archive.
     526             :  *
     527             :  * Called by the archiver to read a block of bytes from the archive
     528             :  */
     529             : static void
     530        9600 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
     531             : {
     532        9600 :     lclContext *ctx = (lclContext *) AH->formatData;
     533             : 
     534             :     /*
     535             :      * If there was an I/O error, we already exited in cfread(), so here we
     536             :      * exit on short reads.
     537             :      */
     538        9600 :     if (cfread(buf, len, ctx->dataFH) != len)
     539           0 :         fatal("could not read from input file: end of file");
     540             : 
     541        9600 :     return;
     542             : }
     543             : 
     544             : /*
     545             :  * Close the archive.
     546             :  *
     547             :  * When writing the archive, this is the routine that actually starts
     548             :  * the process of saving it to files. No data should be written prior
     549             :  * to this point, since the user could sort the TOC after creating it.
     550             :  *
     551             :  * If an archive is to be written, this routine must call:
     552             :  *      WriteHead           to save the archive header
     553             :  *      WriteToc            to save the TOC entries
     554             :  *      WriteDataChunks     to save all DATA & BLOBs.
     555             :  */
     556             : static void
     557          26 : _CloseArchive(ArchiveHandle *AH)
     558             : {
     559          26 :     lclContext *ctx = (lclContext *) AH->formatData;
     560             : 
     561          26 :     if (AH->mode == archModeWrite)
     562             :     {
     563             :         cfp        *tocFH;
     564             :         char        fname[MAXPGPATH];
     565             : 
     566          12 :         setFilePath(AH, fname, "toc.dat");
     567             : 
     568             :         /* this will actually fork the processes for a parallel backup */
     569          12 :         ctx->pstate = ParallelBackupStart(AH);
     570             : 
     571             :         /* The TOC is always created uncompressed */
     572          12 :         tocFH = cfopen_write(fname, PG_BINARY_W, 0);
     573          12 :         if (tocFH == NULL)
     574           0 :             fatal("could not open output file \"%s\": %m", fname);
     575          12 :         ctx->dataFH = tocFH;
     576             : 
     577             :         /*
     578             :          * Write 'tar' in the format field of the toc.dat file. The directory
     579             :          * is compatible with 'tar', so there's no point having a different
     580             :          * format code for it.
     581             :          */
     582          12 :         AH->format = archTar;
     583          12 :         WriteHead(AH);
     584          12 :         AH->format = archDirectory;
     585          12 :         WriteToc(AH);
     586          12 :         if (cfclose(tocFH) != 0)
     587           0 :             fatal("could not close TOC file: %m");
     588          12 :         WriteDataChunks(AH, ctx->pstate);
     589             : 
     590          12 :         ParallelBackupEnd(AH, ctx->pstate);
     591             : 
     592             :         /*
     593             :          * In directory mode, there is no need to sync all the entries
     594             :          * individually. Just recurse once through all the files generated.
     595             :          */
     596          12 :         if (AH->dosync)
     597           4 :             fsync_dir_recurse(ctx->directory);
     598             :     }
     599          26 :     AH->FH = NULL;
     600          26 : }
     601             : 
     602             : /*
     603             :  * Reopen the archive's file handle.
     604             :  */
     605             : static void
     606          12 : _ReopenArchive(ArchiveHandle *AH)
     607             : {
     608             :     /*
     609             :      * Our TOC is in memory, our data files are opened by each child anyway as
     610             :      * they are separate. We support reopening the archive by just doing
     611             :      * nothing.
     612             :      */
     613          12 : }
     614             : 
     615             : /*
     616             :  * BLOB support
     617             :  */
     618             : 
     619             : /*
     620             :  * Called by the archiver when starting to save all BLOB DATA (not schema).
     621             :  * It is called just prior to the dumper's DataDumper routine.
     622             :  *
     623             :  * We open the large object TOC file here, so that we can append a line to
     624             :  * it for each blob.
     625             :  */
     626             : static void
     627           4 : _StartBlobs(ArchiveHandle *AH, TocEntry *te)
     628             : {
     629           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     630             :     char        fname[MAXPGPATH];
     631             : 
     632           4 :     setFilePath(AH, fname, "blobs.toc");
     633             : 
     634             :     /* The blob TOC file is never compressed */
     635           4 :     ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
     636           4 :     if (ctx->blobsTocFH == NULL)
     637           0 :         fatal("could not open output file \"%s\": %m", fname);
     638           4 : }
     639             : 
     640             : /*
     641             :  * Called by the archiver when we're about to start dumping a blob.
     642             :  *
     643             :  * We create a file to write the blob to.
     644             :  */
     645             : static void
     646           4 : _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
     647             : {
     648           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     649             :     char        fname[MAXPGPATH];
     650             : 
     651           4 :     snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
     652             : 
     653           4 :     ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
     654             : 
     655           4 :     if (ctx->dataFH == NULL)
     656           0 :         fatal("could not open output file \"%s\": %m", fname);
     657           4 : }
     658             : 
     659             : /*
     660             :  * Called by the archiver when the dumper is finished writing a blob.
     661             :  *
     662             :  * We close the blob file and write an entry to the blob TOC file for it.
     663             :  */
     664             : static void
     665           4 : _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
     666             : {
     667           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     668             :     char        buf[50];
     669             :     int         len;
     670             : 
     671             :     /* Close the BLOB data file itself */
     672           4 :     cfclose(ctx->dataFH);
     673           4 :     ctx->dataFH = NULL;
     674             : 
     675             :     /* register the blob in blobs.toc */
     676           4 :     len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
     677           4 :     if (cfwrite(buf, len, ctx->blobsTocFH) != len)
     678           0 :         fatal("could not write to blobs TOC file");
     679           4 : }
     680             : 
     681             : /*
     682             :  * Called by the archiver when finishing saving all BLOB DATA.
     683             :  *
     684             :  * We close the blobs TOC file.
     685             :  */
     686             : static void
     687           4 : _EndBlobs(ArchiveHandle *AH, TocEntry *te)
     688             : {
     689           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     690             : 
     691           4 :     cfclose(ctx->blobsTocFH);
     692           4 :     ctx->blobsTocFH = NULL;
     693           4 : }
     694             : 
     695             : /*
     696             :  * Gets a relative file name and prepends the output directory, writing the
     697             :  * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
     698             :  * big. Can't use a static char[MAXPGPATH] inside the function because we run
     699             :  * multithreaded on Windows.
     700             :  */
     701             : static void
     702         176 : setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
     703             : {
     704         176 :     lclContext *ctx = (lclContext *) AH->formatData;
     705             :     char       *dname;
     706             : 
     707         176 :     dname = ctx->directory;
     708             : 
     709         176 :     if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
     710           0 :         fatal("file name too long: \"%s\"", dname);
     711             : 
     712         176 :     strcpy(buf, dname);
     713         176 :     strcat(buf, "/");
     714         176 :     strcat(buf, relativeFilename);
     715         176 : }
     716             : 
     717             : /*
     718             :  * Prepare for parallel restore.
     719             :  *
     720             :  * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
     721             :  * TOC entries' dataLength fields with appropriate values to guide the
     722             :  * ordering of restore jobs.  The source of said data is format-dependent,
     723             :  * as is the exact meaning of the values.
     724             :  *
     725             :  * A format module might also choose to do other setup here.
     726             :  */
     727             : static void
     728           4 : _PrepParallelRestore(ArchiveHandle *AH)
     729             : {
     730             :     TocEntry   *te;
     731             : 
     732          28 :     for (te = AH->toc->next; te != AH->toc; te = te->next)
     733             :     {
     734          24 :         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     735             :         char        fname[MAXPGPATH];
     736             :         struct stat st;
     737             : 
     738             :         /*
     739             :          * A dumpable object has set tctx->filename, any other object has not.
     740             :          * (see _ArchiveEntry).
     741             :          */
     742          24 :         if (tctx->filename == NULL)
     743          40 :             continue;
     744             : 
     745             :         /* We may ignore items not due to be restored */
     746           4 :         if ((te->reqs & REQ_DATA) == 0)
     747           0 :             continue;
     748             : 
     749             :         /*
     750             :          * Stat the file and, if successful, put its size in dataLength.  When
     751             :          * using compression, the physical file size might not be a very good
     752             :          * guide to the amount of work involved in restoring the file, but we
     753             :          * only need an approximate indicator of that.
     754             :          */
     755           4 :         setFilePath(AH, fname, tctx->filename);
     756             : 
     757           4 :         if (stat(fname, &st) == 0)
     758           0 :             te->dataLength = st.st_size;
     759             :         else
     760             :         {
     761             :             /* It might be compressed */
     762           4 :             strlcat(fname, ".gz", sizeof(fname));
     763           4 :             if (stat(fname, &st) == 0)
     764           4 :                 te->dataLength = st.st_size;
     765             :         }
     766             : 
     767             :         /*
     768             :          * If this is the BLOBS entry, what we stat'd was blobs.toc, which
     769             :          * most likely is a lot smaller than the actual blob data.  We don't
     770             :          * have a cheap way to estimate how much smaller, but fortunately it
     771             :          * doesn't matter too much as long as we get the blobs processed
     772             :          * reasonably early.  Arbitrarily scale up by a factor of 1K.
     773             :          */
     774           4 :         if (strcmp(te->desc, "BLOBS") == 0)
     775           0 :             te->dataLength *= 1024;
     776             :     }
     777           4 : }
     778             : 
     779             : /*
     780             :  * Clone format-specific fields during parallel restoration.
     781             :  */
     782             : static void
     783          24 : _Clone(ArchiveHandle *AH)
     784             : {
     785          24 :     lclContext *ctx = (lclContext *) AH->formatData;
     786             : 
     787          24 :     AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
     788          24 :     memcpy(AH->formatData, ctx, sizeof(lclContext));
     789          24 :     ctx = (lclContext *) AH->formatData;
     790             : 
     791             :     /*
     792             :      * Note: we do not make a local lo_buf because we expect at most one BLOBS
     793             :      * entry per archive, so no parallelism is possible.  Likewise,
     794             :      * TOC-entry-local state isn't an issue because any one TOC entry is
     795             :      * touched by just one worker child.
     796             :      */
     797             : 
     798             :     /*
     799             :      * We also don't copy the ParallelState pointer (pstate), only the master
     800             :      * process ever writes to it.
     801             :      */
     802          24 : }
     803             : 
     804             : static void
     805          24 : _DeClone(ArchiveHandle *AH)
     806             : {
     807          24 :     lclContext *ctx = (lclContext *) AH->formatData;
     808             : 
     809          24 :     free(ctx);
     810          24 : }
     811             : 
     812             : /*
     813             :  * This function is executed in the child of a parallel backup for a
     814             :  * directory-format archive and dumps the actual data for one TOC entry.
     815             :  */
     816             : static int
     817          38 : _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
     818             : {
     819             :     /*
     820             :      * This function returns void. We either fail and die horribly or
     821             :      * succeed... A failure will be detected by the parent when the child dies
     822             :      * unexpectedly.
     823             :      */
     824          38 :     WriteDataChunksForTocEntry(AH, te);
     825             : 
     826          38 :     return 0;
     827             : }
     828             : 
     829             : /*
     830             :  * This function is executed in the child of a parallel restore from a
     831             :  * directory-format archive and restores the actual data for one TOC entry.
     832             :  */
     833             : static int
     834           4 : _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
     835             : {
     836           4 :     return parallel_restore(AH, te);
     837             : }

Generated by: LCOV version 1.13