LCOV - code coverage report
Current view: top level - src/bin/pg_dump - pg_backup_directory.c (source / functions) Hit Total Coverage
Test: PostgreSQL 15devel Lines: 227 274 82.8 %
Date: 2021-12-09 04:09:06 Functions: 27 27 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * pg_backup_directory.c
       4             :  *
       5             :  *  A directory format dump is a directory, which contains a "toc.dat" file
       6             :  *  for the TOC, and a separate file for each data entry, named "<oid>.dat".
       7             :  *  Large objects (BLOBs) are stored in separate files named "blob_<oid>.dat",
       8             :  *  and there's a plain-text TOC file for them called "blobs.toc". If
       9             :  *  compression is used, each data file is individually compressed and the
      10             :  *  ".gz" suffix is added to the filenames. The TOC files are never
      11             :  *  compressed by pg_dump, however they are accepted with the .gz suffix too,
      12             :  *  in case the user has manually compressed them with 'gzip'.
      13             :  *
      14             :  *  NOTE: This format is identical to the files written in the tar file in
      15             :  *  the 'tar' format, except that we don't write the restore.sql file (TODO),
      16             :  *  and the tar format doesn't support compression. Please keep the formats in
      17             :  *  sync.
      18             :  *
      19             :  *
      20             :  *  Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
      21             :  *  Portions Copyright (c) 1994, Regents of the University of California
      22             :  *  Portions Copyright (c) 2000, Philip Warner
      23             :  *
      24             :  *  Rights are granted to use this software in any way so long
      25             :  *  as this notice is not removed.
      26             :  *
      27             :  *  The author is not responsible for loss or damages that may
      28             :  *  result from its use.
      29             :  *
      30             :  * IDENTIFICATION
      31             :  *      src/bin/pg_dump/pg_backup_directory.c
      32             :  *
      33             :  *-------------------------------------------------------------------------
      34             :  */
      35             : #include "postgres_fe.h"
      36             : 
      37             : #include <dirent.h>
      38             : #include <sys/stat.h>
      39             : 
      40             : #include "common/file_utils.h"
      41             : #include "compress_io.h"
      42             : #include "parallel.h"
      43             : #include "pg_backup_utils.h"
      44             : 
      45             : typedef struct
      46             : {
      47             :     /*
      48             :      * Our archive location. This is basically what the user specified as his
      49             :      * backup file but of course here it is a directory.
      50             :      */
      51             :     char       *directory;
      52             : 
      53             :     cfp        *dataFH;         /* currently open data file */
      54             : 
      55             :     cfp        *blobsTocFH;     /* file handle for blobs.toc */
      56             :     ParallelState *pstate;      /* for parallel backup / restore */
      57             : } lclContext;
      58             : 
      59             : typedef struct
      60             : {
      61             :     char       *filename;       /* filename excluding the directory (basename) */
      62             : } lclTocEntry;
      63             : 
      64             : /* prototypes for private functions */
      65             : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
      66             : static void _StartData(ArchiveHandle *AH, TocEntry *te);
      67             : static void _EndData(ArchiveHandle *AH, TocEntry *te);
      68             : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
      69             : static int  _WriteByte(ArchiveHandle *AH, const int i);
      70             : static int  _ReadByte(ArchiveHandle *);
      71             : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
      72             : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
      73             : static void _CloseArchive(ArchiveHandle *AH);
      74             : static void _ReopenArchive(ArchiveHandle *AH);
      75             : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
      76             : 
      77             : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
      78             : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
      79             : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
      80             : 
      81             : static void _StartBlobs(ArchiveHandle *AH, TocEntry *te);
      82             : static void _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
      83             : static void _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid);
      84             : static void _EndBlobs(ArchiveHandle *AH, TocEntry *te);
      85             : static void _LoadBlobs(ArchiveHandle *AH);
      86             : 
      87             : static void _PrepParallelRestore(ArchiveHandle *AH);
      88             : static void _Clone(ArchiveHandle *AH);
      89             : static void _DeClone(ArchiveHandle *AH);
      90             : 
      91             : static int  _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te);
      92             : static int  _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te);
      93             : 
      94             : static void setFilePath(ArchiveHandle *AH, char *buf,
      95             :                         const char *relativeFilename);
      96             : 
      97             : /*
      98             :  *  Init routine required by ALL formats. This is a global routine
      99             :  *  and should be declared in pg_backup_archiver.h
     100             :  *
     101             :  *  Its task is to create any extra archive context (using AH->formatData),
     102             :  *  and to initialize the supported function pointers.
     103             :  *
     104             :  *  It should also prepare whatever its input source is for reading/writing,
     105             :  *  and in the case of a read mode connection, it should load the Header & TOC.
     106             :  */
     107             : void
     108          26 : InitArchiveFmt_Directory(ArchiveHandle *AH)
     109             : {
     110             :     lclContext *ctx;
     111             : 
     112             :     /* Assuming static functions, this can be copied for each format. */
     113          26 :     AH->ArchiveEntryPtr = _ArchiveEntry;
     114          26 :     AH->StartDataPtr = _StartData;
     115          26 :     AH->WriteDataPtr = _WriteData;
     116          26 :     AH->EndDataPtr = _EndData;
     117          26 :     AH->WriteBytePtr = _WriteByte;
     118          26 :     AH->ReadBytePtr = _ReadByte;
     119          26 :     AH->WriteBufPtr = _WriteBuf;
     120          26 :     AH->ReadBufPtr = _ReadBuf;
     121          26 :     AH->ClosePtr = _CloseArchive;
     122          26 :     AH->ReopenPtr = _ReopenArchive;
     123          26 :     AH->PrintTocDataPtr = _PrintTocData;
     124          26 :     AH->ReadExtraTocPtr = _ReadExtraToc;
     125          26 :     AH->WriteExtraTocPtr = _WriteExtraToc;
     126          26 :     AH->PrintExtraTocPtr = _PrintExtraToc;
     127             : 
     128          26 :     AH->StartBlobsPtr = _StartBlobs;
     129          26 :     AH->StartBlobPtr = _StartBlob;
     130          26 :     AH->EndBlobPtr = _EndBlob;
     131          26 :     AH->EndBlobsPtr = _EndBlobs;
     132             : 
     133          26 :     AH->PrepParallelRestorePtr = _PrepParallelRestore;
     134          26 :     AH->ClonePtr = _Clone;
     135          26 :     AH->DeClonePtr = _DeClone;
     136             : 
     137          26 :     AH->WorkerJobRestorePtr = _WorkerJobRestoreDirectory;
     138          26 :     AH->WorkerJobDumpPtr = _WorkerJobDumpDirectory;
     139             : 
     140             :     /* Set up our private context */
     141          26 :     ctx = (lclContext *) pg_malloc0(sizeof(lclContext));
     142          26 :     AH->formatData = (void *) ctx;
     143             : 
     144          26 :     ctx->dataFH = NULL;
     145          26 :     ctx->blobsTocFH = NULL;
     146             : 
     147             :     /* Initialize LO buffering */
     148          26 :     AH->lo_buf_size = LOBBUFSIZE;
     149          26 :     AH->lo_buf = (void *) pg_malloc(LOBBUFSIZE);
     150             : 
     151             :     /*
     152             :      * Now open the TOC file
     153             :      */
     154             : 
     155          26 :     if (!AH->fSpec || strcmp(AH->fSpec, "") == 0)
     156           0 :         fatal("no output directory specified");
     157             : 
     158          26 :     ctx->directory = AH->fSpec;
     159             : 
     160          26 :     if (AH->mode == archModeWrite)
     161             :     {
     162             :         struct stat st;
     163          12 :         bool        is_empty = false;
     164             : 
     165             :         /* we accept an empty existing directory */
     166          12 :         if (stat(ctx->directory, &st) == 0 && S_ISDIR(st.st_mode))
     167             :         {
     168           0 :             DIR        *dir = opendir(ctx->directory);
     169             : 
     170           0 :             if (dir)
     171             :             {
     172             :                 struct dirent *d;
     173             : 
     174           0 :                 is_empty = true;
     175           0 :                 while (errno = 0, (d = readdir(dir)))
     176             :                 {
     177           0 :                     if (strcmp(d->d_name, ".") != 0 && strcmp(d->d_name, "..") != 0)
     178             :                     {
     179           0 :                         is_empty = false;
     180           0 :                         break;
     181             :                     }
     182             :                 }
     183             : 
     184           0 :                 if (errno)
     185           0 :                     fatal("could not read directory \"%s\": %m",
     186             :                           ctx->directory);
     187             : 
     188           0 :                 if (closedir(dir))
     189           0 :                     fatal("could not close directory \"%s\": %m",
     190             :                           ctx->directory);
     191             :             }
     192             :         }
     193             : 
     194          12 :         if (!is_empty && mkdir(ctx->directory, 0700) < 0)
     195           0 :             fatal("could not create directory \"%s\": %m",
     196             :                   ctx->directory);
     197             :     }
     198             :     else
     199             :     {                           /* Read Mode */
     200             :         char        fname[MAXPGPATH];
     201             :         cfp        *tocFH;
     202             : 
     203          14 :         setFilePath(AH, fname, "toc.dat");
     204             : 
     205          14 :         tocFH = cfopen_read(fname, PG_BINARY_R);
     206          14 :         if (tocFH == NULL)
     207           0 :             fatal("could not open input file \"%s\": %m", fname);
     208             : 
     209          14 :         ctx->dataFH = tocFH;
     210             : 
     211             :         /*
     212             :          * The TOC of a directory format dump shares the format code of the
     213             :          * tar format.
     214             :          */
     215          14 :         AH->format = archTar;
     216          14 :         ReadHead(AH);
     217          14 :         AH->format = archDirectory;
     218          14 :         ReadToc(AH);
     219             : 
     220             :         /* Nothing else in the file, so close it again... */
     221          14 :         if (cfclose(tocFH) != 0)
     222           0 :             fatal("could not close TOC file: %m");
     223          14 :         ctx->dataFH = NULL;
     224             :     }
     225          26 : }
     226             : 
     227             : /*
     228             :  * Called by the Archiver when the dumper creates a new TOC entry.
     229             :  *
     230             :  * We determine the filename for this entry.
     231             : */
     232             : static void
     233        1162 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
     234             : {
     235             :     lclTocEntry *tctx;
     236             :     char        fn[MAXPGPATH];
     237             : 
     238        1162 :     tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     239        1162 :     if (strcmp(te->desc, "BLOBS") == 0)
     240           4 :         tctx->filename = pg_strdup("blobs.toc");
     241        1158 :     else if (te->dataDumper)
     242             :     {
     243         110 :         snprintf(fn, MAXPGPATH, "%d.dat", te->dumpId);
     244         110 :         tctx->filename = pg_strdup(fn);
     245             :     }
     246             :     else
     247        1048 :         tctx->filename = NULL;
     248             : 
     249        1162 :     te->formatData = (void *) tctx;
     250        1162 : }
     251             : 
     252             : /*
     253             :  * Called by the Archiver to save any extra format-related TOC entry
     254             :  * data.
     255             :  *
     256             :  * Use the Archiver routines to write data - they are non-endian, and
     257             :  * maintain other important file information.
     258             :  */
     259             : static void
     260        1162 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
     261             : {
     262        1162 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     263             : 
     264             :     /*
     265             :      * A dumpable object has set tctx->filename, any other object has not.
     266             :      * (see _ArchiveEntry).
     267             :      */
     268        1162 :     if (tctx->filename)
     269         114 :         WriteStr(AH, tctx->filename);
     270             :     else
     271        1048 :         WriteStr(AH, "");
     272        1162 : }
     273             : 
     274             : /*
     275             :  * Called by the Archiver to read any extra format-related TOC data.
     276             :  *
     277             :  * Needs to match the order defined in _WriteExtraToc, and should also
     278             :  * use the Archiver input routines.
     279             :  */
     280             : static void
     281        1174 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
     282             : {
     283        1174 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     284             : 
     285        1174 :     if (tctx == NULL)
     286             :     {
     287        1174 :         tctx = (lclTocEntry *) pg_malloc0(sizeof(lclTocEntry));
     288        1174 :         te->formatData = (void *) tctx;
     289             :     }
     290             : 
     291        1174 :     tctx->filename = ReadStr(AH);
     292        1174 :     if (strlen(tctx->filename) == 0)
     293             :     {
     294        1058 :         free(tctx->filename);
     295        1058 :         tctx->filename = NULL;
     296             :     }
     297        1174 : }
     298             : 
     299             : /*
     300             :  * Called by the Archiver when restoring an archive to output a comment
     301             :  * that includes useful information about the TOC entry.
     302             :  */
     303             : static void
     304        1100 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
     305             : {
     306        1100 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     307             : 
     308        1100 :     if (AH->public.verbose && tctx->filename)
     309           0 :         ahprintf(AH, "-- File: %s\n", tctx->filename);
     310        1100 : }
     311             : 
     312             : /*
     313             :  * Called by the archiver when saving TABLE DATA (not schema). This routine
     314             :  * should save whatever format-specific information is needed to read
     315             :  * the archive back.
     316             :  *
     317             :  * It is called just prior to the dumper's 'DataDumper' routine being called.
     318             :  *
     319             :  * We create the data file for writing.
     320             :  */
     321             : static void
     322         110 : _StartData(ArchiveHandle *AH, TocEntry *te)
     323             : {
     324         110 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     325         110 :     lclContext *ctx = (lclContext *) AH->formatData;
     326             :     char        fname[MAXPGPATH];
     327             : 
     328         110 :     setFilePath(AH, fname, tctx->filename);
     329             : 
     330         110 :     ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
     331         110 :     if (ctx->dataFH == NULL)
     332           0 :         fatal("could not open output file \"%s\": %m", fname);
     333         110 : }
     334             : 
     335             : /*
     336             :  * Called by archiver when dumper calls WriteData. This routine is
     337             :  * called for both BLOB and TABLE data; it is the responsibility of
     338             :  * the format to manage each kind of data using StartBlob/StartData.
     339             :  *
     340             :  * It should only be called from within a DataDumper routine.
     341             :  *
     342             :  * We write the data to the open data file.
     343             :  */
     344             : static void
     345         230 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
     346             : {
     347         230 :     lclContext *ctx = (lclContext *) AH->formatData;
     348             : 
     349         230 :     errno = 0;
     350         230 :     if (dLen > 0 && cfwrite(data, dLen, ctx->dataFH) != dLen)
     351             :     {
     352             :         /* if write didn't set errno, assume problem is no disk space */
     353           0 :         if (errno == 0)
     354           0 :             errno = ENOSPC;
     355           0 :         fatal("could not write to output file: %s",
     356             :               get_cfp_error(ctx->dataFH));
     357             :     }
     358         230 : }
     359             : 
     360             : /*
     361             :  * Called by the archiver when a dumper's 'DataDumper' routine has
     362             :  * finished.
     363             :  *
     364             :  * We close the data file.
     365             :  */
     366             : static void
     367         110 : _EndData(ArchiveHandle *AH, TocEntry *te)
     368             : {
     369         110 :     lclContext *ctx = (lclContext *) AH->formatData;
     370             : 
     371             :     /* Close the file */
     372         110 :     if (cfclose(ctx->dataFH) != 0)
     373           0 :         fatal("could not close data file: %m");
     374             : 
     375         110 :     ctx->dataFH = NULL;
     376         110 : }
     377             : 
     378             : /*
     379             :  * Print data for a given file (can be a BLOB as well)
     380             :  */
     381             : static void
     382         116 : _PrintFileData(ArchiveHandle *AH, char *filename)
     383             : {
     384             :     size_t      cnt;
     385             :     char       *buf;
     386             :     size_t      buflen;
     387             :     cfp        *cfp;
     388             : 
     389         116 :     if (!filename)
     390           0 :         return;
     391             : 
     392         116 :     cfp = cfopen_read(filename, PG_BINARY_R);
     393             : 
     394         116 :     if (!cfp)
     395           0 :         fatal("could not open input file \"%s\": %m", filename);
     396             : 
     397         116 :     buf = pg_malloc(ZLIB_OUT_SIZE);
     398         116 :     buflen = ZLIB_OUT_SIZE;
     399             : 
     400         232 :     while ((cnt = cfread(buf, buflen, cfp)))
     401             :     {
     402         116 :         ahwrite(buf, 1, cnt, AH);
     403             :     }
     404             : 
     405         116 :     free(buf);
     406         116 :     if (cfclose(cfp) != 0)
     407           0 :         fatal("could not close data file \"%s\": %m", filename);
     408             : }
     409             : 
     410             : /*
     411             :  * Print data for a given TOC entry
     412             : */
     413             : static void
     414         116 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
     415             : {
     416         116 :     lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     417             : 
     418         116 :     if (!tctx->filename)
     419           0 :         return;
     420             : 
     421         116 :     if (strcmp(te->desc, "BLOBS") == 0)
     422           4 :         _LoadBlobs(AH);
     423             :     else
     424             :     {
     425             :         char        fname[MAXPGPATH];
     426             : 
     427         112 :         setFilePath(AH, fname, tctx->filename);
     428         112 :         _PrintFileData(AH, fname);
     429             :     }
     430             : }
     431             : 
     432             : static void
     433           4 : _LoadBlobs(ArchiveHandle *AH)
     434             : {
     435             :     Oid         oid;
     436           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     437             :     char        tocfname[MAXPGPATH];
     438             :     char        line[MAXPGPATH];
     439             : 
     440           4 :     StartRestoreBlobs(AH);
     441             : 
     442           4 :     setFilePath(AH, tocfname, "blobs.toc");
     443             : 
     444           4 :     ctx->blobsTocFH = cfopen_read(tocfname, PG_BINARY_R);
     445             : 
     446           4 :     if (ctx->blobsTocFH == NULL)
     447           0 :         fatal("could not open large object TOC file \"%s\" for input: %m",
     448             :               tocfname);
     449             : 
     450             :     /* Read the blobs TOC file line-by-line, and process each blob */
     451           8 :     while ((cfgets(ctx->blobsTocFH, line, MAXPGPATH)) != NULL)
     452             :     {
     453             :         char        blobfname[MAXPGPATH + 1];
     454             :         char        path[MAXPGPATH];
     455             : 
     456             :         /* Can't overflow because line and blobfname are the same length */
     457           4 :         if (sscanf(line, "%u %" CppAsString2(MAXPGPATH) "s\n", &oid, blobfname) != 2)
     458           0 :             fatal("invalid line in large object TOC file \"%s\": \"%s\"",
     459             :                   tocfname, line);
     460             : 
     461           4 :         StartRestoreBlob(AH, oid, AH->public.ropt->dropSchema);
     462           4 :         snprintf(path, MAXPGPATH, "%s/%s", ctx->directory, blobfname);
     463           4 :         _PrintFileData(AH, path);
     464           4 :         EndRestoreBlob(AH, oid);
     465             :     }
     466           4 :     if (!cfeof(ctx->blobsTocFH))
     467           0 :         fatal("error reading large object TOC file \"%s\"",
     468             :               tocfname);
     469             : 
     470           4 :     if (cfclose(ctx->blobsTocFH) != 0)
     471           0 :         fatal("could not close large object TOC file \"%s\": %m",
     472             :               tocfname);
     473             : 
     474           4 :     ctx->blobsTocFH = NULL;
     475             : 
     476           4 :     EndRestoreBlobs(AH);
     477           4 : }
     478             : 
     479             : 
     480             : /*
     481             :  * Write a byte of data to the archive.
     482             :  * Called by the archiver to do integer & byte output to the archive.
     483             :  * These routines are only used to read & write the headers & TOC.
     484             :  */
     485             : static int
     486      108442 : _WriteByte(ArchiveHandle *AH, const int i)
     487             : {
     488      108442 :     unsigned char c = (unsigned char) i;
     489      108442 :     lclContext *ctx = (lclContext *) AH->formatData;
     490             : 
     491      108442 :     errno = 0;
     492      108442 :     if (cfwrite(&c, 1, ctx->dataFH) != 1)
     493             :     {
     494             :         /* if write didn't set errno, assume problem is no disk space */
     495           0 :         if (errno == 0)
     496           0 :             errno = ENOSPC;
     497           0 :         fatal("could not write to output file: %s",
     498             :               get_cfp_error(ctx->dataFH));
     499             :     }
     500             : 
     501      108442 :     return 1;
     502             : }
     503             : 
     504             : /*
     505             :  * Read a byte of data from the archive.
     506             :  * Called by the archiver to read bytes & integers from the archive.
     507             :  * These routines are only used to read & write headers & TOC.
     508             :  * EOF should be treated as a fatal error.
     509             :  */
     510             : static int
     511      109604 : _ReadByte(ArchiveHandle *AH)
     512             : {
     513      109604 :     lclContext *ctx = (lclContext *) AH->formatData;
     514             : 
     515      109604 :     return cfgetc(ctx->dataFH);
     516             : }
     517             : 
     518             : /*
     519             :  * Write a buffer of data to the archive.
     520             :  * Called by the archiver to write a block of bytes to the TOC or a data file.
     521             :  */
     522             : static void
     523       12916 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
     524             : {
     525       12916 :     lclContext *ctx = (lclContext *) AH->formatData;
     526             : 
     527       12916 :     errno = 0;
     528       12916 :     if (cfwrite(buf, len, ctx->dataFH) != len)
     529             :     {
     530             :         /* if write didn't set errno, assume problem is no disk space */
     531           0 :         if (errno == 0)
     532           0 :             errno = ENOSPC;
     533           0 :         fatal("could not write to output file: %s",
     534             :               get_cfp_error(ctx->dataFH));
     535             :     }
     536       12916 : }
     537             : 
     538             : /*
     539             :  * Read a block of bytes from the archive.
     540             :  *
     541             :  * Called by the archiver to read a block of bytes from the archive
     542             :  */
     543             : static void
     544       13028 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
     545             : {
     546       13028 :     lclContext *ctx = (lclContext *) AH->formatData;
     547             : 
     548             :     /*
     549             :      * If there was an I/O error, we already exited in cfread(), so here we
     550             :      * exit on short reads.
     551             :      */
     552       13028 :     if (cfread(buf, len, ctx->dataFH) != len)
     553           0 :         fatal("could not read from input file: end of file");
     554       13028 : }
     555             : 
     556             : /*
     557             :  * Close the archive.
     558             :  *
     559             :  * When writing the archive, this is the routine that actually starts
     560             :  * the process of saving it to files. No data should be written prior
     561             :  * to this point, since the user could sort the TOC after creating it.
     562             :  *
     563             :  * If an archive is to be written, this routine must call:
     564             :  *      WriteHead           to save the archive header
     565             :  *      WriteToc            to save the TOC entries
     566             :  *      WriteDataChunks     to save all DATA & BLOBs.
     567             :  */
     568             : static void
     569          26 : _CloseArchive(ArchiveHandle *AH)
     570             : {
     571          26 :     lclContext *ctx = (lclContext *) AH->formatData;
     572             : 
     573          26 :     if (AH->mode == archModeWrite)
     574             :     {
     575             :         cfp        *tocFH;
     576             :         char        fname[MAXPGPATH];
     577             : 
     578          12 :         setFilePath(AH, fname, "toc.dat");
     579             : 
     580             :         /* this will actually fork the processes for a parallel backup */
     581          12 :         ctx->pstate = ParallelBackupStart(AH);
     582             : 
     583             :         /* The TOC is always created uncompressed */
     584          12 :         tocFH = cfopen_write(fname, PG_BINARY_W, 0);
     585          12 :         if (tocFH == NULL)
     586           0 :             fatal("could not open output file \"%s\": %m", fname);
     587          12 :         ctx->dataFH = tocFH;
     588             : 
     589             :         /*
     590             :          * Write 'tar' in the format field of the toc.dat file. The directory
     591             :          * is compatible with 'tar', so there's no point having a different
     592             :          * format code for it.
     593             :          */
     594          12 :         AH->format = archTar;
     595          12 :         WriteHead(AH);
     596          12 :         AH->format = archDirectory;
     597          12 :         WriteToc(AH);
     598          12 :         if (cfclose(tocFH) != 0)
     599           0 :             fatal("could not close TOC file: %m");
     600          12 :         WriteDataChunks(AH, ctx->pstate);
     601             : 
     602          12 :         ParallelBackupEnd(AH, ctx->pstate);
     603             : 
     604             :         /*
     605             :          * In directory mode, there is no need to sync all the entries
     606             :          * individually. Just recurse once through all the files generated.
     607             :          */
     608          12 :         if (AH->dosync)
     609           4 :             fsync_dir_recurse(ctx->directory);
     610             :     }
     611          26 :     AH->FH = NULL;
     612          26 : }
     613             : 
     614             : /*
     615             :  * Reopen the archive's file handle.
     616             :  */
     617             : static void
     618          12 : _ReopenArchive(ArchiveHandle *AH)
     619             : {
     620             :     /*
     621             :      * Our TOC is in memory, our data files are opened by each child anyway as
     622             :      * they are separate. We support reopening the archive by just doing
     623             :      * nothing.
     624             :      */
     625          12 : }
     626             : 
     627             : /*
     628             :  * BLOB support
     629             :  */
     630             : 
     631             : /*
     632             :  * Called by the archiver when starting to save all BLOB DATA (not schema).
     633             :  * It is called just prior to the dumper's DataDumper routine.
     634             :  *
     635             :  * We open the large object TOC file here, so that we can append a line to
     636             :  * it for each blob.
     637             :  */
     638             : static void
     639           4 : _StartBlobs(ArchiveHandle *AH, TocEntry *te)
     640             : {
     641           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     642             :     char        fname[MAXPGPATH];
     643             : 
     644           4 :     setFilePath(AH, fname, "blobs.toc");
     645             : 
     646             :     /* The blob TOC file is never compressed */
     647           4 :     ctx->blobsTocFH = cfopen_write(fname, "ab", 0);
     648           4 :     if (ctx->blobsTocFH == NULL)
     649           0 :         fatal("could not open output file \"%s\": %m", fname);
     650           4 : }
     651             : 
     652             : /*
     653             :  * Called by the archiver when we're about to start dumping a blob.
     654             :  *
     655             :  * We create a file to write the blob to.
     656             :  */
     657             : static void
     658           4 : _StartBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
     659             : {
     660           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     661             :     char        fname[MAXPGPATH];
     662             : 
     663           4 :     snprintf(fname, MAXPGPATH, "%s/blob_%u.dat", ctx->directory, oid);
     664             : 
     665           4 :     ctx->dataFH = cfopen_write(fname, PG_BINARY_W, AH->compression);
     666             : 
     667           4 :     if (ctx->dataFH == NULL)
     668           0 :         fatal("could not open output file \"%s\": %m", fname);
     669           4 : }
     670             : 
     671             : /*
     672             :  * Called by the archiver when the dumper is finished writing a blob.
     673             :  *
     674             :  * We close the blob file and write an entry to the blob TOC file for it.
     675             :  */
     676             : static void
     677           4 : _EndBlob(ArchiveHandle *AH, TocEntry *te, Oid oid)
     678             : {
     679           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     680             :     char        buf[50];
     681             :     int         len;
     682             : 
     683             :     /* Close the BLOB data file itself */
     684           4 :     if (cfclose(ctx->dataFH) != 0)
     685           0 :         fatal("could not close blob data file: %m");
     686           4 :     ctx->dataFH = NULL;
     687             : 
     688             :     /* register the blob in blobs.toc */
     689           4 :     len = snprintf(buf, sizeof(buf), "%u blob_%u.dat\n", oid, oid);
     690           4 :     if (cfwrite(buf, len, ctx->blobsTocFH) != len)
     691           0 :         fatal("could not write to blobs TOC file");
     692           4 : }
     693             : 
     694             : /*
     695             :  * Called by the archiver when finishing saving all BLOB DATA.
     696             :  *
     697             :  * We close the blobs TOC file.
     698             :  */
     699             : static void
     700           4 : _EndBlobs(ArchiveHandle *AH, TocEntry *te)
     701             : {
     702           4 :     lclContext *ctx = (lclContext *) AH->formatData;
     703             : 
     704           4 :     if (cfclose(ctx->blobsTocFH) != 0)
     705           0 :         fatal("could not close blobs TOC file: %m");
     706           4 :     ctx->blobsTocFH = NULL;
     707           4 : }
     708             : 
     709             : /*
     710             :  * Gets a relative file name and prepends the output directory, writing the
     711             :  * result to buf. The caller needs to make sure that buf is MAXPGPATH bytes
     712             :  * big. Can't use a static char[MAXPGPATH] inside the function because we run
     713             :  * multithreaded on Windows.
     714             :  */
     715             : static void
     716         260 : setFilePath(ArchiveHandle *AH, char *buf, const char *relativeFilename)
     717             : {
     718         260 :     lclContext *ctx = (lclContext *) AH->formatData;
     719             :     char       *dname;
     720             : 
     721         260 :     dname = ctx->directory;
     722             : 
     723         260 :     if (strlen(dname) + 1 + strlen(relativeFilename) + 1 > MAXPGPATH)
     724           0 :         fatal("file name too long: \"%s\"", dname);
     725             : 
     726         260 :     strcpy(buf, dname);
     727         260 :     strcat(buf, "/");
     728         260 :     strcat(buf, relativeFilename);
     729         260 : }
     730             : 
     731             : /*
     732             :  * Prepare for parallel restore.
     733             :  *
     734             :  * The main thing that needs to happen here is to fill in TABLE DATA and BLOBS
     735             :  * TOC entries' dataLength fields with appropriate values to guide the
     736             :  * ordering of restore jobs.  The source of said data is format-dependent,
     737             :  * as is the exact meaning of the values.
     738             :  *
     739             :  * A format module might also choose to do other setup here.
     740             :  */
     741             : static void
     742           4 : _PrepParallelRestore(ArchiveHandle *AH)
     743             : {
     744             :     TocEntry   *te;
     745             : 
     746          28 :     for (te = AH->toc->next; te != AH->toc; te = te->next)
     747             :     {
     748          24 :         lclTocEntry *tctx = (lclTocEntry *) te->formatData;
     749             :         char        fname[MAXPGPATH];
     750             :         struct stat st;
     751             : 
     752             :         /*
     753             :          * A dumpable object has set tctx->filename, any other object has not.
     754             :          * (see _ArchiveEntry).
     755             :          */
     756          24 :         if (tctx->filename == NULL)
     757          20 :             continue;
     758             : 
     759             :         /* We may ignore items not due to be restored */
     760           4 :         if ((te->reqs & REQ_DATA) == 0)
     761           0 :             continue;
     762             : 
     763             :         /*
     764             :          * Stat the file and, if successful, put its size in dataLength.  When
     765             :          * using compression, the physical file size might not be a very good
     766             :          * guide to the amount of work involved in restoring the file, but we
     767             :          * only need an approximate indicator of that.
     768             :          */
     769           4 :         setFilePath(AH, fname, tctx->filename);
     770             : 
     771           4 :         if (stat(fname, &st) == 0)
     772           0 :             te->dataLength = st.st_size;
     773             :         else
     774             :         {
     775             :             /* It might be compressed */
     776           4 :             strlcat(fname, ".gz", sizeof(fname));
     777           4 :             if (stat(fname, &st) == 0)
     778           4 :                 te->dataLength = st.st_size;
     779             :         }
     780             : 
     781             :         /*
     782             :          * If this is the BLOBS entry, what we stat'd was blobs.toc, which
     783             :          * most likely is a lot smaller than the actual blob data.  We don't
     784             :          * have a cheap way to estimate how much smaller, but fortunately it
     785             :          * doesn't matter too much as long as we get the blobs processed
     786             :          * reasonably early.  Arbitrarily scale up by a factor of 1K.
     787             :          */
     788           4 :         if (strcmp(te->desc, "BLOBS") == 0)
     789           0 :             te->dataLength *= 1024;
     790             :     }
     791           4 : }
     792             : 
     793             : /*
     794             :  * Clone format-specific fields during parallel restoration.
     795             :  */
     796             : static void
     797          24 : _Clone(ArchiveHandle *AH)
     798             : {
     799          24 :     lclContext *ctx = (lclContext *) AH->formatData;
     800             : 
     801          24 :     AH->formatData = (lclContext *) pg_malloc(sizeof(lclContext));
     802          24 :     memcpy(AH->formatData, ctx, sizeof(lclContext));
     803          24 :     ctx = (lclContext *) AH->formatData;
     804             : 
     805             :     /*
     806             :      * Note: we do not make a local lo_buf because we expect at most one BLOBS
     807             :      * entry per archive, so no parallelism is possible.  Likewise,
     808             :      * TOC-entry-local state isn't an issue because any one TOC entry is
     809             :      * touched by just one worker child.
     810             :      */
     811             : 
     812             :     /*
     813             :      * We also don't copy the ParallelState pointer (pstate), only the leader
     814             :      * process ever writes to it.
     815             :      */
     816          24 : }
     817             : 
     818             : static void
     819          24 : _DeClone(ArchiveHandle *AH)
     820             : {
     821          24 :     lclContext *ctx = (lclContext *) AH->formatData;
     822             : 
     823          24 :     free(ctx);
     824          24 : }
     825             : 
     826             : /*
     827             :  * This function is executed in the child of a parallel backup for a
     828             :  * directory-format archive and dumps the actual data for one TOC entry.
     829             :  */
     830             : static int
     831          62 : _WorkerJobDumpDirectory(ArchiveHandle *AH, TocEntry *te)
     832             : {
     833             :     /*
     834             :      * This function returns void. We either fail and die horribly or
     835             :      * succeed... A failure will be detected by the parent when the child dies
     836             :      * unexpectedly.
     837             :      */
     838          62 :     WriteDataChunksForTocEntry(AH, te);
     839             : 
     840          62 :     return 0;
     841             : }
     842             : 
     843             : /*
     844             :  * This function is executed in the child of a parallel restore from a
     845             :  * directory-format archive and restores the actual data for one TOC entry.
     846             :  */
     847             : static int
     848           4 : _WorkerJobRestoreDirectory(ArchiveHandle *AH, TocEntry *te)
     849             : {
     850           4 :     return parallel_restore(AH, te);
     851             : }

Generated by: LCOV version 1.14