Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * pg_backup_tar.c
4 : *
5 : * This file is copied from the 'files' format file, but dumps data into
6 : * one temp file then sends it to the output TAR archive.
7 : *
8 : * The tar format also includes a 'restore.sql' script which is there for
9 : * the benefit of humans. This script is never used by pg_restore.
10 : *
11 : * NOTE: If you untar the created 'tar' file, the resulting files are
12 : * compatible with the 'directory' format. Please keep the two formats in
13 : * sync.
14 : *
15 : * See the headers to pg_backup_directory & pg_restore for more details.
16 : *
17 : * Copyright (c) 2000, Philip Warner
18 : * Rights are granted to use this software in any way so long
19 : * as this notice is not removed.
20 : *
21 : * The author is not responsible for loss or damages that may
22 : * result from its use.
23 : *
24 : *
25 : * IDENTIFICATION
26 : * src/bin/pg_dump/pg_backup_tar.c
27 : *
28 : *-------------------------------------------------------------------------
29 : */
30 : #include "postgres_fe.h"
31 :
32 : #include <sys/stat.h>
33 : #include <ctype.h>
34 : #include <limits.h>
35 : #include <unistd.h>
36 :
37 : #include "common/file_utils.h"
38 : #include "fe_utils/string_utils.h"
39 : #include "pg_backup_archiver.h"
40 : #include "pg_backup_tar.h"
41 : #include "pg_backup_utils.h"
42 : #include "pgtar.h"
43 :
44 : static void _ArchiveEntry(ArchiveHandle *AH, TocEntry *te);
45 : static void _StartData(ArchiveHandle *AH, TocEntry *te);
46 : static void _WriteData(ArchiveHandle *AH, const void *data, size_t dLen);
47 : static void _EndData(ArchiveHandle *AH, TocEntry *te);
48 : static int _WriteByte(ArchiveHandle *AH, const int i);
49 : static int _ReadByte(ArchiveHandle *AH);
50 : static void _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len);
51 : static void _ReadBuf(ArchiveHandle *AH, void *buf, size_t len);
52 : static void _CloseArchive(ArchiveHandle *AH);
53 : static void _PrintTocData(ArchiveHandle *AH, TocEntry *te);
54 : static void _WriteExtraToc(ArchiveHandle *AH, TocEntry *te);
55 : static void _ReadExtraToc(ArchiveHandle *AH, TocEntry *te);
56 : static void _PrintExtraToc(ArchiveHandle *AH, TocEntry *te);
57 :
58 : static void _StartLOs(ArchiveHandle *AH, TocEntry *te);
59 : static void _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
60 : static void _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid);
61 : static void _EndLOs(ArchiveHandle *AH, TocEntry *te);
62 :
63 : #define K_STD_BUF_SIZE 1024
64 :
65 :
66 : typedef struct
67 : {
68 : FILE *nFH;
69 : FILE *tarFH;
70 : FILE *tmpFH;
71 : char *targetFile;
72 : char mode;
73 : pgoff_t pos;
74 : pgoff_t fileLen;
75 : ArchiveHandle *AH;
76 : } TAR_MEMBER;
77 :
78 : typedef struct
79 : {
80 : int hasSeek;
81 : pgoff_t filePos;
82 : TAR_MEMBER *loToc;
83 : FILE *tarFH;
84 : pgoff_t tarFHpos;
85 : pgoff_t tarNextMember;
86 : TAR_MEMBER *FH;
87 : int isSpecialScript;
88 : TAR_MEMBER *scriptTH;
89 : } lclContext;
90 :
91 : typedef struct
92 : {
93 : TAR_MEMBER *TH;
94 : char *filename;
95 : } lclTocEntry;
96 :
97 : static void _LoadLOs(ArchiveHandle *AH, TocEntry *te);
98 :
99 : static TAR_MEMBER *tarOpen(ArchiveHandle *AH, const char *filename, char mode);
100 : static void tarClose(ArchiveHandle *AH, TAR_MEMBER *th);
101 :
102 : #ifdef __NOT_USED__
103 : static char *tarGets(char *buf, size_t len, TAR_MEMBER *th);
104 : #endif
105 : static int tarPrintf(TAR_MEMBER *th, const char *fmt,...) pg_attribute_printf(2, 3);
106 :
107 : static void _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th);
108 : static TAR_MEMBER *_tarPositionTo(ArchiveHandle *AH, const char *filename);
109 : static size_t tarRead(void *buf, size_t len, TAR_MEMBER *th);
110 : static size_t tarWrite(const void *buf, size_t len, TAR_MEMBER *th);
111 : static void _tarWriteHeader(TAR_MEMBER *th);
112 : static int _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th);
113 : static size_t _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh);
114 :
115 : static size_t _scriptOut(ArchiveHandle *AH, const void *buf, size_t len);
116 :
117 : /*
118 : * Initializer
119 : */
120 : void
121 5 : InitArchiveFmt_Tar(ArchiveHandle *AH)
122 : {
123 : lclContext *ctx;
124 :
125 : /* Assuming static functions, this can be copied for each format. */
126 5 : AH->ArchiveEntryPtr = _ArchiveEntry;
127 5 : AH->StartDataPtr = _StartData;
128 5 : AH->WriteDataPtr = _WriteData;
129 5 : AH->EndDataPtr = _EndData;
130 5 : AH->WriteBytePtr = _WriteByte;
131 5 : AH->ReadBytePtr = _ReadByte;
132 5 : AH->WriteBufPtr = _WriteBuf;
133 5 : AH->ReadBufPtr = _ReadBuf;
134 5 : AH->ClosePtr = _CloseArchive;
135 5 : AH->ReopenPtr = NULL;
136 5 : AH->PrintTocDataPtr = _PrintTocData;
137 5 : AH->ReadExtraTocPtr = _ReadExtraToc;
138 5 : AH->WriteExtraTocPtr = _WriteExtraToc;
139 5 : AH->PrintExtraTocPtr = _PrintExtraToc;
140 :
141 5 : AH->StartLOsPtr = _StartLOs;
142 5 : AH->StartLOPtr = _StartLO;
143 5 : AH->EndLOPtr = _EndLO;
144 5 : AH->EndLOsPtr = _EndLOs;
145 5 : AH->ClonePtr = NULL;
146 5 : AH->DeClonePtr = NULL;
147 :
148 5 : AH->WorkerJobDumpPtr = NULL;
149 5 : AH->WorkerJobRestorePtr = NULL;
150 :
151 : /*
152 : * Set up some special context used in compressing data.
153 : */
154 5 : ctx = pg_malloc0_object(lclContext);
155 5 : AH->formatData = ctx;
156 5 : ctx->filePos = 0;
157 5 : ctx->isSpecialScript = 0;
158 :
159 : /*
160 : * Now open the tar file, and load the TOC if we're in read mode.
161 : */
162 5 : if (AH->mode == archModeWrite)
163 : {
164 3 : if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
165 : {
166 2 : ctx->tarFH = fopen(AH->fSpec, PG_BINARY_W);
167 2 : if (ctx->tarFH == NULL)
168 0 : pg_fatal("could not open TOC file \"%s\" for output: %m",
169 : AH->fSpec);
170 : }
171 : else
172 : {
173 1 : ctx->tarFH = stdout;
174 1 : if (ctx->tarFH == NULL)
175 0 : pg_fatal("could not open TOC file for output: %m");
176 : }
177 :
178 3 : ctx->tarFHpos = 0;
179 :
180 : /*
181 : * Make unbuffered since we will dup() it, and the buffers screw each
182 : * other
183 : */
184 : /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
185 :
186 3 : ctx->hasSeek = checkSeek(ctx->tarFH);
187 :
188 : /*
189 : * We don't support compression because reading the files back is not
190 : * possible since gzdopen uses buffered IO which totally screws file
191 : * positioning.
192 : */
193 3 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
194 1 : pg_fatal("compression is not supported by tar archive format");
195 : }
196 : else
197 : { /* Read Mode */
198 2 : if (AH->fSpec && strcmp(AH->fSpec, "") != 0)
199 : {
200 2 : ctx->tarFH = fopen(AH->fSpec, PG_BINARY_R);
201 2 : if (ctx->tarFH == NULL)
202 0 : pg_fatal("could not open TOC file \"%s\" for input: %m",
203 : AH->fSpec);
204 : }
205 : else
206 : {
207 0 : ctx->tarFH = stdin;
208 0 : if (ctx->tarFH == NULL)
209 0 : pg_fatal("could not open TOC file for input: %m");
210 : }
211 :
212 : /*
213 : * Make unbuffered since we will dup() it, and the buffers screw each
214 : * other
215 : */
216 : /* setvbuf(ctx->tarFH, NULL, _IONBF, 0); */
217 :
218 2 : ctx->tarFHpos = 0;
219 :
220 2 : ctx->hasSeek = checkSeek(ctx->tarFH);
221 :
222 2 : ctx->FH = tarOpen(AH, "toc.dat", 'r');
223 2 : ReadHead(AH);
224 2 : ReadToc(AH);
225 2 : tarClose(AH, ctx->FH); /* Nothing else in the file... */
226 : }
227 4 : }
228 :
229 : /*
230 : * - Start a new TOC entry
231 : * Setup the output file name.
232 : */
233 : static void
234 404 : _ArchiveEntry(ArchiveHandle *AH, TocEntry *te)
235 : {
236 : lclTocEntry *ctx;
237 : char fn[K_STD_BUF_SIZE];
238 :
239 404 : ctx = pg_malloc0_object(lclTocEntry);
240 404 : if (te->dataDumper != NULL)
241 : {
242 42 : snprintf(fn, sizeof(fn), "%d.dat", te->dumpId);
243 42 : ctx->filename = pg_strdup(fn);
244 : }
245 : else
246 : {
247 362 : ctx->filename = NULL;
248 362 : ctx->TH = NULL;
249 : }
250 404 : te->formatData = ctx;
251 404 : }
252 :
253 : static void
254 404 : _WriteExtraToc(ArchiveHandle *AH, TocEntry *te)
255 : {
256 404 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
257 :
258 404 : if (ctx->filename)
259 42 : WriteStr(AH, ctx->filename);
260 : else
261 362 : WriteStr(AH, "");
262 404 : }
263 :
264 : static void
265 404 : _ReadExtraToc(ArchiveHandle *AH, TocEntry *te)
266 : {
267 404 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
268 :
269 404 : if (ctx == NULL)
270 : {
271 404 : ctx = pg_malloc0_object(lclTocEntry);
272 404 : te->formatData = ctx;
273 : }
274 :
275 404 : ctx->filename = ReadStr(AH);
276 404 : if (strlen(ctx->filename) == 0)
277 : {
278 362 : free(ctx->filename);
279 362 : ctx->filename = NULL;
280 : }
281 404 : ctx->TH = NULL;
282 404 : }
283 :
284 : static void
285 792 : _PrintExtraToc(ArchiveHandle *AH, TocEntry *te)
286 : {
287 792 : lclTocEntry *ctx = (lclTocEntry *) te->formatData;
288 :
289 792 : if (AH->public.verbose && ctx->filename != NULL)
290 0 : ahprintf(AH, "-- File: %s\n", ctx->filename);
291 792 : }
292 :
293 : static void
294 40 : _StartData(ArchiveHandle *AH, TocEntry *te)
295 : {
296 40 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
297 :
298 40 : tctx->TH = tarOpen(AH, tctx->filename, 'w');
299 40 : }
300 :
301 : static TAR_MEMBER *
302 94 : tarOpen(ArchiveHandle *AH, const char *filename, char mode)
303 : {
304 94 : lclContext *ctx = (lclContext *) AH->formatData;
305 : TAR_MEMBER *tm;
306 :
307 94 : if (mode == 'r')
308 : {
309 46 : tm = _tarPositionTo(AH, filename);
310 46 : if (!tm) /* Not found */
311 : {
312 0 : if (filename)
313 : {
314 : /*
315 : * Couldn't find the requested file. Future: do SEEK(0) and
316 : * retry.
317 : */
318 0 : pg_fatal("could not find file \"%s\" in archive", filename);
319 : }
320 : else
321 : {
322 : /* Any file OK, none left, so return NULL */
323 0 : return NULL;
324 : }
325 : }
326 :
327 46 : if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
328 46 : tm->nFH = ctx->tarFH;
329 : else
330 0 : pg_fatal("compression is not supported by tar archive format");
331 : }
332 : else
333 : {
334 : int old_umask;
335 :
336 48 : tm = pg_malloc0_object(TAR_MEMBER);
337 :
338 : /*
339 : * POSIX does not require, but permits, tmpfile() to restrict file
340 : * permissions. Given an OS crash after we write data, the filesystem
341 : * might retain the data but forget tmpfile()'s unlink(). If so, the
342 : * file mode protects confidentiality of the data written.
343 : */
344 48 : old_umask = umask(S_IRWXG | S_IRWXO);
345 :
346 : #ifndef WIN32
347 48 : tm->tmpFH = tmpfile();
348 : #else
349 :
350 : /*
351 : * On WIN32, tmpfile() generates a filename in the root directory,
352 : * which requires administrative permissions on certain systems. Loop
353 : * until we find a unique file name we can create.
354 : */
355 : while (1)
356 : {
357 : char *name;
358 : int fd;
359 :
360 : name = _tempnam(NULL, "pg_temp_");
361 : if (name == NULL)
362 : break;
363 : fd = open(name, O_RDWR | O_CREAT | O_EXCL | O_BINARY |
364 : O_TEMPORARY, S_IRUSR | S_IWUSR);
365 : free(name);
366 :
367 : if (fd != -1) /* created a file */
368 : {
369 : tm->tmpFH = fdopen(fd, "w+b");
370 : break;
371 : }
372 : else if (errno != EEXIST) /* failure other than file exists */
373 : break;
374 : }
375 : #endif
376 :
377 48 : if (tm->tmpFH == NULL)
378 0 : pg_fatal("could not generate temporary file name: %m");
379 :
380 48 : umask(old_umask);
381 :
382 48 : if (AH->compression_spec.algorithm == PG_COMPRESSION_NONE)
383 48 : tm->nFH = tm->tmpFH;
384 : else
385 0 : pg_fatal("compression is not supported by tar archive format");
386 :
387 48 : tm->AH = AH;
388 48 : tm->targetFile = pg_strdup(filename);
389 : }
390 :
391 94 : tm->mode = mode;
392 94 : tm->tarFH = ctx->tarFH;
393 :
394 94 : return tm;
395 : }
396 :
397 : static void
398 94 : tarClose(ArchiveHandle *AH, TAR_MEMBER *th)
399 : {
400 94 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
401 0 : pg_fatal("compression is not supported by tar archive format");
402 :
403 94 : if (th->mode == 'w')
404 48 : _tarAddFile(AH, th); /* This will close the temp file */
405 :
406 : /*
407 : * else Nothing to do for normal read since we don't dup() normal file
408 : * handle, and we don't use temp files.
409 : */
410 :
411 94 : free(th->targetFile);
412 :
413 94 : th->nFH = NULL;
414 94 : }
415 :
416 : #ifdef __NOT_USED__
417 : static char *
418 : tarGets(char *buf, size_t len, TAR_MEMBER *th)
419 : {
420 : char *s;
421 : size_t cnt = 0;
422 : char c = ' ';
423 : int eof = 0;
424 :
425 : /* Can't read past logical EOF */
426 : if (len > (th->fileLen - th->pos))
427 : len = th->fileLen - th->pos;
428 :
429 : while (cnt < len && c != '\n')
430 : {
431 : if (_tarReadRaw(th->AH, &c, 1, th, NULL) <= 0)
432 : {
433 : eof = 1;
434 : break;
435 : }
436 : buf[cnt++] = c;
437 : }
438 :
439 : if (eof && cnt == 0)
440 : s = NULL;
441 : else
442 : {
443 : buf[cnt++] = '\0';
444 : s = buf;
445 : }
446 :
447 : if (s)
448 : {
449 : len = strlen(s);
450 : th->pos += len;
451 : }
452 :
453 : return s;
454 : }
455 : #endif
456 :
457 : /*
458 : * Just read bytes from the archive. This is the low level read routine
459 : * that is used for ALL reads on a tar file.
460 : */
461 : static size_t
462 64944 : _tarReadRaw(ArchiveHandle *AH, void *buf, size_t len, TAR_MEMBER *th, FILE *fh)
463 : {
464 64944 : lclContext *ctx = (lclContext *) AH->formatData;
465 : size_t avail;
466 64944 : size_t used = 0;
467 64944 : size_t res = 0;
468 :
469 : Assert(th || fh);
470 :
471 64944 : avail = AH->lookaheadLen - AH->lookaheadPos;
472 64944 : if (avail > 0)
473 : {
474 : /* We have some lookahead bytes to use */
475 0 : if (avail >= len) /* Just use the lookahead buffer */
476 0 : used = len;
477 : else
478 0 : used = avail;
479 :
480 : /* Copy, and adjust buffer pos */
481 0 : memcpy(buf, AH->lookahead + AH->lookaheadPos, used);
482 0 : AH->lookaheadPos += used;
483 :
484 : /* Adjust required length */
485 0 : len -= used;
486 : }
487 :
488 : /* Read the file if len > 0 */
489 64944 : if (len > 0)
490 : {
491 64944 : if (fh)
492 : {
493 21019 : res = fread(&((char *) buf)[used], 1, len, fh);
494 21019 : if (res != len && !feof(fh))
495 0 : READ_ERROR_EXIT(fh);
496 : }
497 43925 : else if (th)
498 : {
499 43925 : res = fread(&((char *) buf)[used], 1, len, th->nFH);
500 43925 : if (res != len && !feof(th->nFH))
501 0 : READ_ERROR_EXIT(th->nFH);
502 : }
503 : }
504 :
505 64944 : ctx->tarFHpos += res + used;
506 :
507 64944 : return (res + used);
508 : }
509 :
510 : static size_t
511 44413 : tarRead(void *buf, size_t len, TAR_MEMBER *th)
512 : {
513 : size_t res;
514 :
515 44413 : if (th->pos + len > th->fileLen)
516 83 : len = th->fileLen - th->pos;
517 :
518 44413 : if (len <= 0)
519 488 : return 0;
520 :
521 43925 : res = _tarReadRaw(th->AH, buf, len, th, NULL);
522 :
523 43925 : th->pos += res;
524 :
525 43925 : return res;
526 : }
527 :
528 : static size_t
529 46908 : tarWrite(const void *buf, size_t len, TAR_MEMBER *th)
530 : {
531 : size_t res;
532 :
533 46908 : res = fwrite(buf, 1, len, th->nFH);
534 :
535 46908 : th->pos += res;
536 46908 : return res;
537 : }
538 :
539 : static void
540 272 : _WriteData(ArchiveHandle *AH, const void *data, size_t dLen)
541 : {
542 272 : lclTocEntry *tctx = (lclTocEntry *) AH->currToc->formatData;
543 :
544 272 : if (tarWrite(data, dLen, tctx->TH) != dLen)
545 0 : WRITE_ERROR_EXIT;
546 272 : }
547 :
548 : static void
549 40 : _EndData(ArchiveHandle *AH, TocEntry *te)
550 : {
551 40 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
552 :
553 : /* Close the file */
554 40 : tarClose(AH, tctx->TH);
555 40 : tctx->TH = NULL;
556 40 : }
557 :
558 : /*
559 : * Print data for a given file
560 : */
561 : static void
562 40 : _PrintFileData(ArchiveHandle *AH, char *filename)
563 : {
564 40 : lclContext *ctx = (lclContext *) AH->formatData;
565 : char buf[4096];
566 : size_t cnt;
567 : TAR_MEMBER *th;
568 :
569 40 : if (!filename)
570 0 : return;
571 :
572 40 : th = tarOpen(AH, filename, 'r');
573 40 : ctx->FH = th;
574 :
575 80 : while ((cnt = tarRead(buf, 4095, th)) > 0)
576 : {
577 40 : buf[cnt] = '\0';
578 40 : ahwrite(buf, 1, cnt, AH);
579 : }
580 :
581 40 : tarClose(AH, th);
582 : }
583 :
584 :
585 : /*
586 : * Print data for a given TOC entry
587 : */
588 : static void
589 84 : _PrintTocData(ArchiveHandle *AH, TocEntry *te)
590 : {
591 84 : lclContext *ctx = (lclContext *) AH->formatData;
592 84 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
593 : int pos1;
594 :
595 84 : if (!tctx->filename)
596 0 : return;
597 :
598 : /*
599 : * If we're writing the special restore.sql script, emit a suitable
600 : * command to include each table's data from the corresponding file.
601 : *
602 : * In the COPY case this is a bit klugy because the regular COPY command
603 : * was already printed before we get control.
604 : */
605 84 : if (ctx->isSpecialScript)
606 : {
607 42 : if (te->copyStmt)
608 : {
609 : /* Abort the COPY FROM stdin */
610 40 : ahprintf(AH, "\\.\n");
611 :
612 : /*
613 : * The COPY statement should look like "COPY ... FROM stdin;\n",
614 : * see dumpTableData().
615 : */
616 40 : pos1 = (int) strlen(te->copyStmt) - 13;
617 40 : if (pos1 < 6 || strncmp(te->copyStmt, "COPY ", 5) != 0 ||
618 40 : strcmp(te->copyStmt + pos1, " FROM stdin;\n") != 0)
619 0 : pg_fatal("unexpected COPY statement syntax: \"%s\"",
620 : te->copyStmt);
621 :
622 : /* Emit all but the FROM part ... */
623 40 : ahwrite(te->copyStmt, 1, pos1, AH);
624 : /* ... and insert modified FROM */
625 40 : ahprintf(AH, " FROM '$$PATH$$/%s';\n\n", tctx->filename);
626 : }
627 : else
628 : {
629 : /* --inserts mode, no worries, just include the data file */
630 2 : ahprintf(AH, "\\i $$PATH$$/%s\n\n", tctx->filename);
631 : }
632 :
633 42 : return;
634 : }
635 :
636 42 : if (strcmp(te->desc, "BLOBS") == 0)
637 2 : _LoadLOs(AH, te);
638 : else
639 40 : _PrintFileData(AH, tctx->filename);
640 : }
641 :
642 : static void
643 2 : _LoadLOs(ArchiveHandle *AH, TocEntry *te)
644 : {
645 : Oid oid;
646 2 : lclContext *ctx = (lclContext *) AH->formatData;
647 : TAR_MEMBER *th;
648 : size_t cnt;
649 2 : bool foundLO = false;
650 : char buf[4096];
651 :
652 2 : StartRestoreLOs(AH);
653 :
654 : /*
655 : * The blobs_NNN.toc or blobs.toc file is fairly useless to us because it
656 : * will appear only after the associated blob_NNN.dat files. For archive
657 : * versions >= 16 we can look at the BLOBS entry's te->tag to discover the
658 : * OID of the first blob we want to restore, and then search forward to
659 : * find the appropriate blob_<oid>.dat file. For older versions we rely
660 : * on the knowledge that there was only one BLOBS entry and just search
661 : * for the first blob_<oid>.dat file. Once we find the first blob file to
662 : * restore, restore all blobs until we reach the blobs[_NNN].toc file.
663 : */
664 2 : if (AH->version >= K_VERS_1_16)
665 : {
666 : /* We rely on atooid to not complain about nnnn..nnnn tags */
667 2 : oid = atooid(te->tag);
668 2 : snprintf(buf, sizeof(buf), "blob_%u.dat", oid);
669 2 : th = tarOpen(AH, buf, 'r'); /* Advance to first desired file */
670 : }
671 : else
672 0 : th = tarOpen(AH, NULL, 'r'); /* Open next file */
673 :
674 4 : while (th != NULL)
675 : {
676 4 : ctx->FH = th;
677 :
678 4 : if (strncmp(th->targetFile, "blob_", 5) == 0)
679 : {
680 2 : oid = atooid(&th->targetFile[5]);
681 2 : if (oid != 0)
682 : {
683 2 : pg_log_info("restoring large object with OID %u", oid);
684 :
685 2 : StartRestoreLO(AH, oid, AH->public.ropt->dropSchema);
686 :
687 3 : while ((cnt = tarRead(buf, 4095, th)) > 0)
688 : {
689 1 : buf[cnt] = '\0';
690 1 : ahwrite(buf, 1, cnt, AH);
691 : }
692 2 : EndRestoreLO(AH, oid);
693 2 : foundLO = true;
694 : }
695 2 : tarClose(AH, th);
696 : }
697 : else
698 : {
699 2 : tarClose(AH, th);
700 :
701 : /*
702 : * Once we have found the first LO, stop at the first non-LO entry
703 : * (which will be 'blobs[_NNN].toc'). This coding would eat all
704 : * the rest of the archive if there are no LOs ... but this
705 : * function shouldn't be called at all in that case.
706 : */
707 2 : if (foundLO)
708 2 : break;
709 : }
710 :
711 2 : th = tarOpen(AH, NULL, 'r');
712 : }
713 2 : EndRestoreLOs(AH);
714 2 : }
715 :
716 :
717 : static int
718 39864 : _WriteByte(ArchiveHandle *AH, const int i)
719 : {
720 39864 : lclContext *ctx = (lclContext *) AH->formatData;
721 39864 : char b = i; /* Avoid endian problems */
722 :
723 39864 : if (tarWrite(&b, 1, ctx->FH) != 1)
724 0 : WRITE_ERROR_EXIT;
725 :
726 39864 : ctx->filePos += 1;
727 39864 : return 1;
728 : }
729 :
730 : static int
731 39864 : _ReadByte(ArchiveHandle *AH)
732 : {
733 39864 : lclContext *ctx = (lclContext *) AH->formatData;
734 : size_t res;
735 : unsigned char c;
736 :
737 39864 : res = tarRead(&c, 1, ctx->FH);
738 39864 : if (res != 1)
739 : /* We already would have exited for errors on reads, must be EOF */
740 0 : pg_fatal("could not read from input file: end of file");
741 39864 : ctx->filePos += 1;
742 39864 : return c;
743 : }
744 :
745 : static void
746 4466 : _WriteBuf(ArchiveHandle *AH, const void *buf, size_t len)
747 : {
748 4466 : lclContext *ctx = (lclContext *) AH->formatData;
749 :
750 4466 : if (tarWrite(buf, len, ctx->FH) != len)
751 0 : WRITE_ERROR_EXIT;
752 :
753 4466 : ctx->filePos += len;
754 4466 : }
755 :
756 : static void
757 4466 : _ReadBuf(ArchiveHandle *AH, void *buf, size_t len)
758 : {
759 4466 : lclContext *ctx = (lclContext *) AH->formatData;
760 :
761 4466 : if (tarRead(buf, len, ctx->FH) != len)
762 : /* We already would have exited for errors on reads, must be EOF */
763 0 : pg_fatal("could not read from input file: end of file");
764 :
765 4466 : ctx->filePos += len;
766 4466 : }
767 :
768 : static void
769 4 : _CloseArchive(ArchiveHandle *AH)
770 : {
771 4 : lclContext *ctx = (lclContext *) AH->formatData;
772 : TAR_MEMBER *th;
773 : RestoreOptions *ropt;
774 : RestoreOptions *savRopt;
775 : DumpOptions *savDopt;
776 : int savVerbose,
777 : i;
778 :
779 4 : if (AH->mode == archModeWrite)
780 : {
781 : /*
782 : * Write the Header & TOC to the archive FIRST
783 : */
784 2 : th = tarOpen(AH, "toc.dat", 'w');
785 2 : ctx->FH = th;
786 2 : WriteHead(AH);
787 2 : WriteToc(AH);
788 2 : tarClose(AH, th); /* Not needed any more */
789 :
790 : /*
791 : * Now send the data (tables & LOs)
792 : */
793 2 : WriteDataChunks(AH, NULL);
794 :
795 : /*
796 : * Now this format wants to append a script which does a full restore
797 : * if the files have been extracted.
798 : */
799 2 : th = tarOpen(AH, "restore.sql", 'w');
800 :
801 2 : tarPrintf(th, "--\n"
802 : "-- NOTE:\n"
803 : "--\n"
804 : "-- File paths need to be edited. Search for $$PATH$$ and\n"
805 : "-- replace it with the path to the directory containing\n"
806 : "-- the extracted data files.\n"
807 : "--\n");
808 :
809 2 : AH->CustomOutPtr = _scriptOut;
810 :
811 2 : ctx->isSpecialScript = 1;
812 2 : ctx->scriptTH = th;
813 :
814 2 : ropt = NewRestoreOptions();
815 2 : memcpy(ropt, AH->public.ropt, sizeof(RestoreOptions));
816 2 : ropt->filename = NULL;
817 2 : ropt->dropSchema = 1;
818 2 : ropt->superuser = NULL;
819 2 : ropt->suppressDumpWarnings = true;
820 :
821 2 : savDopt = AH->public.dopt;
822 2 : savRopt = AH->public.ropt;
823 :
824 2 : SetArchiveOptions((Archive *) AH, NULL, ropt);
825 :
826 2 : savVerbose = AH->public.verbose;
827 2 : AH->public.verbose = 0;
828 :
829 2 : RestoreArchive((Archive *) AH);
830 :
831 2 : SetArchiveOptions((Archive *) AH, savDopt, savRopt);
832 :
833 2 : AH->public.verbose = savVerbose;
834 :
835 2 : tarClose(AH, th);
836 :
837 2 : ctx->isSpecialScript = 0;
838 :
839 : /*
840 : * EOF marker for tar files is two blocks of NULLs.
841 : */
842 2050 : for (i = 0; i < TAR_BLOCK_SIZE * 2; i++)
843 : {
844 2048 : if (fputc(0, ctx->tarFH) == EOF)
845 0 : WRITE_ERROR_EXIT;
846 : }
847 :
848 : /* Sync the output file if one is defined */
849 2 : if (AH->dosync && AH->fSpec)
850 1 : (void) fsync_fname(AH->fSpec, false);
851 : }
852 :
853 4 : AH->FH = NULL;
854 4 : }
855 :
856 : static size_t
857 2302 : _scriptOut(ArchiveHandle *AH, const void *buf, size_t len)
858 : {
859 2302 : lclContext *ctx = (lclContext *) AH->formatData;
860 :
861 2302 : return tarWrite(buf, len, ctx->scriptTH);
862 : }
863 :
864 : /*
865 : * Large Object support
866 : */
867 :
868 : /*
869 : * Called by the archiver when starting to save BLOB DATA (not schema).
870 : * This routine should save whatever format-specific information is needed
871 : * to read the LOs back into memory.
872 : *
873 : * It is called just prior to the dumper's DataDumper routine.
874 : *
875 : * Optional, but strongly recommended.
876 : *
877 : */
878 : static void
879 2 : _StartLOs(ArchiveHandle *AH, TocEntry *te)
880 : {
881 2 : lclContext *ctx = (lclContext *) AH->formatData;
882 : char fname[K_STD_BUF_SIZE];
883 :
884 2 : sprintf(fname, "blobs_%d.toc", te->dumpId);
885 2 : ctx->loToc = tarOpen(AH, fname, 'w');
886 2 : }
887 :
888 : /*
889 : * Called by the archiver when the dumper calls StartLO.
890 : *
891 : * Mandatory.
892 : *
893 : * Must save the passed OID for retrieval at restore-time.
894 : */
895 : static void
896 2 : _StartLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
897 : {
898 2 : lclContext *ctx = (lclContext *) AH->formatData;
899 2 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
900 : char fname[255];
901 :
902 2 : if (oid == 0)
903 0 : pg_fatal("invalid OID for large object (%u)", oid);
904 :
905 2 : if (AH->compression_spec.algorithm != PG_COMPRESSION_NONE)
906 0 : pg_fatal("compression is not supported by tar archive format");
907 :
908 2 : sprintf(fname, "blob_%u.dat", oid);
909 :
910 2 : tarPrintf(ctx->loToc, "%u %s\n", oid, fname);
911 :
912 2 : tctx->TH = tarOpen(AH, fname, 'w');
913 2 : }
914 :
915 : /*
916 : * Called by the archiver when the dumper calls EndLO.
917 : *
918 : * Optional.
919 : *
920 : */
921 : static void
922 2 : _EndLO(ArchiveHandle *AH, TocEntry *te, Oid oid)
923 : {
924 2 : lclTocEntry *tctx = (lclTocEntry *) te->formatData;
925 :
926 2 : tarClose(AH, tctx->TH);
927 2 : }
928 :
929 : /*
930 : * Called by the archiver when finishing saving BLOB DATA.
931 : *
932 : * Optional.
933 : *
934 : */
935 : static void
936 2 : _EndLOs(ArchiveHandle *AH, TocEntry *te)
937 : {
938 2 : lclContext *ctx = (lclContext *) AH->formatData;
939 :
940 : /* Write out a fake zero OID to mark end-of-LOs. */
941 : /* WriteInt(AH, 0); */
942 :
943 2 : tarClose(AH, ctx->loToc);
944 2 : }
945 :
946 :
947 :
948 : /*------------
949 : * TAR Support
950 : *------------
951 : */
952 :
953 : static int
954 4 : tarPrintf(TAR_MEMBER *th, const char *fmt,...)
955 : {
956 4 : int save_errno = errno;
957 : char *p;
958 4 : size_t len = 128; /* initial assumption about buffer size */
959 : size_t cnt;
960 :
961 : for (;;)
962 2 : {
963 : va_list args;
964 :
965 : /* Allocate work buffer. */
966 6 : p = (char *) pg_malloc(len);
967 :
968 : /* Try to format the data. */
969 6 : errno = save_errno;
970 6 : va_start(args, fmt);
971 6 : cnt = pvsnprintf(p, len, fmt, args);
972 6 : va_end(args);
973 :
974 6 : if (cnt < len)
975 4 : break; /* success */
976 :
977 : /* Release buffer and loop around to try again with larger len. */
978 2 : free(p);
979 2 : len = cnt;
980 : }
981 :
982 4 : cnt = tarWrite(p, cnt, th);
983 4 : free(p);
984 4 : return (int) cnt;
985 : }
986 :
987 : bool
988 1 : isValidTarHeader(char *header)
989 : {
990 : int sum;
991 1 : int chk = tarChecksum(header);
992 :
993 1 : sum = read_tar_number(&header[TAR_OFFSET_CHECKSUM], 8);
994 :
995 1 : if (sum != chk)
996 0 : return false;
997 :
998 : /* POSIX tar format */
999 1 : if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar\0", 6) == 0 &&
1000 1 : memcmp(&header[TAR_OFFSET_VERSION], "00", 2) == 0)
1001 1 : return true;
1002 : /* GNU tar format */
1003 0 : if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar \0", 8) == 0)
1004 0 : return true;
1005 : /* not-quite-POSIX format written by pre-9.3 pg_dump */
1006 0 : if (memcmp(&header[TAR_OFFSET_MAGIC], "ustar00\0", 8) == 0)
1007 0 : return true;
1008 :
1009 0 : return false;
1010 : }
1011 :
1012 : /* Given the member, write the TAR header & copy the file */
1013 : static void
1014 48 : _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
1015 : {
1016 48 : lclContext *ctx = (lclContext *) AH->formatData;
1017 48 : FILE *tmp = th->tmpFH; /* Grab it for convenience */
1018 : char buf[32768];
1019 : size_t cnt;
1020 48 : pgoff_t len = 0;
1021 : size_t res;
1022 : size_t i,
1023 : pad;
1024 :
1025 : /*
1026 : * Find file len & go back to start.
1027 : */
1028 48 : if (fseeko(tmp, 0, SEEK_END) != 0)
1029 0 : pg_fatal("error during file seek: %m");
1030 48 : th->fileLen = ftello(tmp);
1031 48 : if (th->fileLen < 0)
1032 0 : pg_fatal("could not determine seek position in archive file: %m");
1033 48 : if (fseeko(tmp, 0, SEEK_SET) != 0)
1034 0 : pg_fatal("error during file seek: %m");
1035 :
1036 48 : _tarWriteHeader(th);
1037 :
1038 101 : while ((cnt = fread(buf, 1, sizeof(buf), tmp)) > 0)
1039 : {
1040 53 : if ((res = fwrite(buf, 1, cnt, th->tarFH)) != cnt)
1041 0 : WRITE_ERROR_EXIT;
1042 53 : len += res;
1043 : }
1044 48 : if (!feof(tmp))
1045 0 : READ_ERROR_EXIT(tmp);
1046 :
1047 48 : if (fclose(tmp) != 0) /* This *should* delete it... */
1048 0 : pg_fatal("could not close temporary file: %m");
1049 :
1050 48 : if (len != th->fileLen)
1051 0 : pg_fatal("actual file length (%lld) does not match expected (%lld)",
1052 : (long long) len, (long long) th->fileLen);
1053 :
1054 48 : pad = tarPaddingBytesRequired(len);
1055 22521 : for (i = 0; i < pad; i++)
1056 : {
1057 22473 : if (fputc('\0', th->tarFH) == EOF)
1058 0 : WRITE_ERROR_EXIT;
1059 : }
1060 :
1061 48 : ctx->tarFHpos += len + pad;
1062 48 : }
1063 :
1064 : /* Locate the file in the archive, read header and position to data */
1065 : static TAR_MEMBER *
1066 46 : _tarPositionTo(ArchiveHandle *AH, const char *filename)
1067 : {
1068 46 : lclContext *ctx = (lclContext *) AH->formatData;
1069 46 : TAR_MEMBER *th = pg_malloc0_object(TAR_MEMBER);
1070 : char c;
1071 : char header[TAR_BLOCK_SIZE];
1072 : size_t i,
1073 : len,
1074 : blks;
1075 : int id;
1076 :
1077 46 : th->AH = AH;
1078 :
1079 : /* Go to end of current file, if any */
1080 46 : if (ctx->tarFHpos != 0)
1081 : {
1082 44 : pg_log_debug("moving from position %lld to next member at file position %lld",
1083 : (long long) ctx->tarFHpos, (long long) ctx->tarNextMember);
1084 :
1085 21017 : while (ctx->tarFHpos < ctx->tarNextMember)
1086 20973 : _tarReadRaw(AH, &c, 1, NULL, ctx->tarFH);
1087 : }
1088 :
1089 46 : pg_log_debug("now at file position %lld", (long long) ctx->tarFHpos);
1090 :
1091 : /* We are at the start of the file, or at the next member */
1092 :
1093 : /* Get the header */
1094 46 : if (!_tarGetHeader(AH, th))
1095 : {
1096 0 : if (filename)
1097 0 : pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1098 : else
1099 : {
1100 : /*
1101 : * We're just scanning the archive for the next file, so return
1102 : * null
1103 : */
1104 0 : free(th);
1105 0 : return NULL;
1106 : }
1107 : }
1108 :
1109 46 : while (filename != NULL && strcmp(th->targetFile, filename) != 0)
1110 : {
1111 0 : pg_log_debug("skipping tar member %s", th->targetFile);
1112 :
1113 0 : id = atoi(th->targetFile);
1114 0 : if ((TocIDRequired(AH, id) & REQ_DATA) != 0)
1115 0 : pg_fatal("restoring data out of order is not supported in this archive format: "
1116 : "\"%s\" is required, but comes before \"%s\" in the archive file.",
1117 : th->targetFile, filename);
1118 :
1119 : /* Header doesn't match, so read to next header */
1120 0 : len = th->fileLen;
1121 0 : len += tarPaddingBytesRequired(th->fileLen);
1122 0 : blks = len / TAR_BLOCK_SIZE; /* # of tar blocks */
1123 :
1124 0 : for (i = 0; i < blks; i++)
1125 0 : _tarReadRaw(AH, &header[0], TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1126 :
1127 0 : if (!_tarGetHeader(AH, th))
1128 0 : pg_fatal("could not find header for file \"%s\" in tar archive", filename);
1129 : }
1130 :
1131 92 : ctx->tarNextMember = ctx->tarFHpos + th->fileLen
1132 46 : + tarPaddingBytesRequired(th->fileLen);
1133 46 : th->pos = 0;
1134 :
1135 46 : return th;
1136 : }
1137 :
1138 : /* Read & verify a header */
1139 : static int
1140 46 : _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
1141 : {
1142 46 : lclContext *ctx = (lclContext *) AH->formatData;
1143 : char h[TAR_BLOCK_SIZE];
1144 : char tag[100 + 1];
1145 : int sum,
1146 : chk;
1147 : pgoff_t len;
1148 : pgoff_t hPos;
1149 46 : bool gotBlock = false;
1150 :
1151 92 : while (!gotBlock)
1152 : {
1153 : /* Save the pos for reporting purposes */
1154 46 : hPos = ctx->tarFHpos;
1155 :
1156 : /* Read the next tar block, return EOF, exit if short */
1157 46 : len = _tarReadRaw(AH, h, TAR_BLOCK_SIZE, NULL, ctx->tarFH);
1158 46 : if (len == 0) /* EOF */
1159 0 : return 0;
1160 :
1161 46 : if (len != TAR_BLOCK_SIZE)
1162 0 : pg_fatal(ngettext("incomplete tar header found (%lu byte)",
1163 : "incomplete tar header found (%lu bytes)",
1164 : len),
1165 : (unsigned long) len);
1166 :
1167 : /* Calc checksum */
1168 46 : chk = tarChecksum(h);
1169 46 : sum = read_tar_number(&h[TAR_OFFSET_CHECKSUM], 8);
1170 :
1171 : /*
1172 : * If the checksum failed, see if it is a null block. If so, silently
1173 : * continue to the next block.
1174 : */
1175 46 : if (chk == sum)
1176 46 : gotBlock = true;
1177 : else
1178 : {
1179 : int i;
1180 :
1181 0 : for (i = 0; i < TAR_BLOCK_SIZE; i++)
1182 : {
1183 0 : if (h[i] != 0)
1184 : {
1185 0 : gotBlock = true;
1186 0 : break;
1187 : }
1188 : }
1189 : }
1190 : }
1191 :
1192 : /* Name field is 100 bytes, might not be null-terminated */
1193 46 : strlcpy(tag, &h[TAR_OFFSET_NAME], 100 + 1);
1194 :
1195 46 : len = read_tar_number(&h[TAR_OFFSET_SIZE], 12);
1196 :
1197 46 : pg_log_debug("TOC Entry %s at %llu (length %llu, checksum %d)",
1198 : tag, (unsigned long long) hPos, (unsigned long long) len, sum);
1199 :
1200 46 : if (chk != sum)
1201 0 : pg_fatal("corrupt tar header found in %s (expected %d, computed %d) file position %llu",
1202 : tag, sum, chk, (unsigned long long) ftello(ctx->tarFH));
1203 :
1204 46 : th->targetFile = pg_strdup(tag);
1205 46 : th->fileLen = len;
1206 :
1207 46 : return 1;
1208 : }
1209 :
1210 :
1211 : static void
1212 48 : _tarWriteHeader(TAR_MEMBER *th)
1213 : {
1214 : char h[TAR_BLOCK_SIZE];
1215 :
1216 48 : tarCreateHeader(h, th->targetFile, NULL, th->fileLen,
1217 : 0600, 04000, 02000, time(NULL));
1218 :
1219 : /* Now write the completed header. */
1220 48 : if (fwrite(h, 1, TAR_BLOCK_SIZE, th->tarFH) != TAR_BLOCK_SIZE)
1221 0 : WRITE_ERROR_EXIT;
1222 48 : }
|
