Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * compress_io.c
4 : : * Routines for archivers to write an uncompressed or compressed data
5 : : * stream.
6 : : *
7 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
8 : : * Portions Copyright (c) 1994, Regents of the University of California
9 : : *
10 : : * This file includes two APIs for dealing with compressed data. The first
11 : : * provides more flexibility, using callbacks to read/write data from the
12 : : * underlying stream. The second API is a wrapper around fopen and
13 : : * friends, providing an interface similar to those, but abstracts away
14 : : * the possible compression. The second API is aimed for the resulting
15 : : * files to be easily manipulated with an external compression utility
16 : : * program.
17 : : *
18 : : * Compressor API
19 : : * --------------
20 : : *
21 : : * The interface for writing to an archive consists of three functions:
22 : : * AllocateCompressor, writeData, and EndCompressor. First you call
23 : : * AllocateCompressor, then write all the data by calling writeData as many
24 : : * times as needed, and finally EndCompressor. writeData will call the
25 : : * WriteFunc that was provided to AllocateCompressor for each chunk of
26 : : * compressed data.
27 : : *
28 : : * The interface for reading an archive consists of the same three functions:
29 : : * AllocateCompressor, readData, and EndCompressor. First you call
30 : : * AllocateCompressor, then read all the data by calling readData to read the
31 : : * whole compressed stream which repeatedly calls the given ReadFunc. ReadFunc
32 : : * returns the compressed data one chunk at a time. Then readData decompresses
33 : : * it and passes the decompressed data to ahwrite(), until ReadFunc returns 0
34 : : * to signal EOF. The interface is the same for compressed and uncompressed
35 : : * streams.
36 : : *
37 : : * Compressed stream API
38 : : * ----------------------
39 : : *
40 : : * The compressed stream API is providing a set of function pointers for
41 : : * opening, reading, writing, and finally closing files. The implemented
42 : : * function pointers are documented in the corresponding header file and are
43 : : * common for all streams. It allows the caller to use the same functions for
44 : : * both compressed and uncompressed streams.
45 : : *
46 : : * The interface consists of three functions, InitCompressFileHandle,
47 : : * InitDiscoverCompressFileHandle, and EndCompressFileHandle. If the
48 : : * compression is known, then start by calling InitCompressFileHandle,
49 : : * otherwise discover it by using InitDiscoverCompressFileHandle. Then call
50 : : * the function pointers as required for the read/write operations. Finally
51 : : * call EndCompressFileHandle to end the stream.
52 : : *
53 : : * InitDiscoverCompressFileHandle tries to infer the compression by the
54 : : * filename suffix. If the suffix is not yet known then it tries to simply
55 : : * open the file and if it fails, it tries to open the same file with
56 : : * compressed suffixes (.gz, .lz4 and .zst, in this order).
57 : : *
58 : : * IDENTIFICATION
59 : : * src/bin/pg_dump/compress_io.c
60 : : *
61 : : *-------------------------------------------------------------------------
62 : : */
63 : : #include "postgres_fe.h"
64 : :
65 : : #include <sys/stat.h>
66 : : #include <unistd.h>
67 : :
68 : : #include "compress_gzip.h"
69 : : #include "compress_io.h"
70 : : #include "compress_lz4.h"
71 : : #include "compress_none.h"
72 : : #include "compress_zstd.h"
73 : : #include "pg_backup_utils.h"
74 : :
75 : : /*----------------------
76 : : * Generic functions
77 : : *----------------------
78 : : */
79 : :
80 : : /*
81 : : * Checks whether support for a compression algorithm is implemented in
82 : : * pg_dump/restore.
83 : : *
84 : : * On success returns NULL, otherwise returns a malloc'ed string which can be
85 : : * used by the caller in an error message.
86 : : */
87 : : char *
88 : 479 : supports_compression(const pg_compress_specification compression_spec)
89 : : {
90 : 479 : const pg_compress_algorithm algorithm = compression_spec.algorithm;
91 : 479 : bool supported = false;
92 : :
93 [ + + ]: 479 : if (algorithm == PG_COMPRESSION_NONE)
94 : 268 : supported = true;
95 : : #ifdef HAVE_LIBZ
96 [ + + ]: 479 : if (algorithm == PG_COMPRESSION_GZIP)
97 : 200 : supported = true;
98 : : #endif
99 : : #ifdef USE_LZ4
100 [ + + ]: 479 : if (algorithm == PG_COMPRESSION_LZ4)
101 : 11 : supported = true;
102 : : #endif
103 : : #ifdef USE_ZSTD
104 : : if (algorithm == PG_COMPRESSION_ZSTD)
105 : : supported = true;
106 : : #endif
107 : :
108 [ - + ]: 479 : if (!supported)
109 : 0 : return psprintf(_("this build does not support compression with %s"),
110 : : get_compress_algorithm_name(algorithm));
111 : :
112 : 479 : return NULL;
113 : : }
114 : :
115 : : /*----------------------
116 : : * Compressor API
117 : : *----------------------
118 : : */
119 : :
120 : : /*
121 : : * Allocate a new compressor.
122 : : */
123 : : CompressorState *
124 : 496 : AllocateCompressor(const pg_compress_specification compression_spec,
125 : : ReadFunc readF, WriteFunc writeF)
126 : : {
127 : : CompressorState *cs;
128 : :
129 : 496 : cs = pg_malloc0_object(CompressorState);
130 : 496 : cs->readF = readF;
131 : 496 : cs->writeF = writeF;
132 : :
133 [ + + ]: 496 : if (compression_spec.algorithm == PG_COMPRESSION_NONE)
134 : 6 : InitCompressorNone(cs, compression_spec);
135 [ + + ]: 490 : else if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
136 : 483 : InitCompressorGzip(cs, compression_spec);
137 [ + - ]: 7 : else if (compression_spec.algorithm == PG_COMPRESSION_LZ4)
138 : 7 : InitCompressorLZ4(cs, compression_spec);
139 [ # # ]: 0 : else if (compression_spec.algorithm == PG_COMPRESSION_ZSTD)
140 : 0 : InitCompressorZstd(cs, compression_spec);
141 : :
142 : 496 : return cs;
143 : : }
144 : :
145 : : /*
146 : : * Terminate compression library context and flush its buffers.
147 : : */
148 : : void
149 : 495 : EndCompressor(ArchiveHandle *AH, CompressorState *cs)
150 : : {
151 : 495 : cs->end(AH, cs);
152 : 495 : pg_free(cs);
153 : 495 : }
154 : :
155 : : /*----------------------
156 : : * Compressed stream API
157 : : *----------------------
158 : : */
159 : :
160 : : /*
161 : : * Private routines
162 : : */
163 : : static int
164 : 465 : hasSuffix(const char *filename, const char *suffix)
165 : : {
166 : 465 : int filenamelen = strlen(filename);
167 : 465 : int suffixlen = strlen(suffix);
168 : :
169 [ - + ]: 465 : if (filenamelen < suffixlen)
170 : 0 : return 0;
171 : :
172 : 465 : return memcmp(&filename[filenamelen - suffixlen],
173 : : suffix,
174 : 465 : suffixlen) == 0;
175 : : }
176 : :
177 : : /* free() without changing errno; useful in several places below */
178 : : static void
179 : 1047 : free_keep_errno(void *p)
180 : : {
181 : 1047 : int save_errno = errno;
182 : :
183 : 1047 : free(p);
184 : 1047 : errno = save_errno;
185 : 1047 : }
186 : :
187 : : /*
188 : : * Public interface
189 : : */
190 : :
191 : : /*
192 : : * Initialize a compress file handle for the specified compression algorithm.
193 : : */
194 : : CompressFileHandle *
195 : 781 : InitCompressFileHandle(const pg_compress_specification compression_spec)
196 : : {
197 : : CompressFileHandle *CFH;
198 : :
199 : 781 : CFH = pg_malloc0_object(CompressFileHandle);
200 : :
201 [ + + ]: 781 : if (compression_spec.algorithm == PG_COMPRESSION_NONE)
202 : 516 : InitCompressFileHandleNone(CFH, compression_spec);
203 [ + + ]: 265 : else if (compression_spec.algorithm == PG_COMPRESSION_GZIP)
204 : 260 : InitCompressFileHandleGzip(CFH, compression_spec);
205 [ + - ]: 5 : else if (compression_spec.algorithm == PG_COMPRESSION_LZ4)
206 : 5 : InitCompressFileHandleLZ4(CFH, compression_spec);
207 [ # # ]: 0 : else if (compression_spec.algorithm == PG_COMPRESSION_ZSTD)
208 : 0 : InitCompressFileHandleZstd(CFH, compression_spec);
209 : :
210 : 781 : return CFH;
211 : : }
212 : :
213 : : /*
214 : : * Checks if a compressed file (with the specified extension) exists.
215 : : *
216 : : * The filename of the tested file is stored to fname buffer (the existing
217 : : * buffer is freed, new buffer is allocated and returned through the pointer).
218 : : */
219 : : static bool
220 : 137 : check_compressed_file(const char *path, char **fname, char *ext)
221 : : {
222 : 137 : free_keep_errno(*fname);
223 : 137 : *fname = psprintf("%s.%s", path, ext);
224 : 137 : return (access(*fname, F_OK) == 0);
225 : : }
226 : :
227 : : /*
228 : : * Open a file for reading. 'path' is the file to open, and 'mode' should
229 : : * be either "r" or "rb".
230 : : *
231 : : * If the file at 'path' contains the suffix of a supported compression method,
232 : : * currently this includes ".gz", ".lz4" and ".zst", then this compression will be used
233 : : * throughout. Otherwise the compression will be inferred by iteratively trying
234 : : * to open the file at 'path', first as is, then by appending known compression
235 : : * suffixes. So if you pass "foo" as 'path', this will open either "foo" or
236 : : * "foo.{gz,lz4,zst}", trying in that order.
237 : : *
238 : : * On failure, return NULL with an error code in errno.
239 : : */
240 : : CompressFileHandle *
241 : 155 : InitDiscoverCompressFileHandle(const char *path, const char *mode)
242 : : {
243 : 155 : CompressFileHandle *CFH = NULL;
244 : : struct stat st;
245 : : char *fname;
246 : 155 : pg_compress_specification compression_spec = {0};
247 : :
248 : 155 : compression_spec.algorithm = PG_COMPRESSION_NONE;
249 : :
250 : : Assert(strcmp(mode, PG_BINARY_R) == 0);
251 : :
252 : 155 : fname = pg_strdup(path);
253 : :
254 [ - + ]: 155 : if (hasSuffix(fname, ".gz"))
255 : 0 : compression_spec.algorithm = PG_COMPRESSION_GZIP;
256 [ - + ]: 155 : else if (hasSuffix(fname, ".lz4"))
257 : 0 : compression_spec.algorithm = PG_COMPRESSION_LZ4;
258 [ - + ]: 155 : else if (hasSuffix(fname, ".zst"))
259 : 0 : compression_spec.algorithm = PG_COMPRESSION_ZSTD;
260 : : else
261 : : {
262 [ + + ]: 155 : if (stat(path, &st) == 0)
263 : 19 : compression_spec.algorithm = PG_COMPRESSION_NONE;
264 [ + + ]: 136 : else if (check_compressed_file(path, &fname, "gz"))
265 : 135 : compression_spec.algorithm = PG_COMPRESSION_GZIP;
266 [ + - ]: 1 : else if (check_compressed_file(path, &fname, "lz4"))
267 : 1 : compression_spec.algorithm = PG_COMPRESSION_LZ4;
268 [ # # ]: 0 : else if (check_compressed_file(path, &fname, "zst"))
269 : 0 : compression_spec.algorithm = PG_COMPRESSION_ZSTD;
270 : : }
271 : :
272 : : /*
273 : : * Refuse anything that isn't a regular file, so that a symbolic link
274 : : * can't lead us outside the archive directory. This check is racy, but
275 : : * we're guarding against a compromised dump, not against a directory
276 : : * under the active control of a hostile party.
277 : : */
278 [ + - - + ]: 155 : if (lstat(fname, &st) == 0 && !S_ISREG(st.st_mode))
279 : 0 : pg_fatal("\"%s\" is not a regular file", fname);
280 : :
281 : : #ifdef WIN32
282 : :
283 : : /*
284 : : * lstat() only recognizes junction points on Windows, so check the name's
285 : : * reparse tag ourselves to catch symbolic links to files, too.
286 : : */
287 : : {
288 : : WIN32_FIND_DATA fd;
289 : : HANDLE hFind = FindFirstFile(fname, &fd);
290 : :
291 : : if (hFind != INVALID_HANDLE_VALUE)
292 : : {
293 : : FindClose(hFind);
294 : : if ((fd.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0 &&
295 : : (fd.dwReserved0 == IO_REPARSE_TAG_SYMLINK ||
296 : : fd.dwReserved0 == IO_REPARSE_TAG_MOUNT_POINT))
297 : : pg_fatal("\"%s\" is not a regular file", fname);
298 : : }
299 : : }
300 : : #endif
301 : :
302 : 155 : CFH = InitCompressFileHandle(compression_spec);
303 : 155 : errno = 0;
304 [ - + ]: 155 : if (!CFH->open_func(fname, -1, mode, CFH))
305 : : {
306 : 0 : free_keep_errno(CFH);
307 : 0 : CFH = NULL;
308 : : }
309 : 155 : free_keep_errno(fname);
310 : :
311 : 155 : return CFH;
312 : : }
313 : :
314 : : /*
315 : : * Close an open file handle and release its memory.
316 : : *
317 : : * On failure, returns false and sets errno appropriately.
318 : : */
319 : : bool
320 : 755 : EndCompressFileHandle(CompressFileHandle *CFH)
321 : : {
322 : 755 : bool ret = false;
323 : :
324 : 755 : errno = 0;
325 [ + - ]: 755 : if (CFH->private_data)
326 : 755 : ret = CFH->close_func(CFH);
327 : :
328 : 755 : free_keep_errno(CFH);
329 : :
330 : 755 : return ret;
331 : : }
|
