Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * astreamer.h
4 : : *
5 : : * The "archive streamer" interface is intended to allow frontend code
6 : : * to stream from possibly-compressed archive files from any source and
7 : : * perform arbitrary actions based on the contents of those archives.
8 : : * Archive streamers are intended to be composable, and most tasks will
9 : : * require two or more archive streamers to complete. For instance,
10 : : * if the input is an uncompressed tar stream, a tar parser astreamer
11 : : * could be used to interpret it, and then an extractor astreamer could
12 : : * be used to write each archive member out to a file.
13 : : *
14 : : * In general, each archive streamer is relatively free to take whatever
15 : : * action it desires in the stream of chunks provided by the caller. It
16 : : * may do something simple, like write the archive to a file, perhaps after
17 : : * compressing it, but it can also do more complicated things, like
18 : : * annotating the byte stream to indicate which parts of the data
19 : : * correspond to tar headers or trailing padding, vs. which parts are
20 : : * payload data. A subsequent astreamer may use this information to
21 : : * make further decisions about how to process the data; for example,
22 : : * it might choose to modify the archive contents.
23 : : *
24 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
25 : : *
26 : : * IDENTIFICATION
27 : : * src/include/fe_utils/astreamer.h
28 : : *-------------------------------------------------------------------------
29 : : */
30 : :
31 : : #ifndef ASTREAMER_H
32 : : #define ASTREAMER_H
33 : :
34 : : #include "common/compression.h"
35 : : #include "lib/stringinfo.h"
36 : : #include "pqexpbuffer.h"
37 : :
38 : : struct astreamer;
39 : : struct astreamer_ops;
40 : : typedef struct astreamer astreamer;
41 : : typedef struct astreamer_ops astreamer_ops;
42 : :
43 : : /*
44 : : * Each chunk of archive data passed to a astreamer is classified into one
45 : : * of these categories. When data is initially passed to an archive streamer,
46 : : * each chunk will be categorized as ASTREAMER_UNKNOWN, and the chunks can
47 : : * be of whatever size the caller finds convenient.
48 : : *
49 : : * If the archive is parsed (e.g. see astreamer_tar_parser_new()), then all
50 : : * chunks should be labelled as one of the other types listed here. In
51 : : * addition, there should be exactly one ASTREAMER_MEMBER_HEADER chunk and
52 : : * exactly one ASTREAMER_MEMBER_TRAILER chunk per archive member, even if
53 : : * that means a zero-length call. There can be any number of
54 : : * ASTREAMER_MEMBER_CONTENTS chunks in between those calls. There
55 : : * should exactly ASTREAMER_ARCHIVE_TRAILER chunk, and it should follow the
56 : : * last ASTREAMER_MEMBER_TRAILER chunk.
57 : : *
58 : : * In theory, we could need other classifications here, such as a way of
59 : : * indicating an archive header, but the "tar" format doesn't need anything
60 : : * else, so for the time being there's no point.
61 : : */
62 : : typedef enum
63 : : {
64 : : ASTREAMER_UNKNOWN,
65 : : ASTREAMER_MEMBER_HEADER,
66 : : ASTREAMER_MEMBER_CONTENTS,
67 : : ASTREAMER_MEMBER_TRAILER,
68 : : ASTREAMER_ARCHIVE_TRAILER,
69 : : } astreamer_archive_context;
70 : :
71 : : /*
72 : : * State of the most recently processed compressed frame. When decompression
73 : : * requires more input data to complete, or a bigger output buffer to store
74 : : * the result, the state is set to ASTREAMER_FRAME_INCOMPLETE. Exactly how to
75 : : * resolve an incomplete state is compression library dependent. Before a
76 : : * stream has decompressed any frames, its state is ASTREAMER_STREAM_NEW.
77 : : */
78 : : typedef enum
79 : : {
80 : : ASTREAMER_STREAM_NEW,
81 : : ASTREAMER_FRAME_INCOMPLETE,
82 : : ASTREAMER_FRAME_COMPLETE,
83 : : } astreamer_decompression_state;
84 : :
85 : : /*
86 : : * Each chunk of data that is classified as ASTREAMER_MEMBER_HEADER,
87 : : * ASTREAMER_MEMBER_CONTENTS, or ASTREAMER_MEMBER_TRAILER should also
88 : : * pass a pointer to an instance of this struct. The details are expected
89 : : * to be present in the archive header and used to fill the struct, after
90 : : * which all subsequent calls for the same archive member are expected to
91 : : * pass the same details.
92 : : */
93 : : typedef struct
94 : : {
95 : : char pathname[MAXPGPATH];
96 : : pgoff_t size;
97 : : mode_t mode;
98 : : uid_t uid;
99 : : gid_t gid;
100 : : /* note: special filetypes will set none of these flags */
101 : : bool is_regular;
102 : : bool is_directory;
103 : : bool is_symlink;
104 : : char linktarget[MAXPGPATH];
105 : : } astreamer_member;
106 : :
107 : : /*
108 : : * Generally, each type of astreamer will define its own struct, but the
109 : : * first element should be 'astreamer base'. A astreamer that does not
110 : : * require any additional private data could use this structure directly.
111 : : *
112 : : * bbs_ops is a pointer to the astreamer_ops object which contains the
113 : : * function pointers appropriate to this type of astreamer.
114 : : *
115 : : * bbs_next is a pointer to the successor astreamer, for those types of
116 : : * astreamer which forward data to a successor. It need not be used and
117 : : * should be set to NULL when not relevant.
118 : : *
119 : : * bbs_buffer is a buffer for accumulating data for temporary storage. Each
120 : : * type of astreamer makes its own decisions about whether and how to use
121 : : * this buffer.
122 : : */
123 : : struct astreamer
124 : : {
125 : : const astreamer_ops *bbs_ops;
126 : : astreamer *bbs_next;
127 : : StringInfoData bbs_buffer;
128 : : };
129 : :
130 : : /*
131 : : * There are three callbacks for a astreamer. The 'content' callback is
132 : : * called repeatedly, as described in the astreamer_archive_context comments.
133 : : * Then, the 'finalize' callback is called once at the end, to give the
134 : : * astreamer a chance to perform cleanup such as closing files. Finally,
135 : : * because this code is running in a frontend environment where, as of this
136 : : * writing, there are no memory contexts, the 'free' callback is called to
137 : : * release memory. These callbacks should always be invoked using the static
138 : : * inline functions defined below.
139 : : */
140 : : struct astreamer_ops
141 : : {
142 : : void (*content) (astreamer *streamer, astreamer_member *member,
143 : : const char *data, int len,
144 : : astreamer_archive_context context);
145 : : void (*finalize) (astreamer *streamer);
146 : : void (*free) (astreamer *streamer);
147 : : };
148 : :
149 : : /* Send some content to a astreamer. */
150 : : static inline void
798 rhaas@postgresql.org 151 :CBC 1063561 : astreamer_content(astreamer *streamer, astreamer_member *member,
152 : : const char *data, int len,
153 : : astreamer_archive_context context)
154 : : {
155 [ - + ]: 1063561 : Assert(streamer != NULL);
156 : 1063561 : streamer->bbs_ops->content(streamer, member, data, len, context);
157 : 1063559 : }
158 : :
159 : : /* Finalize a astreamer. */
160 : : static inline void
161 : 489 : astreamer_finalize(astreamer *streamer)
162 : : {
163 [ - + ]: 489 : Assert(streamer != NULL);
164 : 489 : streamer->bbs_ops->finalize(streamer);
165 : 486 : }
166 : :
167 : : /* Free a astreamer. */
168 : : static inline void
169 : 604 : astreamer_free(astreamer *streamer)
170 : : {
171 [ - + ]: 604 : Assert(streamer != NULL);
172 : 604 : streamer->bbs_ops->free(streamer);
173 : 604 : }
174 : :
175 : : /*
176 : : * This is a convenience method for use when implementing a astreamer; it is
177 : : * not for use by outside callers. It adds the amount of data specified by
178 : : * 'nbytes' to the astreamer's buffer and adjusts '*len' and '*data'
179 : : * accordingly.
180 : : */
181 : : static inline void
182 : 209795 : astreamer_buffer_bytes(astreamer *streamer, const char **data, int *len,
183 : : int nbytes)
184 : : {
185 [ - + ]: 209795 : Assert(nbytes <= *len);
186 : :
187 : 209795 : appendBinaryStringInfo(&streamer->bbs_buffer, *data, nbytes);
188 : 209795 : *len -= nbytes;
189 : 209795 : *data += nbytes;
190 : 209795 : }
191 : :
192 : : /*
193 : : * This is a convenience method for use when implementing a astreamer; it is
194 : : * not for use by outsider callers. It attempts to add enough data to the
195 : : * astreamer's buffer to reach a length of target_bytes and adjusts '*len'
196 : : * and '*data' accordingly. It returns true if the target length has been
197 : : * reached and false otherwise.
198 : : */
199 : : static inline bool
200 : 209573 : astreamer_buffer_until(astreamer *streamer, const char **data, int *len,
201 : : int target_bytes)
202 : : {
203 : 209573 : int buflen = streamer->bbs_buffer.len;
204 : :
205 [ - + ]: 209573 : if (buflen >= target_bytes)
206 : : {
207 : : /* Target length already reached; nothing to do. */
798 rhaas@postgresql.org 208 :UBC 0 : return true;
209 : : }
210 : :
798 rhaas@postgresql.org 211 [ - + ]:CBC 209573 : if (buflen + *len < target_bytes)
212 : : {
213 : : /* Not enough data to reach target length; buffer all of it. */
798 rhaas@postgresql.org 214 :UBC 0 : astreamer_buffer_bytes(streamer, data, len, *len);
215 : 0 : return false;
216 : : }
217 : :
218 : : /* Buffer just enough to reach the target length. */
798 rhaas@postgresql.org 219 :CBC 209573 : astreamer_buffer_bytes(streamer, data, len, target_bytes - buflen);
220 : 209573 : return true;
221 : : }
222 : :
223 : : /*
224 : : * Functions for creating astreamer objects of various types. See the header
225 : : * comments for each of these functions for details.
226 : : */
227 : : extern astreamer *astreamer_plain_writer_new(char *pathname, FILE *file);
228 : : extern astreamer *astreamer_gzip_writer_new(char *pathname, FILE *file,
229 : : pg_compress_specification *compress);
230 : : extern astreamer *astreamer_extractor_new(const char *basepath,
231 : : const char *(*link_map) (const char *),
232 : : void (*report_output_file) (const char *));
233 : :
234 : : extern astreamer *astreamer_gzip_decompressor_new(astreamer *next);
235 : : extern astreamer *astreamer_lz4_compressor_new(astreamer *next,
236 : : pg_compress_specification *compress);
237 : : extern astreamer *astreamer_lz4_decompressor_new(astreamer *next);
238 : : extern astreamer *astreamer_zstd_compressor_new(astreamer *next,
239 : : pg_compress_specification *compress);
240 : : extern astreamer *astreamer_zstd_decompressor_new(astreamer *next);
241 : : extern astreamer *astreamer_tar_parser_new(astreamer *next);
242 : : extern astreamer *astreamer_tar_terminator_new(astreamer *next);
243 : : extern astreamer *astreamer_tar_archiver_new(astreamer *next);
244 : :
245 : : #endif
|
