Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * backup_manifest.c
4 : * code for generating and sending a backup manifest
5 : *
6 : * Portions Copyright (c) 2010-2023, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/backup/backup_manifest.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include "access/timeline.h"
16 : #include "backup/backup_manifest.h"
17 : #include "backup/basebackup_sink.h"
18 : #include "libpq/libpq.h"
19 : #include "libpq/pqformat.h"
20 : #include "mb/pg_wchar.h"
21 : #include "utils/builtins.h"
22 : #include "utils/json.h"
23 :
24 : static void AppendStringToManifest(backup_manifest_info *manifest, const char *s);
25 :
26 : /*
27 : * Does the user want a backup manifest?
28 : *
29 : * It's simplest to always have a manifest_info object, so that we don't need
30 : * checks for NULL pointers in too many places. However, if the user doesn't
31 : * want a manifest, we set manifest->buffile to NULL.
32 : */
33 : static inline bool
34 215552 : IsManifestEnabled(backup_manifest_info *manifest)
35 : {
36 215552 : return (manifest->buffile != NULL);
37 : }
38 :
39 : /*
40 : * Convenience macro for appending data to the backup manifest.
41 : */
42 : #define AppendToManifest(manifest, ...) \
43 : { \
44 : char *_manifest_s = psprintf(__VA_ARGS__); \
45 : AppendStringToManifest(manifest, _manifest_s); \
46 : pfree(_manifest_s); \
47 : }
48 :
49 : /*
50 : * Initialize state so that we can construct a backup manifest.
51 : *
52 : * NB: Although the checksum type for the data files is configurable, the
53 : * checksum for the manifest itself always uses SHA-256. See comments in
54 : * SendBackupManifest.
55 : */
56 : void
57 230 : InitializeBackupManifest(backup_manifest_info *manifest,
58 : backup_manifest_option want_manifest,
59 : pg_checksum_type manifest_checksum_type)
60 : {
61 230 : memset(manifest, 0, sizeof(backup_manifest_info));
62 230 : manifest->checksum_type = manifest_checksum_type;
63 :
64 230 : if (want_manifest == MANIFEST_OPTION_NO)
65 2 : manifest->buffile = NULL;
66 : else
67 : {
68 228 : manifest->buffile = BufFileCreateTemp(false);
69 228 : manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
70 228 : if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
71 0 : elog(ERROR, "failed to initialize checksum of backup manifest: %s",
72 : pg_cryptohash_error(manifest->manifest_ctx));
73 : }
74 :
75 230 : manifest->manifest_size = UINT64CONST(0);
76 230 : manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
77 230 : manifest->first_file = true;
78 230 : manifest->still_checksumming = true;
79 :
80 230 : if (want_manifest != MANIFEST_OPTION_NO)
81 228 : AppendToManifest(manifest,
82 : "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
83 : "\"Files\": [");
84 230 : }
85 :
86 : /*
87 : * Free resources assigned to a backup manifest constructed.
88 : */
89 : void
90 212 : FreeBackupManifest(backup_manifest_info *manifest)
91 : {
92 212 : pg_cryptohash_free(manifest->manifest_ctx);
93 212 : manifest->manifest_ctx = NULL;
94 212 : }
95 :
96 : /*
97 : * Add an entry to the backup manifest for a file.
98 : */
99 : void
100 215116 : AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid,
101 : const char *pathname, size_t size, pg_time_t mtime,
102 : pg_checksum_context *checksum_ctx)
103 : {
104 : char pathbuf[MAXPGPATH];
105 : int pathlen;
106 : StringInfoData buf;
107 :
108 215116 : if (!IsManifestEnabled(manifest))
109 1934 : return;
110 :
111 : /*
112 : * If this file is part of a tablespace, the pathname passed to this
113 : * function will be relative to the tar file that contains it. We want the
114 : * pathname relative to the data directory (ignoring the intermediate
115 : * symlink traversal).
116 : */
117 213182 : if (spcoid != NULL)
118 : {
119 32 : snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
120 : pathname);
121 32 : pathname = pathbuf;
122 : }
123 :
124 : /*
125 : * Each file's entry needs to be separated from any entry that follows by
126 : * a comma, but there's no comma before the first one or after the last
127 : * one. To make that work, adding a file to the manifest starts by
128 : * terminating the most recently added line, with a comma if appropriate,
129 : * but does not terminate the line inserted for this file.
130 : */
131 213182 : initStringInfo(&buf);
132 213182 : if (manifest->first_file)
133 : {
134 228 : appendStringInfoChar(&buf, '\n');
135 228 : manifest->first_file = false;
136 : }
137 : else
138 212954 : appendStringInfoString(&buf, ",\n");
139 :
140 : /*
141 : * Write the relative pathname to this file out to the manifest. The
142 : * manifest is always stored in UTF-8, so we have to encode paths that are
143 : * not valid in that encoding.
144 : */
145 213182 : pathlen = strlen(pathname);
146 424432 : if (!manifest->force_encode &&
147 211250 : pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
148 : {
149 211250 : appendStringInfoString(&buf, "{ \"Path\": ");
150 211250 : escape_json(&buf, pathname);
151 211250 : appendStringInfoString(&buf, ", ");
152 : }
153 : else
154 : {
155 1932 : appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
156 1932 : enlargeStringInfo(&buf, 2 * pathlen);
157 3864 : buf.len += hex_encode(pathname, pathlen,
158 1932 : &buf.data[buf.len]);
159 1932 : appendStringInfoString(&buf, "\", ");
160 : }
161 :
162 213182 : appendStringInfo(&buf, "\"Size\": %zu, ", size);
163 :
164 : /*
165 : * Convert last modification time to a string and append it to the
166 : * manifest. Since it's not clear what time zone to use and since time
167 : * zone definitions can change, possibly causing confusion, use GMT
168 : * always.
169 : */
170 213182 : appendStringInfoString(&buf, "\"Last-Modified\": \"");
171 213182 : enlargeStringInfo(&buf, 128);
172 213182 : buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
173 213182 : pg_gmtime(&mtime));
174 213182 : appendStringInfoChar(&buf, '"');
175 :
176 : /* Add checksum information. */
177 213182 : if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
178 : {
179 : uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
180 : int checksumlen;
181 :
182 211250 : checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
183 211250 : if (checksumlen < 0)
184 0 : elog(ERROR, "could not finalize checksum of file \"%s\"",
185 : pathname);
186 :
187 211250 : appendStringInfo(&buf,
188 : ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
189 : pg_checksum_type_name(checksum_ctx->type));
190 211250 : enlargeStringInfo(&buf, 2 * checksumlen);
191 422500 : buf.len += hex_encode((char *) checksumbuf, checksumlen,
192 211250 : &buf.data[buf.len]);
193 211250 : appendStringInfoChar(&buf, '"');
194 : }
195 :
196 : /* Close out the object. */
197 213182 : appendStringInfoString(&buf, " }");
198 :
199 : /* OK, add it to the manifest. */
200 213182 : AppendStringToManifest(manifest, buf.data);
201 :
202 : /* Avoid leaking memory. */
203 213182 : pfree(buf.data);
204 : }
205 :
206 : /*
207 : * Add information about the WAL that will need to be replayed when restoring
208 : * this backup to the manifest.
209 : */
210 : void
211 218 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
212 : TimeLineID starttli, XLogRecPtr endptr,
213 : TimeLineID endtli)
214 : {
215 : List *timelines;
216 : ListCell *lc;
217 218 : bool first_wal_range = true;
218 218 : bool found_start_timeline = false;
219 :
220 218 : if (!IsManifestEnabled(manifest))
221 2 : return;
222 :
223 : /* Terminate the list of files. */
224 216 : AppendStringToManifest(manifest, "\n],\n");
225 :
226 : /* Read the timeline history for the ending timeline. */
227 216 : timelines = readTimeLineHistory(endtli);
228 :
229 : /* Start a list of LSN ranges. */
230 216 : AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
231 :
232 216 : foreach(lc, timelines)
233 : {
234 216 : TimeLineHistoryEntry *entry = lfirst(lc);
235 : XLogRecPtr tl_beginptr;
236 :
237 : /*
238 : * We only care about timelines that were active during the backup.
239 : * Skip any that ended before the backup started. (Note that if
240 : * entry->end is InvalidXLogRecPtr, it means that the timeline has not
241 : * yet ended.)
242 : */
243 216 : if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
244 0 : continue;
245 :
246 : /*
247 : * Because the timeline history file lists newer timelines before
248 : * older ones, the first timeline we encounter that is new enough to
249 : * matter ought to match the ending timeline of the backup.
250 : */
251 216 : if (first_wal_range && endtli != entry->tli)
252 0 : ereport(ERROR,
253 : errmsg("expected end timeline %u but found timeline %u",
254 : starttli, entry->tli));
255 :
256 : /*
257 : * If this timeline entry matches with the timeline on which the
258 : * backup started, WAL needs to be checked from the start LSN of the
259 : * backup. If this entry refers to a newer timeline, WAL needs to be
260 : * checked since the beginning of this timeline, so use the LSN where
261 : * the timeline began.
262 : */
263 216 : if (starttli == entry->tli)
264 216 : tl_beginptr = startptr;
265 : else
266 : {
267 0 : tl_beginptr = entry->begin;
268 :
269 : /*
270 : * If we reach a TLI that has no valid beginning LSN, there can't
271 : * be any more timelines in the history after this point, so we'd
272 : * better have arrived at the expected starting TLI. If not,
273 : * something's gone horribly wrong.
274 : */
275 0 : if (XLogRecPtrIsInvalid(entry->begin))
276 0 : ereport(ERROR,
277 : errmsg("expected start timeline %u but found timeline %u",
278 : starttli, entry->tli));
279 : }
280 :
281 216 : AppendToManifest(manifest,
282 : "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
283 : first_wal_range ? "" : ",\n",
284 : entry->tli,
285 : LSN_FORMAT_ARGS(tl_beginptr),
286 : LSN_FORMAT_ARGS(endptr));
287 :
288 216 : if (starttli == entry->tli)
289 : {
290 216 : found_start_timeline = true;
291 216 : break;
292 : }
293 :
294 0 : endptr = entry->begin;
295 0 : first_wal_range = false;
296 : }
297 :
298 : /*
299 : * The last entry in the timeline history for the ending timeline should
300 : * be the ending timeline itself. Verify that this is what we observed.
301 : */
302 216 : if (!found_start_timeline)
303 0 : ereport(ERROR,
304 : errmsg("start timeline %u not found in history of timeline %u",
305 : starttli, endtli));
306 :
307 : /* Terminate the list of WAL ranges. */
308 216 : AppendStringToManifest(manifest, "\n],\n");
309 : }
310 :
311 : /*
312 : * Finalize the backup manifest, and send it to the client.
313 : */
314 : void
315 218 : SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
316 : {
317 : uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
318 : char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
319 218 : size_t manifest_bytes_done = 0;
320 :
321 218 : if (!IsManifestEnabled(manifest))
322 2 : return;
323 :
324 : /*
325 : * Append manifest checksum, so that the problems with the manifest itself
326 : * can be detected.
327 : *
328 : * We always use SHA-256 for this, regardless of what algorithm is chosen
329 : * for checksumming the files. If we ever want to make the checksum
330 : * algorithm used for the manifest file variable, the client will need a
331 : * way to figure out which algorithm to use as close to the beginning of
332 : * the manifest file as possible, to avoid having to read the whole thing
333 : * twice.
334 : */
335 216 : manifest->still_checksumming = false;
336 216 : if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
337 : sizeof(checksumbuf)) < 0)
338 0 : elog(ERROR, "failed to finalize checksum of backup manifest: %s",
339 : pg_cryptohash_error(manifest->manifest_ctx));
340 216 : AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
341 :
342 216 : hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
343 216 : checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
344 :
345 216 : AppendStringToManifest(manifest, checksumstringbuf);
346 216 : AppendStringToManifest(manifest, "\"}\n");
347 :
348 : /*
349 : * We've written all the data to the manifest file. Rewind the file so
350 : * that we can read it all back.
351 : */
352 216 : if (BufFileSeek(manifest->buffile, 0, 0L, SEEK_SET))
353 0 : ereport(ERROR,
354 : (errcode_for_file_access(),
355 : errmsg("could not rewind temporary file")));
356 :
357 :
358 : /*
359 : * Send the backup manifest.
360 : */
361 216 : bbsink_begin_manifest(sink);
362 1316 : while (manifest_bytes_done < manifest->manifest_size)
363 : {
364 : size_t bytes_to_read;
365 :
366 1100 : bytes_to_read = Min(sink->bbs_buffer_length,
367 : manifest->manifest_size - manifest_bytes_done);
368 1100 : BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
369 1100 : bbsink_manifest_contents(sink, bytes_to_read);
370 1100 : manifest_bytes_done += bytes_to_read;
371 : }
372 216 : bbsink_end_manifest(sink);
373 :
374 : /* Release resources */
375 216 : BufFileClose(manifest->buffile);
376 : }
377 :
378 : /*
379 : * Append a cstring to the manifest.
380 : */
381 : static void
382 214922 : AppendStringToManifest(backup_manifest_info *manifest, const char *s)
383 : {
384 214922 : int len = strlen(s);
385 :
386 : Assert(manifest != NULL);
387 214922 : if (manifest->still_checksumming)
388 : {
389 214274 : if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
390 0 : elog(ERROR, "failed to update checksum of backup manifest: %s",
391 : pg_cryptohash_error(manifest->manifest_ctx));
392 : }
393 214922 : BufFileWrite(manifest->buffile, s, len);
394 214922 : manifest->manifest_size += len;
395 214922 : }
|
