Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * backup_manifest.c
4 : * code for generating and sending a backup manifest
5 : *
6 : * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/backup/backup_manifest.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include "access/timeline.h"
16 : #include "access/xlog.h"
17 : #include "backup/backup_manifest.h"
18 : #include "backup/basebackup_sink.h"
19 : #include "mb/pg_wchar.h"
20 : #include "utils/builtins.h"
21 : #include "utils/json.h"
22 :
23 : static void AppendStringToManifest(backup_manifest_info *manifest, const char *s);
24 :
25 : /*
26 : * Does the user want a backup manifest?
27 : *
28 : * It's simplest to always have a manifest_info object, so that we don't need
29 : * checks for NULL pointers in too many places. However, if the user doesn't
30 : * want a manifest, we set manifest->buffile to NULL.
31 : */
32 : static inline bool
33 268064 : IsManifestEnabled(backup_manifest_info *manifest)
34 : {
35 268064 : return (manifest->buffile != NULL);
36 : }
37 :
38 : /*
39 : * Convenience macro for appending data to the backup manifest.
40 : */
41 : #define AppendToManifest(manifest, ...) \
42 : { \
43 : char *_manifest_s = psprintf(__VA_ARGS__); \
44 : AppendStringToManifest(manifest, _manifest_s); \
45 : pfree(_manifest_s); \
46 : }
47 :
48 : /*
49 : * Initialize state so that we can construct a backup manifest.
50 : *
51 : * NB: Although the checksum type for the data files is configurable, the
52 : * checksum for the manifest itself always uses SHA-256. See comments in
53 : * SendBackupManifest.
54 : */
55 : void
56 278 : InitializeBackupManifest(backup_manifest_info *manifest,
57 : backup_manifest_option want_manifest,
58 : pg_checksum_type manifest_checksum_type)
59 : {
60 278 : memset(manifest, 0, sizeof(backup_manifest_info));
61 278 : manifest->checksum_type = manifest_checksum_type;
62 :
63 278 : if (want_manifest == MANIFEST_OPTION_NO)
64 2 : manifest->buffile = NULL;
65 : else
66 : {
67 276 : manifest->buffile = BufFileCreateTemp(false);
68 276 : manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
69 276 : if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
70 0 : elog(ERROR, "failed to initialize checksum of backup manifest: %s",
71 : pg_cryptohash_error(manifest->manifest_ctx));
72 : }
73 :
74 278 : manifest->manifest_size = UINT64CONST(0);
75 278 : manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
76 278 : manifest->first_file = true;
77 278 : manifest->still_checksumming = true;
78 :
79 278 : if (want_manifest != MANIFEST_OPTION_NO)
80 276 : AppendToManifest(manifest,
81 : "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
82 : "\"System-Identifier\": " UINT64_FORMAT ",\n"
83 : "\"Files\": [",
84 : GetSystemIdentifier());
85 278 : }
86 :
87 : /*
88 : * Free resources assigned to a backup manifest constructed.
89 : */
90 : void
91 260 : FreeBackupManifest(backup_manifest_info *manifest)
92 : {
93 260 : pg_cryptohash_free(manifest->manifest_ctx);
94 260 : manifest->manifest_ctx = NULL;
95 260 : }
96 :
97 : /*
98 : * Add an entry to the backup manifest for a file.
99 : */
100 : void
101 267532 : AddFileToBackupManifest(backup_manifest_info *manifest, Oid spcoid,
102 : const char *pathname, size_t size, pg_time_t mtime,
103 : pg_checksum_context *checksum_ctx)
104 : {
105 : char pathbuf[MAXPGPATH];
106 : int pathlen;
107 : StringInfoData buf;
108 :
109 267532 : if (!IsManifestEnabled(manifest))
110 1934 : return;
111 :
112 : /*
113 : * If this file is part of a tablespace, the pathname passed to this
114 : * function will be relative to the tar file that contains it. We want the
115 : * pathname relative to the data directory (ignoring the intermediate
116 : * symlink traversal).
117 : */
118 265598 : if (OidIsValid(spcoid))
119 : {
120 680 : snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%u/%s", spcoid,
121 : pathname);
122 680 : pathname = pathbuf;
123 : }
124 :
125 : /*
126 : * Each file's entry needs to be separated from any entry that follows by
127 : * a comma, but there's no comma before the first one or after the last
128 : * one. To make that work, adding a file to the manifest starts by
129 : * terminating the most recently added line, with a comma if appropriate,
130 : * but does not terminate the line inserted for this file.
131 : */
132 265598 : initStringInfo(&buf);
133 265598 : if (manifest->first_file)
134 : {
135 276 : appendStringInfoChar(&buf, '\n');
136 276 : manifest->first_file = false;
137 : }
138 : else
139 265322 : appendStringInfoString(&buf, ",\n");
140 :
141 : /*
142 : * Write the relative pathname to this file out to the manifest. The
143 : * manifest is always stored in UTF-8, so we have to encode paths that are
144 : * not valid in that encoding.
145 : */
146 265598 : pathlen = strlen(pathname);
147 529264 : if (!manifest->force_encode &&
148 263666 : pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
149 : {
150 263666 : appendStringInfoString(&buf, "{ \"Path\": ");
151 263666 : escape_json(&buf, pathname);
152 263666 : appendStringInfoString(&buf, ", ");
153 : }
154 : else
155 : {
156 1932 : appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
157 1932 : enlargeStringInfo(&buf, 2 * pathlen);
158 3864 : buf.len += hex_encode(pathname, pathlen,
159 1932 : &buf.data[buf.len]);
160 1932 : appendStringInfoString(&buf, "\", ");
161 : }
162 :
163 265598 : appendStringInfo(&buf, "\"Size\": %zu, ", size);
164 :
165 : /*
166 : * Convert last modification time to a string and append it to the
167 : * manifest. Since it's not clear what time zone to use and since time
168 : * zone definitions can change, possibly causing confusion, use GMT
169 : * always.
170 : */
171 265598 : appendStringInfoString(&buf, "\"Last-Modified\": \"");
172 265598 : enlargeStringInfo(&buf, 128);
173 265598 : buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
174 265598 : pg_gmtime(&mtime));
175 265598 : appendStringInfoChar(&buf, '"');
176 :
177 : /* Add checksum information. */
178 265598 : if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
179 : {
180 : uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
181 : int checksumlen;
182 :
183 263666 : checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
184 263666 : if (checksumlen < 0)
185 0 : elog(ERROR, "could not finalize checksum of file \"%s\"",
186 : pathname);
187 :
188 263666 : appendStringInfo(&buf,
189 : ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
190 : pg_checksum_type_name(checksum_ctx->type));
191 263666 : enlargeStringInfo(&buf, 2 * checksumlen);
192 527332 : buf.len += hex_encode((char *) checksumbuf, checksumlen,
193 263666 : &buf.data[buf.len]);
194 263666 : appendStringInfoChar(&buf, '"');
195 : }
196 :
197 : /* Close out the object. */
198 265598 : appendStringInfoString(&buf, " }");
199 :
200 : /* OK, add it to the manifest. */
201 265598 : AppendStringToManifest(manifest, buf.data);
202 :
203 : /* Avoid leaking memory. */
204 265598 : pfree(buf.data);
205 : }
206 :
207 : /*
208 : * Add information about the WAL that will need to be replayed when restoring
209 : * this backup to the manifest.
210 : */
211 : void
212 266 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
213 : TimeLineID starttli, XLogRecPtr endptr,
214 : TimeLineID endtli)
215 : {
216 : List *timelines;
217 : ListCell *lc;
218 266 : bool first_wal_range = true;
219 266 : bool found_start_timeline = false;
220 :
221 266 : if (!IsManifestEnabled(manifest))
222 2 : return;
223 :
224 : /* Terminate the list of files. */
225 264 : AppendStringToManifest(manifest, "\n],\n");
226 :
227 : /* Read the timeline history for the ending timeline. */
228 264 : timelines = readTimeLineHistory(endtli);
229 :
230 : /* Start a list of LSN ranges. */
231 264 : AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
232 :
233 264 : foreach(lc, timelines)
234 : {
235 264 : TimeLineHistoryEntry *entry = lfirst(lc);
236 : XLogRecPtr tl_beginptr;
237 :
238 : /*
239 : * We only care about timelines that were active during the backup.
240 : * Skip any that ended before the backup started. (Note that if
241 : * entry->end is InvalidXLogRecPtr, it means that the timeline has not
242 : * yet ended.)
243 : */
244 264 : if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
245 0 : continue;
246 :
247 : /*
248 : * Because the timeline history file lists newer timelines before
249 : * older ones, the first timeline we encounter that is new enough to
250 : * matter ought to match the ending timeline of the backup.
251 : */
252 264 : if (first_wal_range && endtli != entry->tli)
253 0 : ereport(ERROR,
254 : errmsg("expected end timeline %u but found timeline %u",
255 : starttli, entry->tli));
256 :
257 : /*
258 : * If this timeline entry matches with the timeline on which the
259 : * backup started, WAL needs to be checked from the start LSN of the
260 : * backup. If this entry refers to a newer timeline, WAL needs to be
261 : * checked since the beginning of this timeline, so use the LSN where
262 : * the timeline began.
263 : */
264 264 : if (starttli == entry->tli)
265 264 : tl_beginptr = startptr;
266 : else
267 : {
268 0 : tl_beginptr = entry->begin;
269 :
270 : /*
271 : * If we reach a TLI that has no valid beginning LSN, there can't
272 : * be any more timelines in the history after this point, so we'd
273 : * better have arrived at the expected starting TLI. If not,
274 : * something's gone horribly wrong.
275 : */
276 0 : if (XLogRecPtrIsInvalid(entry->begin))
277 0 : ereport(ERROR,
278 : errmsg("expected start timeline %u but found timeline %u",
279 : starttli, entry->tli));
280 : }
281 :
282 264 : AppendToManifest(manifest,
283 : "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
284 : first_wal_range ? "" : ",\n",
285 : entry->tli,
286 : LSN_FORMAT_ARGS(tl_beginptr),
287 : LSN_FORMAT_ARGS(endptr));
288 :
289 264 : if (starttli == entry->tli)
290 : {
291 264 : found_start_timeline = true;
292 264 : break;
293 : }
294 :
295 0 : endptr = entry->begin;
296 0 : first_wal_range = false;
297 : }
298 :
299 : /*
300 : * The last entry in the timeline history for the ending timeline should
301 : * be the ending timeline itself. Verify that this is what we observed.
302 : */
303 264 : if (!found_start_timeline)
304 0 : ereport(ERROR,
305 : errmsg("start timeline %u not found in history of timeline %u",
306 : starttli, endtli));
307 :
308 : /* Terminate the list of WAL ranges. */
309 264 : AppendStringToManifest(manifest, "\n],\n");
310 : }
311 :
312 : /*
313 : * Finalize the backup manifest, and send it to the client.
314 : */
315 : void
316 266 : SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
317 : {
318 : uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
319 : char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
320 266 : size_t manifest_bytes_done = 0;
321 :
322 266 : if (!IsManifestEnabled(manifest))
323 2 : return;
324 :
325 : /*
326 : * Append manifest checksum, so that the problems with the manifest itself
327 : * can be detected.
328 : *
329 : * We always use SHA-256 for this, regardless of what algorithm is chosen
330 : * for checksumming the files. If we ever want to make the checksum
331 : * algorithm used for the manifest file variable, the client will need a
332 : * way to figure out which algorithm to use as close to the beginning of
333 : * the manifest file as possible, to avoid having to read the whole thing
334 : * twice.
335 : */
336 264 : manifest->still_checksumming = false;
337 264 : if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
338 : sizeof(checksumbuf)) < 0)
339 0 : elog(ERROR, "failed to finalize checksum of backup manifest: %s",
340 : pg_cryptohash_error(manifest->manifest_ctx));
341 264 : AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
342 :
343 264 : hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
344 264 : checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
345 :
346 264 : AppendStringToManifest(manifest, checksumstringbuf);
347 264 : AppendStringToManifest(manifest, "\"}\n");
348 :
349 : /*
350 : * We've written all the data to the manifest file. Rewind the file so
351 : * that we can read it all back.
352 : */
353 264 : if (BufFileSeek(manifest->buffile, 0, 0, SEEK_SET))
354 0 : ereport(ERROR,
355 : (errcode_for_file_access(),
356 : errmsg("could not rewind temporary file")));
357 :
358 :
359 : /*
360 : * Send the backup manifest.
361 : */
362 264 : bbsink_begin_manifest(sink);
363 1626 : while (manifest_bytes_done < manifest->manifest_size)
364 : {
365 : size_t bytes_to_read;
366 :
367 1362 : bytes_to_read = Min(sink->bbs_buffer_length,
368 : manifest->manifest_size - manifest_bytes_done);
369 1362 : BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
370 1362 : bbsink_manifest_contents(sink, bytes_to_read);
371 1362 : manifest_bytes_done += bytes_to_read;
372 : }
373 264 : bbsink_end_manifest(sink);
374 :
375 : /* Release resources */
376 264 : BufFileClose(manifest->buffile);
377 : }
378 :
379 : /*
380 : * Append a cstring to the manifest.
381 : */
382 : static void
383 267722 : AppendStringToManifest(backup_manifest_info *manifest, const char *s)
384 : {
385 267722 : int len = strlen(s);
386 :
387 : Assert(manifest != NULL);
388 267722 : if (manifest->still_checksumming)
389 : {
390 266930 : if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
391 0 : elog(ERROR, "failed to update checksum of backup manifest: %s",
392 : pg_cryptohash_error(manifest->manifest_ctx));
393 : }
394 267722 : BufFileWrite(manifest->buffile, s, len);
395 267722 : manifest->manifest_size += len;
396 267722 : }
|
