Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * backup_manifest.c
4 : * code for generating and sending a backup manifest
5 : *
6 : * Portions Copyright (c) 2010-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/backup/backup_manifest.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include "access/timeline.h"
16 : #include "access/xlog.h"
17 : #include "backup/backup_manifest.h"
18 : #include "backup/basebackup_sink.h"
19 : #include "common/relpath.h"
20 : #include "mb/pg_wchar.h"
21 : #include "utils/builtins.h"
22 : #include "utils/json.h"
23 :
24 : static void AppendStringToManifest(backup_manifest_info *manifest, const char *s);
25 :
26 : /*
27 : * Does the user want a backup manifest?
28 : *
29 : * It's simplest to always have a manifest_info object, so that we don't need
30 : * checks for NULL pointers in too many places. However, if the user doesn't
31 : * want a manifest, we set manifest->buffile to NULL.
32 : */
33 : static inline bool
34 300622 : IsManifestEnabled(backup_manifest_info *manifest)
35 : {
36 300622 : return (manifest->buffile != NULL);
37 : }
38 :
39 : /*
40 : * Convenience macro for appending data to the backup manifest.
41 : */
42 : #define AppendToManifest(manifest, ...) \
43 : { \
44 : char *_manifest_s = psprintf(__VA_ARGS__); \
45 : AppendStringToManifest(manifest, _manifest_s); \
46 : pfree(_manifest_s); \
47 : }
48 :
49 : /*
50 : * Initialize state so that we can construct a backup manifest.
51 : *
52 : * NB: Although the checksum type for the data files is configurable, the
53 : * checksum for the manifest itself always uses SHA-256. See comments in
54 : * SendBackupManifest.
55 : */
56 : void
57 312 : InitializeBackupManifest(backup_manifest_info *manifest,
58 : backup_manifest_option want_manifest,
59 : pg_checksum_type manifest_checksum_type)
60 : {
61 312 : memset(manifest, 0, sizeof(backup_manifest_info));
62 312 : manifest->checksum_type = manifest_checksum_type;
63 :
64 312 : if (want_manifest == MANIFEST_OPTION_NO)
65 4 : manifest->buffile = NULL;
66 : else
67 : {
68 308 : manifest->buffile = BufFileCreateTemp(false);
69 308 : manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
70 308 : if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
71 0 : elog(ERROR, "failed to initialize checksum of backup manifest: %s",
72 : pg_cryptohash_error(manifest->manifest_ctx));
73 : }
74 :
75 312 : manifest->manifest_size = UINT64CONST(0);
76 312 : manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
77 312 : manifest->first_file = true;
78 312 : manifest->still_checksumming = true;
79 :
80 312 : if (want_manifest != MANIFEST_OPTION_NO)
81 308 : AppendToManifest(manifest,
82 : "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
83 : "\"System-Identifier\": " UINT64_FORMAT ",\n"
84 : "\"Files\": [",
85 : GetSystemIdentifier());
86 312 : }
87 :
88 : /*
89 : * Free resources assigned to a backup manifest constructed.
90 : */
91 : void
92 292 : FreeBackupManifest(backup_manifest_info *manifest)
93 : {
94 292 : pg_cryptohash_free(manifest->manifest_ctx);
95 292 : manifest->manifest_ctx = NULL;
96 292 : }
97 :
98 : /*
99 : * Add an entry to the backup manifest for a file.
100 : */
101 : void
102 300026 : AddFileToBackupManifest(backup_manifest_info *manifest, Oid spcoid,
103 : const char *pathname, size_t size, pg_time_t mtime,
104 : pg_checksum_context *checksum_ctx)
105 : {
106 : char pathbuf[MAXPGPATH];
107 : int pathlen;
108 : StringInfoData buf;
109 :
110 300026 : if (!IsManifestEnabled(manifest))
111 1946 : return;
112 :
113 : /*
114 : * If this file is part of a tablespace, the pathname passed to this
115 : * function will be relative to the tar file that contains it. We want the
116 : * pathname relative to the data directory (ignoring the intermediate
117 : * symlink traversal).
118 : */
119 298080 : if (OidIsValid(spcoid))
120 : {
121 690 : snprintf(pathbuf, sizeof(pathbuf), "%s/%u/%s", PG_TBLSPC_DIR, spcoid,
122 : pathname);
123 690 : pathname = pathbuf;
124 : }
125 :
126 : /*
127 : * Each file's entry needs to be separated from any entry that follows by
128 : * a comma, but there's no comma before the first one or after the last
129 : * one. To make that work, adding a file to the manifest starts by
130 : * terminating the most recently added line, with a comma if appropriate,
131 : * but does not terminate the line inserted for this file.
132 : */
133 298080 : initStringInfo(&buf);
134 298080 : if (manifest->first_file)
135 : {
136 308 : appendStringInfoChar(&buf, '\n');
137 308 : manifest->first_file = false;
138 : }
139 : else
140 297772 : appendStringInfoString(&buf, ",\n");
141 :
142 : /*
143 : * Write the relative pathname to this file out to the manifest. The
144 : * manifest is always stored in UTF-8, so we have to encode paths that are
145 : * not valid in that encoding.
146 : */
147 298080 : pathlen = strlen(pathname);
148 594220 : if (!manifest->force_encode &&
149 296140 : pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
150 : {
151 296140 : appendStringInfoString(&buf, "{ \"Path\": ");
152 296140 : escape_json_with_len(&buf, pathname, pathlen);
153 296140 : appendStringInfoString(&buf, ", ");
154 : }
155 : else
156 : {
157 1940 : appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
158 1940 : enlargeStringInfo(&buf, 2 * pathlen);
159 3880 : buf.len += hex_encode(pathname, pathlen,
160 1940 : &buf.data[buf.len]);
161 1940 : appendStringInfoString(&buf, "\", ");
162 : }
163 :
164 298080 : appendStringInfo(&buf, "\"Size\": %zu, ", size);
165 :
166 : /*
167 : * Convert last modification time to a string and append it to the
168 : * manifest. Since it's not clear what time zone to use and since time
169 : * zone definitions can change, possibly causing confusion, use GMT
170 : * always.
171 : */
172 298080 : appendStringInfoString(&buf, "\"Last-Modified\": \"");
173 298080 : enlargeStringInfo(&buf, 128);
174 298080 : buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
175 298080 : pg_gmtime(&mtime));
176 298080 : appendStringInfoChar(&buf, '"');
177 :
178 : /* Add checksum information. */
179 298080 : if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
180 : {
181 : uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
182 : int checksumlen;
183 :
184 294198 : checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
185 294198 : if (checksumlen < 0)
186 0 : elog(ERROR, "could not finalize checksum of file \"%s\"",
187 : pathname);
188 :
189 294198 : appendStringInfo(&buf,
190 : ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
191 : pg_checksum_type_name(checksum_ctx->type));
192 294198 : enlargeStringInfo(&buf, 2 * checksumlen);
193 588396 : buf.len += hex_encode((char *) checksumbuf, checksumlen,
194 294198 : &buf.data[buf.len]);
195 294198 : appendStringInfoChar(&buf, '"');
196 : }
197 :
198 : /* Close out the object. */
199 298080 : appendStringInfoString(&buf, " }");
200 :
201 : /* OK, add it to the manifest. */
202 298080 : AppendStringToManifest(manifest, buf.data);
203 :
204 : /* Avoid leaking memory. */
205 298080 : pfree(buf.data);
206 : }
207 :
208 : /*
209 : * Add information about the WAL that will need to be replayed when restoring
210 : * this backup to the manifest.
211 : */
212 : void
213 298 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
214 : TimeLineID starttli, XLogRecPtr endptr,
215 : TimeLineID endtli)
216 : {
217 : List *timelines;
218 : ListCell *lc;
219 298 : bool first_wal_range = true;
220 298 : bool found_start_timeline = false;
221 :
222 298 : if (!IsManifestEnabled(manifest))
223 2 : return;
224 :
225 : /* Terminate the list of files. */
226 296 : AppendStringToManifest(manifest, "\n],\n");
227 :
228 : /* Read the timeline history for the ending timeline. */
229 296 : timelines = readTimeLineHistory(endtli);
230 :
231 : /* Start a list of LSN ranges. */
232 296 : AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
233 :
234 296 : foreach(lc, timelines)
235 : {
236 296 : TimeLineHistoryEntry *entry = lfirst(lc);
237 : XLogRecPtr tl_beginptr;
238 :
239 : /*
240 : * We only care about timelines that were active during the backup.
241 : * Skip any that ended before the backup started. (Note that if
242 : * entry->end is InvalidXLogRecPtr, it means that the timeline has not
243 : * yet ended.)
244 : */
245 296 : if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
246 0 : continue;
247 :
248 : /*
249 : * Because the timeline history file lists newer timelines before
250 : * older ones, the first timeline we encounter that is new enough to
251 : * matter ought to match the ending timeline of the backup.
252 : */
253 296 : if (first_wal_range && endtli != entry->tli)
254 0 : ereport(ERROR,
255 : errmsg("expected end timeline %u but found timeline %u",
256 : starttli, entry->tli));
257 :
258 : /*
259 : * If this timeline entry matches with the timeline on which the
260 : * backup started, WAL needs to be checked from the start LSN of the
261 : * backup. If this entry refers to a newer timeline, WAL needs to be
262 : * checked since the beginning of this timeline, so use the LSN where
263 : * the timeline began.
264 : */
265 296 : if (starttli == entry->tli)
266 296 : tl_beginptr = startptr;
267 : else
268 : {
269 0 : tl_beginptr = entry->begin;
270 :
271 : /*
272 : * If we reach a TLI that has no valid beginning LSN, there can't
273 : * be any more timelines in the history after this point, so we'd
274 : * better have arrived at the expected starting TLI. If not,
275 : * something's gone horribly wrong.
276 : */
277 0 : if (XLogRecPtrIsInvalid(entry->begin))
278 0 : ereport(ERROR,
279 : errmsg("expected start timeline %u but found timeline %u",
280 : starttli, entry->tli));
281 : }
282 :
283 296 : AppendToManifest(manifest,
284 : "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
285 : first_wal_range ? "" : ",\n",
286 : entry->tli,
287 : LSN_FORMAT_ARGS(tl_beginptr),
288 : LSN_FORMAT_ARGS(endptr));
289 :
290 296 : if (starttli == entry->tli)
291 : {
292 296 : found_start_timeline = true;
293 296 : break;
294 : }
295 :
296 0 : endptr = entry->begin;
297 0 : first_wal_range = false;
298 : }
299 :
300 : /*
301 : * The last entry in the timeline history for the ending timeline should
302 : * be the ending timeline itself. Verify that this is what we observed.
303 : */
304 296 : if (!found_start_timeline)
305 0 : ereport(ERROR,
306 : errmsg("start timeline %u not found in history of timeline %u",
307 : starttli, endtli));
308 :
309 : /* Terminate the list of WAL ranges. */
310 296 : AppendStringToManifest(manifest, "\n],\n");
311 : }
312 :
313 : /*
314 : * Finalize the backup manifest, and send it to the client.
315 : */
316 : void
317 298 : SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
318 : {
319 : uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
320 : char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
321 298 : size_t manifest_bytes_done = 0;
322 :
323 298 : if (!IsManifestEnabled(manifest))
324 2 : return;
325 :
326 : /*
327 : * Append manifest checksum, so that the problems with the manifest itself
328 : * can be detected.
329 : *
330 : * We always use SHA-256 for this, regardless of what algorithm is chosen
331 : * for checksumming the files. If we ever want to make the checksum
332 : * algorithm used for the manifest file variable, the client will need a
333 : * way to figure out which algorithm to use as close to the beginning of
334 : * the manifest file as possible, to avoid having to read the whole thing
335 : * twice.
336 : */
337 296 : manifest->still_checksumming = false;
338 296 : if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
339 : sizeof(checksumbuf)) < 0)
340 0 : elog(ERROR, "failed to finalize checksum of backup manifest: %s",
341 : pg_cryptohash_error(manifest->manifest_ctx));
342 296 : AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
343 :
344 296 : hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
345 296 : checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
346 :
347 296 : AppendStringToManifest(manifest, checksumstringbuf);
348 296 : AppendStringToManifest(manifest, "\"}\n");
349 :
350 : /*
351 : * We've written all the data to the manifest file. Rewind the file so
352 : * that we can read it all back.
353 : */
354 296 : if (BufFileSeek(manifest->buffile, 0, 0, SEEK_SET))
355 0 : ereport(ERROR,
356 : (errcode_for_file_access(),
357 : errmsg("could not rewind temporary file")));
358 :
359 :
360 : /*
361 : * Send the backup manifest.
362 : */
363 296 : bbsink_begin_manifest(sink);
364 1828 : while (manifest_bytes_done < manifest->manifest_size)
365 : {
366 : size_t bytes_to_read;
367 :
368 1532 : bytes_to_read = Min(sink->bbs_buffer_length,
369 : manifest->manifest_size - manifest_bytes_done);
370 1532 : BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
371 1532 : bbsink_manifest_contents(sink, bytes_to_read);
372 1532 : manifest_bytes_done += bytes_to_read;
373 : }
374 296 : bbsink_end_manifest(sink);
375 :
376 : /* Release resources */
377 296 : BufFileClose(manifest->buffile);
378 : }
379 :
380 : /*
381 : * Append a cstring to the manifest.
382 : */
383 : static void
384 300460 : AppendStringToManifest(backup_manifest_info *manifest, const char *s)
385 : {
386 300460 : int len = strlen(s);
387 :
388 : Assert(manifest != NULL);
389 300460 : if (manifest->still_checksumming)
390 : {
391 299572 : if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
392 0 : elog(ERROR, "failed to update checksum of backup manifest: %s",
393 : pg_cryptohash_error(manifest->manifest_ctx));
394 : }
395 300460 : BufFileWrite(manifest->buffile, s, len);
396 300460 : manifest->manifest_size += len;
397 300460 : }
|
