Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * backup_manifest.c
4 : * code for generating and sending a backup manifest
5 : *
6 : * Portions Copyright (c) 2010-2023, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/backup/backup_manifest.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 : #include "postgres.h"
14 :
15 : #include "access/timeline.h"
16 : #include "backup/backup_manifest.h"
17 : #include "backup/basebackup_sink.h"
18 : #include "libpq/libpq.h"
19 : #include "libpq/pqformat.h"
20 : #include "mb/pg_wchar.h"
21 : #include "utils/builtins.h"
22 : #include "utils/json.h"
23 :
24 : static void AppendStringToManifest(backup_manifest_info *manifest, const char *s);
25 :
26 : /*
27 : * Does the user want a backup manifest?
28 : *
29 : * It's simplest to always have a manifest_info object, so that we don't need
30 : * checks for NULL pointers in too many places. However, if the user doesn't
31 : * want a manifest, we set manifest->buffile to NULL.
32 : */
33 : static inline bool
34 222568 : IsManifestEnabled(backup_manifest_info *manifest)
35 : {
36 222568 : return (manifest->buffile != NULL);
37 : }
38 :
39 : /*
40 : * Convenience macro for appending data to the backup manifest.
41 : */
42 : #define AppendToManifest(manifest, ...) \
43 : { \
44 : char *_manifest_s = psprintf(__VA_ARGS__); \
45 : AppendStringToManifest(manifest, _manifest_s); \
46 : pfree(_manifest_s); \
47 : }
48 :
49 : /*
50 : * Initialize state so that we can construct a backup manifest.
51 : *
52 : * NB: Although the checksum type for the data files is configurable, the
53 : * checksum for the manifest itself always uses SHA-256. See comments in
54 : * SendBackupManifest.
55 : */
56 : void
57 236 : InitializeBackupManifest(backup_manifest_info *manifest,
58 : backup_manifest_option want_manifest,
59 : pg_checksum_type manifest_checksum_type)
60 : {
61 236 : memset(manifest, 0, sizeof(backup_manifest_info));
62 236 : manifest->checksum_type = manifest_checksum_type;
63 :
64 236 : if (want_manifest == MANIFEST_OPTION_NO)
65 2 : manifest->buffile = NULL;
66 : else
67 : {
68 234 : manifest->buffile = BufFileCreateTemp(false);
69 234 : manifest->manifest_ctx = pg_cryptohash_create(PG_SHA256);
70 234 : if (pg_cryptohash_init(manifest->manifest_ctx) < 0)
71 0 : elog(ERROR, "failed to initialize checksum of backup manifest: %s",
72 : pg_cryptohash_error(manifest->manifest_ctx));
73 : }
74 :
75 236 : manifest->manifest_size = UINT64CONST(0);
76 236 : manifest->force_encode = (want_manifest == MANIFEST_OPTION_FORCE_ENCODE);
77 236 : manifest->first_file = true;
78 236 : manifest->still_checksumming = true;
79 :
80 236 : if (want_manifest != MANIFEST_OPTION_NO)
81 234 : AppendToManifest(manifest,
82 : "{ \"PostgreSQL-Backup-Manifest-Version\": 1,\n"
83 : "\"Files\": [");
84 236 : }
85 :
86 : /*
87 : * Free resources assigned to a backup manifest constructed.
88 : */
89 : void
90 218 : FreeBackupManifest(backup_manifest_info *manifest)
91 : {
92 218 : pg_cryptohash_free(manifest->manifest_ctx);
93 218 : manifest->manifest_ctx = NULL;
94 218 : }
95 :
96 : /*
97 : * Add an entry to the backup manifest for a file.
98 : */
99 : void
100 222120 : AddFileToBackupManifest(backup_manifest_info *manifest, const char *spcoid,
101 : const char *pathname, size_t size, pg_time_t mtime,
102 : pg_checksum_context *checksum_ctx)
103 : {
104 : char pathbuf[MAXPGPATH];
105 : int pathlen;
106 : StringInfoData buf;
107 :
108 222120 : if (!IsManifestEnabled(manifest))
109 1934 : return;
110 :
111 : /*
112 : * If this file is part of a tablespace, the pathname passed to this
113 : * function will be relative to the tar file that contains it. We want the
114 : * pathname relative to the data directory (ignoring the intermediate
115 : * symlink traversal).
116 : */
117 220186 : if (spcoid != NULL)
118 : {
119 648 : snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", spcoid,
120 : pathname);
121 648 : pathname = pathbuf;
122 : }
123 :
124 : /*
125 : * Each file's entry needs to be separated from any entry that follows by
126 : * a comma, but there's no comma before the first one or after the last
127 : * one. To make that work, adding a file to the manifest starts by
128 : * terminating the most recently added line, with a comma if appropriate,
129 : * but does not terminate the line inserted for this file.
130 : */
131 220186 : initStringInfo(&buf);
132 220186 : if (manifest->first_file)
133 : {
134 234 : appendStringInfoChar(&buf, '\n');
135 234 : manifest->first_file = false;
136 : }
137 : else
138 219952 : appendStringInfoString(&buf, ",\n");
139 :
140 : /*
141 : * Write the relative pathname to this file out to the manifest. The
142 : * manifest is always stored in UTF-8, so we have to encode paths that are
143 : * not valid in that encoding.
144 : */
145 220186 : pathlen = strlen(pathname);
146 438440 : if (!manifest->force_encode &&
147 218254 : pg_verify_mbstr(PG_UTF8, pathname, pathlen, true))
148 : {
149 218254 : appendStringInfoString(&buf, "{ \"Path\": ");
150 218254 : escape_json(&buf, pathname);
151 218254 : appendStringInfoString(&buf, ", ");
152 : }
153 : else
154 : {
155 1932 : appendStringInfoString(&buf, "{ \"Encoded-Path\": \"");
156 1932 : enlargeStringInfo(&buf, 2 * pathlen);
157 3864 : buf.len += hex_encode(pathname, pathlen,
158 1932 : &buf.data[buf.len]);
159 1932 : appendStringInfoString(&buf, "\", ");
160 : }
161 :
162 220186 : appendStringInfo(&buf, "\"Size\": %zu, ", size);
163 :
164 : /*
165 : * Convert last modification time to a string and append it to the
166 : * manifest. Since it's not clear what time zone to use and since time
167 : * zone definitions can change, possibly causing confusion, use GMT
168 : * always.
169 : */
170 220186 : appendStringInfoString(&buf, "\"Last-Modified\": \"");
171 220186 : enlargeStringInfo(&buf, 128);
172 220186 : buf.len += pg_strftime(&buf.data[buf.len], 128, "%Y-%m-%d %H:%M:%S %Z",
173 220186 : pg_gmtime(&mtime));
174 220186 : appendStringInfoChar(&buf, '"');
175 :
176 : /* Add checksum information. */
177 220186 : if (checksum_ctx->type != CHECKSUM_TYPE_NONE)
178 : {
179 : uint8 checksumbuf[PG_CHECKSUM_MAX_LENGTH];
180 : int checksumlen;
181 :
182 218254 : checksumlen = pg_checksum_final(checksum_ctx, checksumbuf);
183 218254 : if (checksumlen < 0)
184 0 : elog(ERROR, "could not finalize checksum of file \"%s\"",
185 : pathname);
186 :
187 218254 : appendStringInfo(&buf,
188 : ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
189 : pg_checksum_type_name(checksum_ctx->type));
190 218254 : enlargeStringInfo(&buf, 2 * checksumlen);
191 436508 : buf.len += hex_encode((char *) checksumbuf, checksumlen,
192 218254 : &buf.data[buf.len]);
193 218254 : appendStringInfoChar(&buf, '"');
194 : }
195 :
196 : /* Close out the object. */
197 220186 : appendStringInfoString(&buf, " }");
198 :
199 : /* OK, add it to the manifest. */
200 220186 : AppendStringToManifest(manifest, buf.data);
201 :
202 : /* Avoid leaking memory. */
203 220186 : pfree(buf.data);
204 : }
205 :
206 : /*
207 : * Add information about the WAL that will need to be replayed when restoring
208 : * this backup to the manifest.
209 : */
210 : void
211 224 : AddWALInfoToBackupManifest(backup_manifest_info *manifest, XLogRecPtr startptr,
212 : TimeLineID starttli, XLogRecPtr endptr,
213 : TimeLineID endtli)
214 : {
215 : List *timelines;
216 : ListCell *lc;
217 224 : bool first_wal_range = true;
218 224 : bool found_start_timeline = false;
219 :
220 224 : if (!IsManifestEnabled(manifest))
221 2 : return;
222 :
223 : /* Terminate the list of files. */
224 222 : AppendStringToManifest(manifest, "\n],\n");
225 :
226 : /* Read the timeline history for the ending timeline. */
227 222 : timelines = readTimeLineHistory(endtli);
228 :
229 : /* Start a list of LSN ranges. */
230 222 : AppendStringToManifest(manifest, "\"WAL-Ranges\": [\n");
231 :
232 222 : foreach(lc, timelines)
233 : {
234 222 : TimeLineHistoryEntry *entry = lfirst(lc);
235 : XLogRecPtr tl_beginptr;
236 :
237 : /*
238 : * We only care about timelines that were active during the backup.
239 : * Skip any that ended before the backup started. (Note that if
240 : * entry->end is InvalidXLogRecPtr, it means that the timeline has not
241 : * yet ended.)
242 : */
243 222 : if (!XLogRecPtrIsInvalid(entry->end) && entry->end < startptr)
244 0 : continue;
245 :
246 : /*
247 : * Because the timeline history file lists newer timelines before
248 : * older ones, the first timeline we encounter that is new enough to
249 : * matter ought to match the ending timeline of the backup.
250 : */
251 222 : if (first_wal_range && endtli != entry->tli)
252 0 : ereport(ERROR,
253 : errmsg("expected end timeline %u but found timeline %u",
254 : starttli, entry->tli));
255 :
256 : /*
257 : * If this timeline entry matches with the timeline on which the
258 : * backup started, WAL needs to be checked from the start LSN of the
259 : * backup. If this entry refers to a newer timeline, WAL needs to be
260 : * checked since the beginning of this timeline, so use the LSN where
261 : * the timeline began.
262 : */
263 222 : if (starttli == entry->tli)
264 222 : tl_beginptr = startptr;
265 : else
266 : {
267 0 : tl_beginptr = entry->begin;
268 :
269 : /*
270 : * If we reach a TLI that has no valid beginning LSN, there can't
271 : * be any more timelines in the history after this point, so we'd
272 : * better have arrived at the expected starting TLI. If not,
273 : * something's gone horribly wrong.
274 : */
275 0 : if (XLogRecPtrIsInvalid(entry->begin))
276 0 : ereport(ERROR,
277 : errmsg("expected start timeline %u but found timeline %u",
278 : starttli, entry->tli));
279 : }
280 :
281 222 : AppendToManifest(manifest,
282 : "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
283 : first_wal_range ? "" : ",\n",
284 : entry->tli,
285 : LSN_FORMAT_ARGS(tl_beginptr),
286 : LSN_FORMAT_ARGS(endptr));
287 :
288 222 : if (starttli == entry->tli)
289 : {
290 222 : found_start_timeline = true;
291 222 : break;
292 : }
293 :
294 0 : endptr = entry->begin;
295 0 : first_wal_range = false;
296 : }
297 :
298 : /*
299 : * The last entry in the timeline history for the ending timeline should
300 : * be the ending timeline itself. Verify that this is what we observed.
301 : */
302 222 : if (!found_start_timeline)
303 0 : ereport(ERROR,
304 : errmsg("start timeline %u not found in history of timeline %u",
305 : starttli, endtli));
306 :
307 : /* Terminate the list of WAL ranges. */
308 222 : AppendStringToManifest(manifest, "\n],\n");
309 : }
310 :
311 : /*
312 : * Finalize the backup manifest, and send it to the client.
313 : */
314 : void
315 224 : SendBackupManifest(backup_manifest_info *manifest, bbsink *sink)
316 : {
317 : uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
318 : char checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH];
319 224 : size_t manifest_bytes_done = 0;
320 :
321 224 : if (!IsManifestEnabled(manifest))
322 2 : return;
323 :
324 : /*
325 : * Append manifest checksum, so that the problems with the manifest itself
326 : * can be detected.
327 : *
328 : * We always use SHA-256 for this, regardless of what algorithm is chosen
329 : * for checksumming the files. If we ever want to make the checksum
330 : * algorithm used for the manifest file variable, the client will need a
331 : * way to figure out which algorithm to use as close to the beginning of
332 : * the manifest file as possible, to avoid having to read the whole thing
333 : * twice.
334 : */
335 222 : manifest->still_checksumming = false;
336 222 : if (pg_cryptohash_final(manifest->manifest_ctx, checksumbuf,
337 : sizeof(checksumbuf)) < 0)
338 0 : elog(ERROR, "failed to finalize checksum of backup manifest: %s",
339 : pg_cryptohash_error(manifest->manifest_ctx));
340 222 : AppendStringToManifest(manifest, "\"Manifest-Checksum\": \"");
341 :
342 222 : hex_encode((char *) checksumbuf, sizeof checksumbuf, checksumstringbuf);
343 222 : checksumstringbuf[PG_SHA256_DIGEST_STRING_LENGTH - 1] = '\0';
344 :
345 222 : AppendStringToManifest(manifest, checksumstringbuf);
346 222 : AppendStringToManifest(manifest, "\"}\n");
347 :
348 : /*
349 : * We've written all the data to the manifest file. Rewind the file so
350 : * that we can read it all back.
351 : */
352 222 : if (BufFileSeek(manifest->buffile, 0, 0, SEEK_SET))
353 0 : ereport(ERROR,
354 : (errcode_for_file_access(),
355 : errmsg("could not rewind temporary file")));
356 :
357 :
358 : /*
359 : * Send the backup manifest.
360 : */
361 222 : bbsink_begin_manifest(sink);
362 1358 : while (manifest_bytes_done < manifest->manifest_size)
363 : {
364 : size_t bytes_to_read;
365 :
366 1136 : bytes_to_read = Min(sink->bbs_buffer_length,
367 : manifest->manifest_size - manifest_bytes_done);
368 1136 : BufFileReadExact(manifest->buffile, sink->bbs_buffer, bytes_to_read);
369 1136 : bbsink_manifest_contents(sink, bytes_to_read);
370 1136 : manifest_bytes_done += bytes_to_read;
371 : }
372 222 : bbsink_end_manifest(sink);
373 :
374 : /* Release resources */
375 222 : BufFileClose(manifest->buffile);
376 : }
377 :
378 : /*
379 : * Append a cstring to the manifest.
380 : */
381 : static void
382 221974 : AppendStringToManifest(backup_manifest_info *manifest, const char *s)
383 : {
384 221974 : int len = strlen(s);
385 :
386 : Assert(manifest != NULL);
387 221974 : if (manifest->still_checksumming)
388 : {
389 221308 : if (pg_cryptohash_update(manifest->manifest_ctx, (uint8 *) s, len) < 0)
390 0 : elog(ERROR, "failed to update checksum of backup manifest: %s",
391 : pg_cryptohash_error(manifest->manifest_ctx));
392 : }
393 221974 : BufFileWrite(manifest->buffile, s, len);
394 221974 : manifest->manifest_size += len;
395 221974 : }
|
