Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * Write a new backup manifest.
4 : *
5 : * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
6 : * Portions Copyright (c) 1994, Regents of the University of California
7 : *
8 : * src/bin/pg_combinebackup/write_manifest.c
9 : *
10 : *-------------------------------------------------------------------------
11 : */
12 :
13 : #include "postgres_fe.h"
14 :
15 : #include <fcntl.h>
16 : #include <time.h>
17 : #include <unistd.h>
18 :
19 : #include "common/checksum_helper.h"
20 : #include "common/file_perm.h"
21 : #include "common/logging.h"
22 : #include "lib/stringinfo.h"
23 : #include "load_manifest.h"
24 : #include "mb/pg_wchar.h"
25 : #include "write_manifest.h"
26 :
27 : struct manifest_writer
28 : {
29 : char pathname[MAXPGPATH];
30 : int fd;
31 : StringInfoData buf;
32 : bool first_file;
33 : bool still_checksumming;
34 : pg_checksum_context manifest_ctx;
35 : };
36 :
37 : static void escape_json(StringInfo buf, const char *str);
38 : static void flush_manifest(manifest_writer *mwriter);
39 : static size_t hex_encode(const uint8 *src, size_t len, char *dst);
40 :
41 : /*
42 : * Create a new backup manifest writer.
43 : *
44 : * The backup manifest will be written into a file named backup_manifest
45 : * in the specified directory.
46 : */
47 : manifest_writer *
48 18 : create_manifest_writer(char *directory, uint64 system_identifier)
49 : {
50 18 : manifest_writer *mwriter = pg_malloc(sizeof(manifest_writer));
51 :
52 18 : snprintf(mwriter->pathname, MAXPGPATH, "%s/backup_manifest", directory);
53 18 : mwriter->fd = -1;
54 18 : initStringInfo(&mwriter->buf);
55 18 : mwriter->first_file = true;
56 18 : mwriter->still_checksumming = true;
57 18 : pg_checksum_init(&mwriter->manifest_ctx, CHECKSUM_TYPE_SHA256);
58 :
59 18 : appendStringInfo(&mwriter->buf,
60 : "{ \"PostgreSQL-Backup-Manifest-Version\": 2,\n"
61 : "\"System-Identifier\": " UINT64_FORMAT ",\n"
62 : "\"Files\": [",
63 : system_identifier);
64 :
65 18 : return mwriter;
66 : }
67 :
68 : /*
69 : * Add an entry for a file to a backup manifest.
70 : *
71 : * This is very similar to the backend's AddFileToBackupManifest, but
72 : * various adjustments are required due to frontend/backend differences
73 : * and other details.
74 : */
75 : void
76 18680 : add_file_to_manifest(manifest_writer *mwriter, const char *manifest_path,
77 : size_t size, time_t mtime,
78 : pg_checksum_type checksum_type,
79 : int checksum_length,
80 : uint8 *checksum_payload)
81 : {
82 18680 : int pathlen = strlen(manifest_path);
83 :
84 18680 : if (mwriter->first_file)
85 : {
86 18 : appendStringInfoChar(&mwriter->buf, '\n');
87 18 : mwriter->first_file = false;
88 : }
89 : else
90 18662 : appendStringInfoString(&mwriter->buf, ",\n");
91 :
92 18680 : if (pg_encoding_verifymbstr(PG_UTF8, manifest_path, pathlen) == pathlen)
93 : {
94 18680 : appendStringInfoString(&mwriter->buf, "{ \"Path\": ");
95 18680 : escape_json(&mwriter->buf, manifest_path);
96 18680 : appendStringInfoString(&mwriter->buf, ", ");
97 : }
98 : else
99 : {
100 0 : appendStringInfoString(&mwriter->buf, "{ \"Encoded-Path\": \"");
101 0 : enlargeStringInfo(&mwriter->buf, 2 * pathlen);
102 0 : mwriter->buf.len += hex_encode((const uint8 *) manifest_path, pathlen,
103 0 : &mwriter->buf.data[mwriter->buf.len]);
104 0 : appendStringInfoString(&mwriter->buf, "\", ");
105 : }
106 :
107 18680 : appendStringInfo(&mwriter->buf, "\"Size\": %zu, ", size);
108 :
109 18680 : appendStringInfoString(&mwriter->buf, "\"Last-Modified\": \"");
110 18680 : enlargeStringInfo(&mwriter->buf, 128);
111 18680 : mwriter->buf.len += strftime(&mwriter->buf.data[mwriter->buf.len], 128,
112 : "%Y-%m-%d %H:%M:%S %Z",
113 18680 : gmtime(&mtime));
114 18680 : appendStringInfoChar(&mwriter->buf, '"');
115 :
116 18680 : if (mwriter->buf.len > 128 * 1024)
117 14 : flush_manifest(mwriter);
118 :
119 18680 : if (checksum_length > 0)
120 : {
121 16728 : appendStringInfo(&mwriter->buf,
122 : ", \"Checksum-Algorithm\": \"%s\", \"Checksum\": \"",
123 : pg_checksum_type_name(checksum_type));
124 :
125 16728 : enlargeStringInfo(&mwriter->buf, 2 * checksum_length);
126 33456 : mwriter->buf.len += hex_encode(checksum_payload, checksum_length,
127 16728 : &mwriter->buf.data[mwriter->buf.len]);
128 :
129 16728 : appendStringInfoChar(&mwriter->buf, '"');
130 : }
131 :
132 18680 : appendStringInfoString(&mwriter->buf, " }");
133 :
134 18680 : if (mwriter->buf.len > 128 * 1024)
135 2 : flush_manifest(mwriter);
136 18680 : }
137 :
138 : /*
139 : * Finalize the backup_manifest.
140 : */
141 : void
142 18 : finalize_manifest(manifest_writer *mwriter,
143 : manifest_wal_range *first_wal_range)
144 : {
145 : uint8 checksumbuf[PG_SHA256_DIGEST_LENGTH];
146 : int len;
147 : manifest_wal_range *wal_range;
148 :
149 : /* Terminate the list of files. */
150 18 : appendStringInfoString(&mwriter->buf, "\n],\n");
151 :
152 : /* Start a list of LSN ranges. */
153 18 : appendStringInfoString(&mwriter->buf, "\"WAL-Ranges\": [\n");
154 :
155 36 : for (wal_range = first_wal_range; wal_range != NULL;
156 18 : wal_range = wal_range->next)
157 18 : appendStringInfo(&mwriter->buf,
158 : "%s{ \"Timeline\": %u, \"Start-LSN\": \"%X/%X\", \"End-LSN\": \"%X/%X\" }",
159 : wal_range == first_wal_range ? "" : ",\n",
160 : wal_range->tli,
161 18 : LSN_FORMAT_ARGS(wal_range->start_lsn),
162 18 : LSN_FORMAT_ARGS(wal_range->end_lsn));
163 :
164 : /* Terminate the list of WAL ranges. */
165 18 : appendStringInfoString(&mwriter->buf, "\n],\n");
166 :
167 : /* Flush accumulated data and update checksum calculation. */
168 18 : flush_manifest(mwriter);
169 :
170 : /* Checksum only includes data up to this point. */
171 18 : mwriter->still_checksumming = false;
172 :
173 : /* Compute and insert manifest checksum. */
174 18 : appendStringInfoString(&mwriter->buf, "\"Manifest-Checksum\": \"");
175 18 : enlargeStringInfo(&mwriter->buf, 2 * PG_SHA256_DIGEST_STRING_LENGTH);
176 18 : len = pg_checksum_final(&mwriter->manifest_ctx, checksumbuf);
177 : Assert(len == PG_SHA256_DIGEST_LENGTH);
178 18 : mwriter->buf.len +=
179 18 : hex_encode(checksumbuf, len, &mwriter->buf.data[mwriter->buf.len]);
180 18 : appendStringInfoString(&mwriter->buf, "\"}\n");
181 :
182 : /* Flush the last manifest checksum itself. */
183 18 : flush_manifest(mwriter);
184 :
185 : /* Close the file. */
186 18 : if (close(mwriter->fd) != 0)
187 0 : pg_fatal("could not close \"%s\": %m", mwriter->pathname);
188 18 : mwriter->fd = -1;
189 18 : }
190 :
191 : /*
192 : * Produce a JSON string literal, properly escaping characters in the text.
193 : */
194 : static void
195 18680 : escape_json(StringInfo buf, const char *str)
196 : {
197 : const char *p;
198 :
199 18680 : appendStringInfoCharMacro(buf, '"');
200 249348 : for (p = str; *p; p++)
201 : {
202 230668 : switch (*p)
203 : {
204 0 : case '\b':
205 0 : appendStringInfoString(buf, "\\b");
206 0 : break;
207 0 : case '\f':
208 0 : appendStringInfoString(buf, "\\f");
209 0 : break;
210 0 : case '\n':
211 0 : appendStringInfoString(buf, "\\n");
212 0 : break;
213 0 : case '\r':
214 0 : appendStringInfoString(buf, "\\r");
215 0 : break;
216 0 : case '\t':
217 0 : appendStringInfoString(buf, "\\t");
218 0 : break;
219 0 : case '"':
220 0 : appendStringInfoString(buf, "\\\"");
221 0 : break;
222 0 : case '\\':
223 0 : appendStringInfoString(buf, "\\\\");
224 0 : break;
225 230668 : default:
226 230668 : if ((unsigned char) *p < ' ')
227 0 : appendStringInfo(buf, "\\u%04x", (int) *p);
228 : else
229 230668 : appendStringInfoCharMacro(buf, *p);
230 230668 : break;
231 : }
232 : }
233 18680 : appendStringInfoCharMacro(buf, '"');
234 18680 : }
235 :
236 : /*
237 : * Flush whatever portion of the backup manifest we have generated and
238 : * buffered in memory out to a file on disk.
239 : *
240 : * The first call to this function will create the file. After that, we
241 : * keep it open and just append more data.
242 : */
243 : static void
244 52 : flush_manifest(manifest_writer *mwriter)
245 : {
246 52 : if (mwriter->fd == -1 &&
247 18 : (mwriter->fd = open(mwriter->pathname,
248 : O_WRONLY | O_CREAT | O_EXCL | PG_BINARY,
249 : pg_file_create_mode)) < 0)
250 0 : pg_fatal("could not open file \"%s\": %m", mwriter->pathname);
251 :
252 52 : if (mwriter->buf.len > 0)
253 : {
254 : ssize_t wb;
255 :
256 52 : wb = write(mwriter->fd, mwriter->buf.data, mwriter->buf.len);
257 52 : if (wb != mwriter->buf.len)
258 : {
259 0 : if (wb < 0)
260 0 : pg_fatal("could not write \"%s\": %m", mwriter->pathname);
261 : else
262 0 : pg_fatal("could not write file \"%s\": wrote only %d of %d bytes",
263 : mwriter->pathname, (int) wb, mwriter->buf.len);
264 : }
265 :
266 86 : if (mwriter->still_checksumming &&
267 34 : pg_checksum_update(&mwriter->manifest_ctx,
268 34 : (uint8 *) mwriter->buf.data,
269 34 : mwriter->buf.len) < 0)
270 0 : pg_fatal("could not update checksum of file \"%s\"",
271 : mwriter->pathname);
272 52 : resetStringInfo(&mwriter->buf);
273 : }
274 52 : }
275 :
276 : /*
277 : * Encode bytes using two hexadecimal digits for each one.
278 : */
279 : static size_t
280 16746 : hex_encode(const uint8 *src, size_t len, char *dst)
281 : {
282 16746 : const uint8 *end = src + len;
283 :
284 130602 : while (src < end)
285 : {
286 113856 : unsigned n1 = (*src >> 4) & 0xF;
287 113856 : unsigned n2 = *src & 0xF;
288 :
289 113856 : *dst++ = n1 < 10 ? '0' + n1 : 'a' + n1 - 10;
290 113856 : *dst++ = n2 < 10 ? '0' + n2 : 'a' + n2 - 10;
291 113856 : ++src;
292 : }
293 :
294 16746 : return len * 2;
295 : }
|
