Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * parsexlog.c
4 : * Functions for reading Write-Ahead-Log
5 : *
6 : * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7 : * Portions Copyright (c) 1994, Regents of the University of California
8 : *
9 : *-------------------------------------------------------------------------
10 : */
11 :
12 : #include "postgres_fe.h"
13 :
14 : #include <unistd.h>
15 :
16 : #include "access/rmgr.h"
17 : #include "access/xlog_internal.h"
18 : #include "access/xlogreader.h"
19 : #include "catalog/pg_control.h"
20 : #include "catalog/storage_xlog.h"
21 : #include "commands/dbcommands_xlog.h"
22 : #include "common/fe_archive.h"
23 : #include "filemap.h"
24 : #include "pg_rewind.h"
25 :
26 : /*
27 : * RmgrNames is an array of resource manager names, to make error messages
28 : * a bit nicer.
29 : */
30 : #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
31 : name,
32 :
33 : static const char *RmgrNames[RM_MAX_ID + 1] = {
34 : #include "access/rmgrlist.h"
35 : };
36 :
37 : static void extractPageInfo(XLogReaderState *record);
38 :
39 : static int xlogreadfd = -1;
40 : static XLogSegNo xlogreadsegno = -1;
41 : static char xlogfpath[MAXPGPATH];
42 :
43 : typedef struct XLogPageReadPrivate
44 : {
45 : const char *restoreCommand;
46 : int tliIndex;
47 : } XLogPageReadPrivate;
48 :
49 : static int SimpleXLogPageRead(XLogReaderState *xlogreader,
50 : XLogRecPtr targetPagePtr,
51 : int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
52 :
53 : /*
54 : * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
55 : * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
56 : * the data blocks touched by the WAL records, and return them in a page map.
57 : */
58 : void
59 20 : extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
60 : XLogRecPtr endpoint, const char *restoreCommand)
61 : {
62 : XLogRecord *record;
63 : XLogReaderState *xlogreader;
64 : char *errormsg;
65 : XLogPageReadPrivate private;
66 :
67 20 : private.tliIndex = tliIndex;
68 20 : private.restoreCommand = restoreCommand;
69 20 : xlogreader = XLogReaderAllocate(WalSegSz, datadir,
70 20 : XL_ROUTINE(.page_read = &SimpleXLogPageRead),
71 : &private);
72 20 : if (xlogreader == NULL)
73 0 : pg_fatal("out of memory");
74 :
75 20 : XLogBeginRead(xlogreader, startpoint);
76 : do
77 : {
78 163236 : record = XLogReadRecord(xlogreader, &errormsg);
79 :
80 163236 : if (record == NULL)
81 : {
82 0 : XLogRecPtr errptr = xlogreader->EndRecPtr;
83 :
84 0 : if (errormsg)
85 0 : pg_fatal("could not read WAL record at %X/%X: %s",
86 : (uint32) (errptr >> 32), (uint32) (errptr),
87 : errormsg);
88 : else
89 0 : pg_fatal("could not read WAL record at %X/%X",
90 : (uint32) (errptr >> 32), (uint32) (errptr));
91 : }
92 :
93 163236 : extractPageInfo(xlogreader);
94 :
95 163236 : } while (xlogreader->ReadRecPtr != endpoint);
96 :
97 20 : XLogReaderFree(xlogreader);
98 20 : if (xlogreadfd != -1)
99 : {
100 20 : close(xlogreadfd);
101 20 : xlogreadfd = -1;
102 : }
103 20 : }
104 :
105 : /*
106 : * Reads one WAL record. Returns the end position of the record, without
107 : * doing anything with the record itself.
108 : */
109 : XLogRecPtr
110 0 : readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
111 : const char *restoreCommand)
112 : {
113 : XLogRecord *record;
114 : XLogReaderState *xlogreader;
115 : char *errormsg;
116 : XLogPageReadPrivate private;
117 : XLogRecPtr endptr;
118 :
119 0 : private.tliIndex = tliIndex;
120 0 : private.restoreCommand = restoreCommand;
121 0 : xlogreader = XLogReaderAllocate(WalSegSz, datadir,
122 0 : XL_ROUTINE(.page_read = &SimpleXLogPageRead),
123 : &private);
124 0 : if (xlogreader == NULL)
125 0 : pg_fatal("out of memory");
126 :
127 0 : XLogBeginRead(xlogreader, ptr);
128 0 : record = XLogReadRecord(xlogreader, &errormsg);
129 0 : if (record == NULL)
130 : {
131 0 : if (errormsg)
132 0 : pg_fatal("could not read WAL record at %X/%X: %s",
133 : (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
134 : else
135 0 : pg_fatal("could not read WAL record at %X/%X",
136 : (uint32) (ptr >> 32), (uint32) (ptr));
137 : }
138 0 : endptr = xlogreader->EndRecPtr;
139 :
140 0 : XLogReaderFree(xlogreader);
141 0 : if (xlogreadfd != -1)
142 : {
143 0 : close(xlogreadfd);
144 0 : xlogreadfd = -1;
145 : }
146 :
147 0 : return endptr;
148 : }
149 :
150 : /*
151 : * Find the previous checkpoint preceding given WAL location.
152 : */
153 : void
154 20 : findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
155 : XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
156 : XLogRecPtr *lastchkptredo, const char *restoreCommand)
157 : {
158 : /* Walk backwards, starting from the given record */
159 : XLogRecord *record;
160 : XLogRecPtr searchptr;
161 : XLogReaderState *xlogreader;
162 : char *errormsg;
163 : XLogPageReadPrivate private;
164 :
165 : /*
166 : * The given fork pointer points to the end of the last common record,
167 : * which is not necessarily the beginning of the next record, if the
168 : * previous record happens to end at a page boundary. Skip over the page
169 : * header in that case to find the next record.
170 : */
171 20 : if (forkptr % XLOG_BLCKSZ == 0)
172 : {
173 4 : if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
174 4 : forkptr += SizeOfXLogLongPHD;
175 : else
176 0 : forkptr += SizeOfXLogShortPHD;
177 : }
178 :
179 20 : private.tliIndex = tliIndex;
180 20 : private.restoreCommand = restoreCommand;
181 20 : xlogreader = XLogReaderAllocate(WalSegSz, datadir,
182 20 : XL_ROUTINE(.page_read = &SimpleXLogPageRead),
183 : &private);
184 20 : if (xlogreader == NULL)
185 0 : pg_fatal("out of memory");
186 :
187 20 : searchptr = forkptr;
188 : for (;;)
189 212 : {
190 : uint8 info;
191 :
192 232 : XLogBeginRead(xlogreader, searchptr);
193 232 : record = XLogReadRecord(xlogreader, &errormsg);
194 :
195 232 : if (record == NULL)
196 : {
197 0 : if (errormsg)
198 0 : pg_fatal("could not find previous WAL record at %X/%X: %s",
199 : (uint32) (searchptr >> 32), (uint32) (searchptr),
200 : errormsg);
201 : else
202 0 : pg_fatal("could not find previous WAL record at %X/%X",
203 : (uint32) (searchptr >> 32), (uint32) (searchptr));
204 : }
205 :
206 : /*
207 : * Check if it is a checkpoint record. This checkpoint record needs to
208 : * be the latest checkpoint before WAL forked and not the checkpoint
209 : * where the master has been stopped to be rewinded.
210 : */
211 232 : info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
212 232 : if (searchptr < forkptr &&
213 212 : XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
214 36 : (info == XLOG_CHECKPOINT_SHUTDOWN ||
215 : info == XLOG_CHECKPOINT_ONLINE))
216 : {
217 : CheckPoint checkPoint;
218 :
219 20 : memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
220 20 : *lastchkptrec = searchptr;
221 20 : *lastchkpttli = checkPoint.ThisTimeLineID;
222 20 : *lastchkptredo = checkPoint.redo;
223 20 : break;
224 : }
225 :
226 : /* Walk backwards to previous record. */
227 212 : searchptr = record->xl_prev;
228 : }
229 :
230 20 : XLogReaderFree(xlogreader);
231 20 : if (xlogreadfd != -1)
232 : {
233 20 : close(xlogreadfd);
234 20 : xlogreadfd = -1;
235 : }
236 20 : }
237 :
238 : /* XLogReader callback function, to read a WAL page */
239 : static int
240 2474 : SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
241 : int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
242 : {
243 2474 : XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
244 : uint32 targetPageOff;
245 : XLogRecPtr targetSegEnd;
246 : XLogSegNo targetSegNo;
247 : int r;
248 :
249 2474 : XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
250 2474 : XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
251 2474 : targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
252 :
253 : /*
254 : * See if we need to switch to a new segment because the requested record
255 : * is not in the currently open one.
256 : */
257 2474 : if (xlogreadfd >= 0 &&
258 2434 : !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
259 : {
260 8 : close(xlogreadfd);
261 8 : xlogreadfd = -1;
262 : }
263 :
264 2474 : XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
265 :
266 2474 : if (xlogreadfd < 0)
267 : {
268 : char xlogfname[MAXFNAMELEN];
269 :
270 : /*
271 : * Since incomplete segments are copied into next timelines, switch to
272 : * the timeline holding the required segment. Assuming this scan can
273 : * be done both forward and backward, consider also switching timeline
274 : * accordingly.
275 : */
276 48 : while (private->tliIndex < targetNentries - 1 &&
277 0 : targetHistory[private->tliIndex].end < targetSegEnd)
278 0 : private->tliIndex++;
279 48 : while (private->tliIndex > 0 &&
280 0 : targetHistory[private->tliIndex].begin >= targetSegEnd)
281 0 : private->tliIndex--;
282 :
283 48 : XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
284 : xlogreadsegno, WalSegSz);
285 :
286 48 : snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
287 48 : xlogreader->segcxt.ws_dir, xlogfname);
288 :
289 48 : xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
290 :
291 48 : if (xlogreadfd < 0)
292 : {
293 : /*
294 : * If we have no restore_command to execute, then exit.
295 : */
296 2 : if (private->restoreCommand == NULL)
297 : {
298 0 : pg_log_error("could not open file \"%s\": %m", xlogfpath);
299 0 : return -1;
300 : }
301 :
302 : /*
303 : * Since we have restore_command, then try to retrieve missing WAL
304 : * file from the archive.
305 : */
306 2 : xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
307 : xlogfname,
308 : WalSegSz,
309 : private->restoreCommand);
310 :
311 2 : if (xlogreadfd < 0)
312 0 : return -1;
313 : else
314 2 : pg_log_debug("using file \"%s\" restored from archive",
315 : xlogfpath);
316 : }
317 : }
318 :
319 : /*
320 : * At this point, we have the right segment open.
321 : */
322 : Assert(xlogreadfd != -1);
323 :
324 : /* Read the requested page */
325 2474 : if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
326 : {
327 0 : pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
328 0 : return -1;
329 : }
330 :
331 :
332 2474 : r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
333 2474 : if (r != XLOG_BLCKSZ)
334 : {
335 0 : if (r < 0)
336 0 : pg_log_error("could not read file \"%s\": %m", xlogfpath);
337 : else
338 0 : pg_log_error("could not read file \"%s\": read %d of %zu",
339 : xlogfpath, r, (Size) XLOG_BLCKSZ);
340 :
341 0 : return -1;
342 : }
343 :
344 : Assert(targetSegNo == xlogreadsegno);
345 :
346 2474 : xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
347 2474 : return XLOG_BLCKSZ;
348 : }
349 :
350 : /*
351 : * Extract information on which blocks the current record modifies.
352 : */
353 : static void
354 163236 : extractPageInfo(XLogReaderState *record)
355 : {
356 : int block_id;
357 163236 : RmgrId rmid = XLogRecGetRmid(record);
358 163236 : uint8 info = XLogRecGetInfo(record);
359 163236 : uint8 rminfo = info & ~XLR_INFO_MASK;
360 :
361 : /* Is this a special record type that I recognize? */
362 :
363 163236 : if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
364 : {
365 : /*
366 : * New databases can be safely ignored. It won't be present in the
367 : * source system, so it will be deleted. There's one corner-case,
368 : * though: if a new, different, database is also created in the source
369 : * system, we'll see that the files already exist and not copy them.
370 : * That's OK, though; WAL replay of creating the new database, from
371 : * the source systems's WAL, will re-copy the new database,
372 : * overwriting the database created in the target system.
373 : */
374 : }
375 163232 : else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
376 : {
377 : /*
378 : * An existing database was dropped. We'll see that the files don't
379 : * exist in the target data dir, and copy them in toto from the source
380 : * system. No need to do anything special here.
381 : */
382 : }
383 163232 : else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
384 : {
385 : /*
386 : * We can safely ignore these. The file will be removed from the
387 : * target, if it doesn't exist in source system. If a file with same
388 : * name is created in source system, too, there will be WAL records
389 : * for all the blocks in it.
390 : */
391 : }
392 163224 : else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
393 : {
394 : /*
395 : * We can safely ignore these. When we compare the sizes later on,
396 : * we'll notice that they differ, and copy the missing tail from
397 : * source system.
398 : */
399 : }
400 163216 : else if (info & XLR_SPECIAL_REL_UPDATE)
401 : {
402 : /*
403 : * This record type modifies a relation file in some special way, but
404 : * we don't recognize the type. That's bad - we don't know how to
405 : * track that change.
406 : */
407 0 : pg_fatal("WAL record modifies a relation, but record type is not recognized: "
408 : "lsn: %X/%X, rmgr: %s, info: %02X",
409 : (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
410 : RmgrNames[rmid], info);
411 : }
412 :
413 326974 : for (block_id = 0; block_id <= record->max_block_id; block_id++)
414 : {
415 : RelFileNode rnode;
416 : ForkNumber forknum;
417 : BlockNumber blkno;
418 :
419 163738 : if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
420 728 : continue;
421 :
422 : /* We only care about the main fork; others are copied in toto */
423 163738 : if (forknum != MAIN_FORKNUM)
424 728 : continue;
425 :
426 163010 : process_block_change(forknum, rnode, blkno);
427 : }
428 163236 : }
|
