Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * parsexlog.c
4 : * Functions for reading Write-Ahead-Log
5 : *
6 : * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
7 : * Portions Copyright (c) 1994, Regents of the University of California
8 : *
9 : *-------------------------------------------------------------------------
10 : */
11 :
12 : #include "postgres_fe.h"
13 :
14 : #include <unistd.h>
15 :
16 : #include "access/rmgr.h"
17 : #include "access/xact.h"
18 : #include "access/xlog_internal.h"
19 : #include "access/xlogreader.h"
20 : #include "catalog/pg_control.h"
21 : #include "catalog/storage_xlog.h"
22 : #include "commands/dbcommands_xlog.h"
23 : #include "fe_utils/archive.h"
24 : #include "filemap.h"
25 : #include "pg_rewind.h"
26 :
27 : /*
28 : * RmgrNames is an array of resource manager names, to make error messages
29 : * a bit nicer.
30 : */
31 : #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
32 : name,
33 :
34 : static const char *RmgrNames[RM_MAX_ID + 1] = {
35 : #include "access/rmgrlist.h"
36 : };
37 :
38 : static void extractPageInfo(XLogReaderState *record);
39 :
40 : static int xlogreadfd = -1;
41 : static XLogSegNo xlogreadsegno = -1;
42 : static char xlogfpath[MAXPGPATH];
43 :
44 : typedef struct XLogPageReadPrivate
45 : {
46 : const char *restoreCommand;
47 : int tliIndex;
48 : } XLogPageReadPrivate;
49 :
50 : static int SimpleXLogPageRead(XLogReaderState *xlogreader,
51 : XLogRecPtr targetPagePtr,
52 : int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
53 :
54 : /*
55 : * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
56 : * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
57 : * the data blocks touched by the WAL records, and return them in a page map.
58 : *
59 : * 'endpoint' is the end of the last record to read. The record starting at
60 : * 'endpoint' is the first one that is not read.
61 : */
62 : void
63 24 : extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
64 : XLogRecPtr endpoint, const char *restoreCommand)
65 : {
66 : XLogRecord *record;
67 : XLogReaderState *xlogreader;
68 : char *errormsg;
69 : XLogPageReadPrivate private;
70 :
71 24 : private.tliIndex = tliIndex;
72 24 : private.restoreCommand = restoreCommand;
73 24 : xlogreader = XLogReaderAllocate(WalSegSz, datadir,
74 24 : XL_ROUTINE(.page_read = &SimpleXLogPageRead),
75 : &private);
76 24 : if (xlogreader == NULL)
77 0 : pg_fatal("out of memory");
78 :
79 24 : XLogBeginRead(xlogreader, startpoint);
80 : do
81 : {
82 163558 : record = XLogReadRecord(xlogreader, &errormsg);
83 :
84 163558 : if (record == NULL)
85 : {
86 0 : XLogRecPtr errptr = xlogreader->EndRecPtr;
87 :
88 0 : if (errormsg)
89 0 : pg_fatal("could not read WAL record at %X/%X: %s",
90 : (uint32) (errptr >> 32), (uint32) (errptr),
91 : errormsg);
92 : else
93 0 : pg_fatal("could not read WAL record at %X/%X",
94 : (uint32) (errptr >> 32), (uint32) (errptr));
95 : }
96 :
97 163558 : extractPageInfo(xlogreader);
98 :
99 163558 : } while (xlogreader->EndRecPtr < endpoint);
100 :
101 : /*
102 : * If 'endpoint' didn't point exactly at a record boundary, the caller
103 : * messed up.
104 : */
105 : Assert(xlogreader->EndRecPtr == endpoint);
106 :
107 24 : XLogReaderFree(xlogreader);
108 24 : if (xlogreadfd != -1)
109 : {
110 24 : close(xlogreadfd);
111 24 : xlogreadfd = -1;
112 : }
113 24 : }
114 :
115 : /*
116 : * Reads one WAL record. Returns the end position of the record, without
117 : * doing anything with the record itself.
118 : */
119 : XLogRecPtr
120 24 : readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
121 : const char *restoreCommand)
122 : {
123 : XLogRecord *record;
124 : XLogReaderState *xlogreader;
125 : char *errormsg;
126 : XLogPageReadPrivate private;
127 : XLogRecPtr endptr;
128 :
129 24 : private.tliIndex = tliIndex;
130 24 : private.restoreCommand = restoreCommand;
131 24 : xlogreader = XLogReaderAllocate(WalSegSz, datadir,
132 24 : XL_ROUTINE(.page_read = &SimpleXLogPageRead),
133 : &private);
134 24 : if (xlogreader == NULL)
135 0 : pg_fatal("out of memory");
136 :
137 24 : XLogBeginRead(xlogreader, ptr);
138 24 : record = XLogReadRecord(xlogreader, &errormsg);
139 24 : if (record == NULL)
140 : {
141 0 : if (errormsg)
142 0 : pg_fatal("could not read WAL record at %X/%X: %s",
143 : (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
144 : else
145 0 : pg_fatal("could not read WAL record at %X/%X",
146 : (uint32) (ptr >> 32), (uint32) (ptr));
147 : }
148 24 : endptr = xlogreader->EndRecPtr;
149 :
150 24 : XLogReaderFree(xlogreader);
151 24 : if (xlogreadfd != -1)
152 : {
153 24 : close(xlogreadfd);
154 24 : xlogreadfd = -1;
155 : }
156 :
157 24 : return endptr;
158 : }
159 :
160 : /*
161 : * Find the previous checkpoint preceding given WAL location.
162 : */
163 : void
164 24 : findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
165 : XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
166 : XLogRecPtr *lastchkptredo, const char *restoreCommand)
167 : {
168 : /* Walk backwards, starting from the given record */
169 : XLogRecord *record;
170 : XLogRecPtr searchptr;
171 : XLogReaderState *xlogreader;
172 : char *errormsg;
173 : XLogPageReadPrivate private;
174 :
175 : /*
176 : * The given fork pointer points to the end of the last common record,
177 : * which is not necessarily the beginning of the next record, if the
178 : * previous record happens to end at a page boundary. Skip over the page
179 : * header in that case to find the next record.
180 : */
181 24 : if (forkptr % XLOG_BLCKSZ == 0)
182 : {
183 4 : if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
184 4 : forkptr += SizeOfXLogLongPHD;
185 : else
186 0 : forkptr += SizeOfXLogShortPHD;
187 : }
188 :
189 24 : private.tliIndex = tliIndex;
190 24 : private.restoreCommand = restoreCommand;
191 24 : xlogreader = XLogReaderAllocate(WalSegSz, datadir,
192 24 : XL_ROUTINE(.page_read = &SimpleXLogPageRead),
193 : &private);
194 24 : if (xlogreader == NULL)
195 0 : pg_fatal("out of memory");
196 :
197 24 : searchptr = forkptr;
198 : for (;;)
199 200 : {
200 : uint8 info;
201 :
202 224 : XLogBeginRead(xlogreader, searchptr);
203 224 : record = XLogReadRecord(xlogreader, &errormsg);
204 :
205 224 : if (record == NULL)
206 : {
207 0 : if (errormsg)
208 0 : pg_fatal("could not find previous WAL record at %X/%X: %s",
209 : (uint32) (searchptr >> 32), (uint32) (searchptr),
210 : errormsg);
211 : else
212 0 : pg_fatal("could not find previous WAL record at %X/%X",
213 : (uint32) (searchptr >> 32), (uint32) (searchptr));
214 : }
215 :
216 : /*
217 : * Check if it is a checkpoint record. This checkpoint record needs to
218 : * be the latest checkpoint before WAL forked and not the checkpoint
219 : * where the primary has been stopped to be rewound.
220 : */
221 224 : info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
222 224 : if (searchptr < forkptr &&
223 200 : XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
224 40 : (info == XLOG_CHECKPOINT_SHUTDOWN ||
225 : info == XLOG_CHECKPOINT_ONLINE))
226 : {
227 : CheckPoint checkPoint;
228 :
229 24 : memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
230 24 : *lastchkptrec = searchptr;
231 24 : *lastchkpttli = checkPoint.ThisTimeLineID;
232 24 : *lastchkptredo = checkPoint.redo;
233 24 : break;
234 : }
235 :
236 : /* Walk backwards to previous record. */
237 200 : searchptr = record->xl_prev;
238 : }
239 :
240 24 : XLogReaderFree(xlogreader);
241 24 : if (xlogreadfd != -1)
242 : {
243 24 : close(xlogreadfd);
244 24 : xlogreadfd = -1;
245 : }
246 24 : }
247 :
248 : /* XLogReader callback function, to read a WAL page */
249 : static int
250 2666 : SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
251 : int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
252 : {
253 2666 : XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
254 : uint32 targetPageOff;
255 : XLogRecPtr targetSegEnd;
256 : XLogSegNo targetSegNo;
257 : int r;
258 :
259 2666 : XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
260 2666 : XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
261 2666 : targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
262 :
263 : /*
264 : * See if we need to switch to a new segment because the requested record
265 : * is not in the currently open one.
266 : */
267 2666 : if (xlogreadfd >= 0 &&
268 2594 : !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
269 : {
270 8 : close(xlogreadfd);
271 8 : xlogreadfd = -1;
272 : }
273 :
274 2666 : XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
275 :
276 2666 : if (xlogreadfd < 0)
277 : {
278 : char xlogfname[MAXFNAMELEN];
279 :
280 : /*
281 : * Since incomplete segments are copied into next timelines, switch to
282 : * the timeline holding the required segment. Assuming this scan can
283 : * be done both forward and backward, consider also switching timeline
284 : * accordingly.
285 : */
286 84 : while (private->tliIndex < targetNentries - 1 &&
287 4 : targetHistory[private->tliIndex].end < targetSegEnd)
288 4 : private->tliIndex++;
289 80 : while (private->tliIndex > 0 &&
290 12 : targetHistory[private->tliIndex].begin >= targetSegEnd)
291 0 : private->tliIndex--;
292 :
293 80 : XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
294 : xlogreadsegno, WalSegSz);
295 :
296 80 : snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
297 80 : xlogreader->segcxt.ws_dir, xlogfname);
298 :
299 80 : xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
300 :
301 80 : if (xlogreadfd < 0)
302 : {
303 : /*
304 : * If we have no restore_command to execute, then exit.
305 : */
306 2 : if (private->restoreCommand == NULL)
307 : {
308 0 : pg_log_error("could not open file \"%s\": %m", xlogfpath);
309 0 : return -1;
310 : }
311 :
312 : /*
313 : * Since we have restore_command, then try to retrieve missing WAL
314 : * file from the archive.
315 : */
316 2 : xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
317 : xlogfname,
318 : WalSegSz,
319 : private->restoreCommand);
320 :
321 2 : if (xlogreadfd < 0)
322 0 : return -1;
323 : else
324 2 : pg_log_debug("using file \"%s\" restored from archive",
325 : xlogfpath);
326 : }
327 : }
328 :
329 : /*
330 : * At this point, we have the right segment open.
331 : */
332 : Assert(xlogreadfd != -1);
333 :
334 : /* Read the requested page */
335 2666 : if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
336 : {
337 0 : pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
338 0 : return -1;
339 : }
340 :
341 :
342 2666 : r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
343 2666 : if (r != XLOG_BLCKSZ)
344 : {
345 0 : if (r < 0)
346 0 : pg_log_error("could not read file \"%s\": %m", xlogfpath);
347 : else
348 0 : pg_log_error("could not read file \"%s\": read %d of %zu",
349 : xlogfpath, r, (Size) XLOG_BLCKSZ);
350 :
351 0 : return -1;
352 : }
353 :
354 : Assert(targetSegNo == xlogreadsegno);
355 :
356 2666 : xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
357 2666 : return XLOG_BLCKSZ;
358 : }
359 :
360 : /*
361 : * Extract information on which blocks the current record modifies.
362 : */
363 : static void
364 163558 : extractPageInfo(XLogReaderState *record)
365 : {
366 : int block_id;
367 163558 : RmgrId rmid = XLogRecGetRmid(record);
368 163558 : uint8 info = XLogRecGetInfo(record);
369 163558 : uint8 rminfo = info & ~XLR_INFO_MASK;
370 :
371 : /* Is this a special record type that I recognize? */
372 :
373 163558 : if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
374 : {
375 : /*
376 : * New databases can be safely ignored. It won't be present in the
377 : * source system, so it will be deleted. There's one corner-case,
378 : * though: if a new, different, database is also created in the source
379 : * system, we'll see that the files already exist and not copy them.
380 : * That's OK, though; WAL replay of creating the new database, from
381 : * the source systems's WAL, will re-copy the new database,
382 : * overwriting the database created in the target system.
383 : */
384 : }
385 163554 : else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
386 : {
387 : /*
388 : * An existing database was dropped. We'll see that the files don't
389 : * exist in the target data dir, and copy them in toto from the source
390 : * system. No need to do anything special here.
391 : */
392 : }
393 163554 : else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
394 : {
395 : /*
396 : * We can safely ignore these. The file will be removed from the
397 : * target, if it doesn't exist in source system. If a file with same
398 : * name is created in source system, too, there will be WAL records
399 : * for all the blocks in it.
400 : */
401 : }
402 163546 : else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
403 : {
404 : /*
405 : * We can safely ignore these. When we compare the sizes later on,
406 : * we'll notice that they differ, and copy the missing tail from
407 : * source system.
408 : */
409 : }
410 163610 : else if (rmid == RM_XACT_ID &&
411 72 : ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT ||
412 0 : (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT_PREPARED ||
413 0 : (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT ||
414 0 : (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT_PREPARED))
415 : {
416 : /*
417 : * These records can include "dropped rels". We can safely ignore
418 : * them, we will see that they are missing and copy them from the
419 : * source.
420 : */
421 : }
422 163466 : else if (info & XLR_SPECIAL_REL_UPDATE)
423 : {
424 : /*
425 : * This record type modifies a relation file in some special way, but
426 : * we don't recognize the type. That's bad - we don't know how to
427 : * track that change.
428 : */
429 0 : pg_fatal("WAL record modifies a relation, but record type is not recognized: "
430 : "lsn: %X/%X, rmgr: %s, info: %02X",
431 : (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
432 : RmgrNames[rmid], info);
433 : }
434 :
435 327566 : for (block_id = 0; block_id <= record->max_block_id; block_id++)
436 : {
437 : RelFileNode rnode;
438 : ForkNumber forknum;
439 : BlockNumber blkno;
440 :
441 164008 : if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
442 720 : continue;
443 :
444 : /* We only care about the main fork; others are copied in toto */
445 164008 : if (forknum != MAIN_FORKNUM)
446 720 : continue;
447 :
448 163288 : process_target_wal_block_change(forknum, rnode, blkno);
449 : }
450 163558 : }
|
