LCOV - code coverage report
Current view: top level - src/bin/pg_rewind - parsexlog.c (source / functions) Hit Total Coverage
Test: PostgreSQL 14devel Lines: 108 136 79.4 %
Date: 2021-01-26 03:06:49 Functions: 5 5 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * parsexlog.c
       4             :  *    Functions for reading Write-Ahead-Log
       5             :  *
       6             :  * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group
       7             :  * Portions Copyright (c) 1994, Regents of the University of California
       8             :  *
       9             :  *-------------------------------------------------------------------------
      10             :  */
      11             : 
      12             : #include "postgres_fe.h"
      13             : 
      14             : #include <unistd.h>
      15             : 
      16             : #include "access/rmgr.h"
      17             : #include "access/xact.h"
      18             : #include "access/xlog_internal.h"
      19             : #include "access/xlogreader.h"
      20             : #include "catalog/pg_control.h"
      21             : #include "catalog/storage_xlog.h"
      22             : #include "commands/dbcommands_xlog.h"
      23             : #include "fe_utils/archive.h"
      24             : #include "filemap.h"
      25             : #include "pg_rewind.h"
      26             : 
      27             : /*
      28             :  * RmgrNames is an array of resource manager names, to make error messages
      29             :  * a bit nicer.
      30             :  */
      31             : #define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \
      32             :   name,
      33             : 
      34             : static const char *RmgrNames[RM_MAX_ID + 1] = {
      35             : #include "access/rmgrlist.h"
      36             : };
      37             : 
      38             : static void extractPageInfo(XLogReaderState *record);
      39             : 
      40             : static int  xlogreadfd = -1;
      41             : static XLogSegNo xlogreadsegno = -1;
      42             : static char xlogfpath[MAXPGPATH];
      43             : 
      44             : typedef struct XLogPageReadPrivate
      45             : {
      46             :     const char *restoreCommand;
      47             :     int         tliIndex;
      48             : } XLogPageReadPrivate;
      49             : 
      50             : static int  SimpleXLogPageRead(XLogReaderState *xlogreader,
      51             :                                XLogRecPtr targetPagePtr,
      52             :                                int reqLen, XLogRecPtr targetRecPtr, char *readBuf);
      53             : 
      54             : /*
      55             :  * Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
      56             :  * index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
      57             :  * the data blocks touched by the WAL records, and return them in a page map.
      58             :  *
      59             :  * 'endpoint' is the end of the last record to read. The record starting at
      60             :  * 'endpoint' is the first one that is not read.
      61             :  */
      62             : void
      63          24 : extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
      64             :                XLogRecPtr endpoint, const char *restoreCommand)
      65             : {
      66             :     XLogRecord *record;
      67             :     XLogReaderState *xlogreader;
      68             :     char       *errormsg;
      69             :     XLogPageReadPrivate private;
      70             : 
      71          24 :     private.tliIndex = tliIndex;
      72          24 :     private.restoreCommand = restoreCommand;
      73          24 :     xlogreader = XLogReaderAllocate(WalSegSz, datadir,
      74          24 :                                     XL_ROUTINE(.page_read = &SimpleXLogPageRead),
      75             :                                     &private);
      76          24 :     if (xlogreader == NULL)
      77           0 :         pg_fatal("out of memory");
      78             : 
      79          24 :     XLogBeginRead(xlogreader, startpoint);
      80             :     do
      81             :     {
      82      163558 :         record = XLogReadRecord(xlogreader, &errormsg);
      83             : 
      84      163558 :         if (record == NULL)
      85             :         {
      86           0 :             XLogRecPtr  errptr = xlogreader->EndRecPtr;
      87             : 
      88           0 :             if (errormsg)
      89           0 :                 pg_fatal("could not read WAL record at %X/%X: %s",
      90             :                          (uint32) (errptr >> 32), (uint32) (errptr),
      91             :                          errormsg);
      92             :             else
      93           0 :                 pg_fatal("could not read WAL record at %X/%X",
      94             :                          (uint32) (errptr >> 32), (uint32) (errptr));
      95             :         }
      96             : 
      97      163558 :         extractPageInfo(xlogreader);
      98             : 
      99      163558 :     } while (xlogreader->EndRecPtr < endpoint);
     100             : 
     101             :     /*
     102             :      * If 'endpoint' didn't point exactly at a record boundary, the caller
     103             :      * messed up.
     104             :      */
     105             :     Assert(xlogreader->EndRecPtr == endpoint);
     106             : 
     107          24 :     XLogReaderFree(xlogreader);
     108          24 :     if (xlogreadfd != -1)
     109             :     {
     110          24 :         close(xlogreadfd);
     111          24 :         xlogreadfd = -1;
     112             :     }
     113          24 : }
     114             : 
     115             : /*
     116             :  * Reads one WAL record. Returns the end position of the record, without
     117             :  * doing anything with the record itself.
     118             :  */
     119             : XLogRecPtr
     120          24 : readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex,
     121             :               const char *restoreCommand)
     122             : {
     123             :     XLogRecord *record;
     124             :     XLogReaderState *xlogreader;
     125             :     char       *errormsg;
     126             :     XLogPageReadPrivate private;
     127             :     XLogRecPtr  endptr;
     128             : 
     129          24 :     private.tliIndex = tliIndex;
     130          24 :     private.restoreCommand = restoreCommand;
     131          24 :     xlogreader = XLogReaderAllocate(WalSegSz, datadir,
     132          24 :                                     XL_ROUTINE(.page_read = &SimpleXLogPageRead),
     133             :                                     &private);
     134          24 :     if (xlogreader == NULL)
     135           0 :         pg_fatal("out of memory");
     136             : 
     137          24 :     XLogBeginRead(xlogreader, ptr);
     138          24 :     record = XLogReadRecord(xlogreader, &errormsg);
     139          24 :     if (record == NULL)
     140             :     {
     141           0 :         if (errormsg)
     142           0 :             pg_fatal("could not read WAL record at %X/%X: %s",
     143             :                      (uint32) (ptr >> 32), (uint32) (ptr), errormsg);
     144             :         else
     145           0 :             pg_fatal("could not read WAL record at %X/%X",
     146             :                      (uint32) (ptr >> 32), (uint32) (ptr));
     147             :     }
     148          24 :     endptr = xlogreader->EndRecPtr;
     149             : 
     150          24 :     XLogReaderFree(xlogreader);
     151          24 :     if (xlogreadfd != -1)
     152             :     {
     153          24 :         close(xlogreadfd);
     154          24 :         xlogreadfd = -1;
     155             :     }
     156             : 
     157          24 :     return endptr;
     158             : }
     159             : 
     160             : /*
     161             :  * Find the previous checkpoint preceding given WAL location.
     162             :  */
     163             : void
     164          24 : findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
     165             :                    XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
     166             :                    XLogRecPtr *lastchkptredo, const char *restoreCommand)
     167             : {
     168             :     /* Walk backwards, starting from the given record */
     169             :     XLogRecord *record;
     170             :     XLogRecPtr  searchptr;
     171             :     XLogReaderState *xlogreader;
     172             :     char       *errormsg;
     173             :     XLogPageReadPrivate private;
     174             : 
     175             :     /*
     176             :      * The given fork pointer points to the end of the last common record,
     177             :      * which is not necessarily the beginning of the next record, if the
     178             :      * previous record happens to end at a page boundary. Skip over the page
     179             :      * header in that case to find the next record.
     180             :      */
     181          24 :     if (forkptr % XLOG_BLCKSZ == 0)
     182             :     {
     183           4 :         if (XLogSegmentOffset(forkptr, WalSegSz) == 0)
     184           4 :             forkptr += SizeOfXLogLongPHD;
     185             :         else
     186           0 :             forkptr += SizeOfXLogShortPHD;
     187             :     }
     188             : 
     189          24 :     private.tliIndex = tliIndex;
     190          24 :     private.restoreCommand = restoreCommand;
     191          24 :     xlogreader = XLogReaderAllocate(WalSegSz, datadir,
     192          24 :                                     XL_ROUTINE(.page_read = &SimpleXLogPageRead),
     193             :                                     &private);
     194          24 :     if (xlogreader == NULL)
     195           0 :         pg_fatal("out of memory");
     196             : 
     197          24 :     searchptr = forkptr;
     198             :     for (;;)
     199         200 :     {
     200             :         uint8       info;
     201             : 
     202         224 :         XLogBeginRead(xlogreader, searchptr);
     203         224 :         record = XLogReadRecord(xlogreader, &errormsg);
     204             : 
     205         224 :         if (record == NULL)
     206             :         {
     207           0 :             if (errormsg)
     208           0 :                 pg_fatal("could not find previous WAL record at %X/%X: %s",
     209             :                          (uint32) (searchptr >> 32), (uint32) (searchptr),
     210             :                          errormsg);
     211             :             else
     212           0 :                 pg_fatal("could not find previous WAL record at %X/%X",
     213             :                          (uint32) (searchptr >> 32), (uint32) (searchptr));
     214             :         }
     215             : 
     216             :         /*
     217             :          * Check if it is a checkpoint record. This checkpoint record needs to
     218             :          * be the latest checkpoint before WAL forked and not the checkpoint
     219             :          * where the primary has been stopped to be rewound.
     220             :          */
     221         224 :         info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
     222         224 :         if (searchptr < forkptr &&
     223         200 :             XLogRecGetRmid(xlogreader) == RM_XLOG_ID &&
     224          40 :             (info == XLOG_CHECKPOINT_SHUTDOWN ||
     225             :              info == XLOG_CHECKPOINT_ONLINE))
     226             :         {
     227             :             CheckPoint  checkPoint;
     228             : 
     229          24 :             memcpy(&checkPoint, XLogRecGetData(xlogreader), sizeof(CheckPoint));
     230          24 :             *lastchkptrec = searchptr;
     231          24 :             *lastchkpttli = checkPoint.ThisTimeLineID;
     232          24 :             *lastchkptredo = checkPoint.redo;
     233          24 :             break;
     234             :         }
     235             : 
     236             :         /* Walk backwards to previous record. */
     237         200 :         searchptr = record->xl_prev;
     238             :     }
     239             : 
     240          24 :     XLogReaderFree(xlogreader);
     241          24 :     if (xlogreadfd != -1)
     242             :     {
     243          24 :         close(xlogreadfd);
     244          24 :         xlogreadfd = -1;
     245             :     }
     246          24 : }
     247             : 
     248             : /* XLogReader callback function, to read a WAL page */
     249             : static int
     250        2666 : SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
     251             :                    int reqLen, XLogRecPtr targetRecPtr, char *readBuf)
     252             : {
     253        2666 :     XLogPageReadPrivate *private = (XLogPageReadPrivate *) xlogreader->private_data;
     254             :     uint32      targetPageOff;
     255             :     XLogRecPtr  targetSegEnd;
     256             :     XLogSegNo   targetSegNo;
     257             :     int         r;
     258             : 
     259        2666 :     XLByteToSeg(targetPagePtr, targetSegNo, WalSegSz);
     260        2666 :     XLogSegNoOffsetToRecPtr(targetSegNo + 1, 0, WalSegSz, targetSegEnd);
     261        2666 :     targetPageOff = XLogSegmentOffset(targetPagePtr, WalSegSz);
     262             : 
     263             :     /*
     264             :      * See if we need to switch to a new segment because the requested record
     265             :      * is not in the currently open one.
     266             :      */
     267        2666 :     if (xlogreadfd >= 0 &&
     268        2594 :         !XLByteInSeg(targetPagePtr, xlogreadsegno, WalSegSz))
     269             :     {
     270           8 :         close(xlogreadfd);
     271           8 :         xlogreadfd = -1;
     272             :     }
     273             : 
     274        2666 :     XLByteToSeg(targetPagePtr, xlogreadsegno, WalSegSz);
     275             : 
     276        2666 :     if (xlogreadfd < 0)
     277             :     {
     278             :         char        xlogfname[MAXFNAMELEN];
     279             : 
     280             :         /*
     281             :          * Since incomplete segments are copied into next timelines, switch to
     282             :          * the timeline holding the required segment. Assuming this scan can
     283             :          * be done both forward and backward, consider also switching timeline
     284             :          * accordingly.
     285             :          */
     286          84 :         while (private->tliIndex < targetNentries - 1 &&
     287           4 :                targetHistory[private->tliIndex].end < targetSegEnd)
     288           4 :             private->tliIndex++;
     289          80 :         while (private->tliIndex > 0 &&
     290          12 :                targetHistory[private->tliIndex].begin >= targetSegEnd)
     291           0 :             private->tliIndex--;
     292             : 
     293          80 :         XLogFileName(xlogfname, targetHistory[private->tliIndex].tli,
     294             :                      xlogreadsegno, WalSegSz);
     295             : 
     296          80 :         snprintf(xlogfpath, MAXPGPATH, "%s/" XLOGDIR "/%s",
     297          80 :                  xlogreader->segcxt.ws_dir, xlogfname);
     298             : 
     299          80 :         xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
     300             : 
     301          80 :         if (xlogreadfd < 0)
     302             :         {
     303             :             /*
     304             :              * If we have no restore_command to execute, then exit.
     305             :              */
     306           2 :             if (private->restoreCommand == NULL)
     307             :             {
     308           0 :                 pg_log_error("could not open file \"%s\": %m", xlogfpath);
     309           0 :                 return -1;
     310             :             }
     311             : 
     312             :             /*
     313             :              * Since we have restore_command, then try to retrieve missing WAL
     314             :              * file from the archive.
     315             :              */
     316           2 :             xlogreadfd = RestoreArchivedFile(xlogreader->segcxt.ws_dir,
     317             :                                              xlogfname,
     318             :                                              WalSegSz,
     319             :                                              private->restoreCommand);
     320             : 
     321           2 :             if (xlogreadfd < 0)
     322           0 :                 return -1;
     323             :             else
     324           2 :                 pg_log_debug("using file \"%s\" restored from archive",
     325             :                              xlogfpath);
     326             :         }
     327             :     }
     328             : 
     329             :     /*
     330             :      * At this point, we have the right segment open.
     331             :      */
     332             :     Assert(xlogreadfd != -1);
     333             : 
     334             :     /* Read the requested page */
     335        2666 :     if (lseek(xlogreadfd, (off_t) targetPageOff, SEEK_SET) < 0)
     336             :     {
     337           0 :         pg_log_error("could not seek in file \"%s\": %m", xlogfpath);
     338           0 :         return -1;
     339             :     }
     340             : 
     341             : 
     342        2666 :     r = read(xlogreadfd, readBuf, XLOG_BLCKSZ);
     343        2666 :     if (r != XLOG_BLCKSZ)
     344             :     {
     345           0 :         if (r < 0)
     346           0 :             pg_log_error("could not read file \"%s\": %m", xlogfpath);
     347             :         else
     348           0 :             pg_log_error("could not read file \"%s\": read %d of %zu",
     349             :                          xlogfpath, r, (Size) XLOG_BLCKSZ);
     350             : 
     351           0 :         return -1;
     352             :     }
     353             : 
     354             :     Assert(targetSegNo == xlogreadsegno);
     355             : 
     356        2666 :     xlogreader->seg.ws_tli = targetHistory[private->tliIndex].tli;
     357        2666 :     return XLOG_BLCKSZ;
     358             : }
     359             : 
     360             : /*
     361             :  * Extract information on which blocks the current record modifies.
     362             :  */
     363             : static void
     364      163558 : extractPageInfo(XLogReaderState *record)
     365             : {
     366             :     int         block_id;
     367      163558 :     RmgrId      rmid = XLogRecGetRmid(record);
     368      163558 :     uint8       info = XLogRecGetInfo(record);
     369      163558 :     uint8       rminfo = info & ~XLR_INFO_MASK;
     370             : 
     371             :     /* Is this a special record type that I recognize? */
     372             : 
     373      163558 :     if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_CREATE)
     374             :     {
     375             :         /*
     376             :          * New databases can be safely ignored. It won't be present in the
     377             :          * source system, so it will be deleted. There's one corner-case,
     378             :          * though: if a new, different, database is also created in the source
     379             :          * system, we'll see that the files already exist and not copy them.
     380             :          * That's OK, though; WAL replay of creating the new database, from
     381             :          * the source systems's WAL, will re-copy the new database,
     382             :          * overwriting the database created in the target system.
     383             :          */
     384             :     }
     385      163554 :     else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP)
     386             :     {
     387             :         /*
     388             :          * An existing database was dropped. We'll see that the files don't
     389             :          * exist in the target data dir, and copy them in toto from the source
     390             :          * system. No need to do anything special here.
     391             :          */
     392             :     }
     393      163554 :     else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE)
     394             :     {
     395             :         /*
     396             :          * We can safely ignore these. The file will be removed from the
     397             :          * target, if it doesn't exist in source system. If a file with same
     398             :          * name is created in source system, too, there will be WAL records
     399             :          * for all the blocks in it.
     400             :          */
     401             :     }
     402      163546 :     else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE)
     403             :     {
     404             :         /*
     405             :          * We can safely ignore these. When we compare the sizes later on,
     406             :          * we'll notice that they differ, and copy the missing tail from
     407             :          * source system.
     408             :          */
     409             :     }
     410      163610 :     else if (rmid == RM_XACT_ID &&
     411          72 :              ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT ||
     412           0 :               (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT_PREPARED ||
     413           0 :               (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT ||
     414           0 :               (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT_PREPARED))
     415             :     {
     416             :         /*
     417             :          * These records can include "dropped rels". We can safely ignore
     418             :          * them, we will see that they are missing and copy them from the
     419             :          * source.
     420             :          */
     421             :     }
     422      163466 :     else if (info & XLR_SPECIAL_REL_UPDATE)
     423             :     {
     424             :         /*
     425             :          * This record type modifies a relation file in some special way, but
     426             :          * we don't recognize the type. That's bad - we don't know how to
     427             :          * track that change.
     428             :          */
     429           0 :         pg_fatal("WAL record modifies a relation, but record type is not recognized: "
     430             :                  "lsn: %X/%X, rmgr: %s, info: %02X",
     431             :                  (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr),
     432             :                  RmgrNames[rmid], info);
     433             :     }
     434             : 
     435      327566 :     for (block_id = 0; block_id <= record->max_block_id; block_id++)
     436             :     {
     437             :         RelFileNode rnode;
     438             :         ForkNumber  forknum;
     439             :         BlockNumber blkno;
     440             : 
     441      164008 :         if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno))
     442         720 :             continue;
     443             : 
     444             :         /* We only care about the main fork; others are copied in toto */
     445      164008 :         if (forknum != MAIN_FORKNUM)
     446         720 :             continue;
     447             : 
     448      163288 :         process_target_wal_block_change(forknum, rnode, blkno);
     449             :     }
     450      163558 : }

Generated by: LCOV version 1.13