Branch data Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * walsummarizer.c
4 : : *
5 : : * Background process to perform WAL summarization, if it is enabled.
6 : : * It continuously scans the write-ahead log and periodically emits a
7 : : * summary file which indicates which blocks in which relation forks
8 : : * were modified by WAL records in the LSN range covered by the summary
9 : : * file. See walsummary.c and blkreftable.c for more details on the
10 : : * naming and contents of WAL summary files.
11 : : *
12 : : * If configured to do, this background process will also remove WAL
13 : : * summary files when the file timestamp is older than a configurable
14 : : * threshold (but only if the WAL has been removed first).
15 : : *
16 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
17 : : *
18 : : * IDENTIFICATION
19 : : * src/backend/postmaster/walsummarizer.c
20 : : *
21 : : *-------------------------------------------------------------------------
22 : : */
23 : : #include "postgres.h"
24 : :
25 : : #include "access/timeline.h"
26 : : #include "access/visibilitymap.h"
27 : : #include "access/xlog.h"
28 : : #include "access/xlog_internal.h"
29 : : #include "access/xlogrecovery.h"
30 : : #include "access/xlogutils.h"
31 : : #include "backup/walsummary.h"
32 : : #include "catalog/storage_xlog.h"
33 : : #include "commands/dbcommands_xlog.h"
34 : : #include "common/blkreftable.h"
35 : : #include "libpq/pqsignal.h"
36 : : #include "miscadmin.h"
37 : : #include "pgstat.h"
38 : : #include "postmaster/auxprocess.h"
39 : : #include "postmaster/interrupt.h"
40 : : #include "postmaster/walsummarizer.h"
41 : : #include "replication/walreceiver.h"
42 : : #include "storage/aio_subsys.h"
43 : : #include "storage/fd.h"
44 : : #include "storage/ipc.h"
45 : : #include "storage/latch.h"
46 : : #include "storage/lwlock.h"
47 : : #include "storage/proc.h"
48 : : #include "storage/procsignal.h"
49 : : #include "storage/shmem.h"
50 : : #include "storage/subsystems.h"
51 : : #include "utils/guc.h"
52 : : #include "utils/memutils.h"
53 : : #include "utils/wait_event.h"
54 : :
55 : : /*
56 : : * Data in shared memory related to WAL summarization.
57 : : */
58 : : typedef struct
59 : : {
60 : : /*
61 : : * These fields are protected by WALSummarizerLock.
62 : : *
63 : : * Until we've discovered what summary files already exist on disk and
64 : : * stored that information in shared memory, initialized is false and the
65 : : * other fields here contain no meaningful information. After that has
66 : : * been done, initialized is true.
67 : : *
68 : : * summarized_tli and summarized_lsn indicate the last LSN and TLI at
69 : : * which the next summary file will start. Normally, these are the LSN and
70 : : * TLI at which the last file ended; in such case, lsn_is_exact is true.
71 : : * If, however, the LSN is just an approximation, then lsn_is_exact is
72 : : * false. This can happen if, for example, there are no existing WAL
73 : : * summary files at startup. In that case, we have to derive the position
74 : : * at which to start summarizing from the WAL files that exist on disk,
75 : : * and so the LSN might point to the start of the next file even though
76 : : * that might happen to be in the middle of a WAL record.
77 : : *
78 : : * summarizer_pgprocno is the proc number of the summarizer process, if
79 : : * one is running, or else INVALID_PROC_NUMBER.
80 : : *
81 : : * pending_lsn is used by the summarizer to advertise the ending LSN of a
82 : : * record it has recently read. It shouldn't ever be less than
83 : : * summarized_lsn, but might be greater, because the summarizer buffers
84 : : * data for a range of LSNs in memory before writing out a new file.
85 : : */
86 : : bool initialized;
87 : : TimeLineID summarized_tli;
88 : : XLogRecPtr summarized_lsn;
89 : : bool lsn_is_exact;
90 : : ProcNumber summarizer_pgprocno;
91 : : XLogRecPtr pending_lsn;
92 : :
93 : : /*
94 : : * This field handles its own synchronization.
95 : : */
96 : : ConditionVariable summary_file_cv;
97 : : } WalSummarizerData;
98 : :
99 : : /*
100 : : * Private data for our xlogreader's page read callback.
101 : : */
102 : : typedef struct
103 : : {
104 : : TimeLineID tli;
105 : : bool historic;
106 : : XLogRecPtr read_upto;
107 : : bool end_of_wal;
108 : : int num_descendant_tlis;
109 : : TimeLineID *descendant_tlis;
110 : : } SummarizerReadLocalXLogPrivate;
111 : :
112 : : /* Pointer to shared memory state. */
113 : : static WalSummarizerData *WalSummarizerCtl;
114 : :
115 : : static void WalSummarizerShmemRequest(void *arg);
116 : : static void WalSummarizerShmemInit(void *arg);
117 : :
118 : : const ShmemCallbacks WalSummarizerShmemCallbacks = {
119 : : .request_fn = WalSummarizerShmemRequest,
120 : : .init_fn = WalSummarizerShmemInit,
121 : : };
122 : :
123 : : /*
124 : : * When we reach end of WAL and need to read more, we sleep for a number of
125 : : * milliseconds that is an integer multiple of MS_PER_SLEEP_QUANTUM. This is
126 : : * the multiplier. It should vary between 1 and MAX_SLEEP_QUANTA, depending
127 : : * on system activity. See summarizer_wait_for_wal() for how we adjust this.
128 : : */
129 : : static long sleep_quanta = 1;
130 : :
131 : : /*
132 : : * The sleep time will always be a multiple of 200ms and will not exceed
133 : : * thirty seconds (150 * 200 = 30 * 1000). Note that the timeout here needs
134 : : * to be substantially less than the maximum amount of time for which an
135 : : * incremental backup will wait for this process to catch up. Otherwise, an
136 : : * incremental backup might time out on an idle system just because we sleep
137 : : * for too long.
138 : : */
139 : : #define MAX_SLEEP_QUANTA 150
140 : : #define MS_PER_SLEEP_QUANTUM 200
141 : :
142 : : /*
143 : : * This is a count of the number of pages of WAL that we've read since the
144 : : * last time we waited for more WAL to appear.
145 : : */
146 : : static long pages_read_since_last_sleep = 0;
147 : :
148 : : /*
149 : : * Most recent RedoRecPtr value observed by MaybeRemoveOldWalSummaries.
150 : : */
151 : : static XLogRecPtr redo_pointer_at_last_summary_removal = InvalidXLogRecPtr;
152 : :
153 : : /*
154 : : * GUC parameters
155 : : */
156 : : bool summarize_wal = false;
157 : : int wal_summary_keep_time = 10 * HOURS_PER_DAY * MINS_PER_HOUR;
158 : :
159 : : static void WalSummarizerShutdown(int code, Datum arg);
160 : : static XLogRecPtr GetLatestLSN(TimeLineID *tli);
161 : : static XLogRecPtr WalSummarizerSwitchPoint(TimeLineID current_tli, List *tles,
162 : : int *num_descendant_tlis,
163 : : TimeLineID **descendant_tlis);
164 : : static void ProcessWalSummarizerInterrupts(void);
165 : : static XLogRecPtr SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn,
166 : : bool exact, XLogRecPtr switch_lsn,
167 : : XLogRecPtr maximum_lsn,
168 : : int num_descendant_tlis, TimeLineID *descendant_tlis);
169 : : static void SummarizeDbaseRecord(XLogReaderState *xlogreader,
170 : : BlockRefTable *brtab);
171 : : static void SummarizeSmgrRecord(XLogReaderState *xlogreader,
172 : : BlockRefTable *brtab);
173 : : static void SummarizeXactRecord(XLogReaderState *xlogreader,
174 : : BlockRefTable *brtab);
175 : : static bool SummarizeXlogRecord(XLogReaderState *xlogreader,
176 : : bool *new_fast_forward);
177 : : static void summarizer_wal_segment_open(XLogReaderState *state,
178 : : XLogSegNo nextSegNo,
179 : : TimeLineID *tli_p);
180 : : static int summarizer_read_local_xlog_page(XLogReaderState *state,
181 : : XLogRecPtr targetPagePtr,
182 : : int reqLen,
183 : : XLogRecPtr targetRecPtr,
184 : : char *cur_page);
185 : : static void summarizer_wait_for_wal(void);
186 : : static void MaybeRemoveOldWalSummaries(void);
187 : :
188 : : /*
189 : : * Register shared memory space needed by this module.
190 : : */
191 : : static void
192 : 1355 : WalSummarizerShmemRequest(void *arg)
193 : : {
194 : 1355 : ShmemRequestStruct(.name = "Wal Summarizer Ctl",
195 : : .size = sizeof(WalSummarizerData),
196 : : .ptr = (void **) &WalSummarizerCtl,
197 : : );
198 : 1355 : }
199 : :
200 : : /*
201 : : * Initialize shared memory for this module.
202 : : */
203 : : static void
204 : 1351 : WalSummarizerShmemInit(void *arg)
205 : : {
206 : : /*
207 : : * We're just filling in dummy values here -- the real initialization will
208 : : * happen when GetOldestUnsummarizedLSN() is called for the first time.
209 : : */
210 : 1351 : WalSummarizerCtl->initialized = false;
211 : 1351 : WalSummarizerCtl->summarized_tli = 0;
212 : 1351 : WalSummarizerCtl->summarized_lsn = InvalidXLogRecPtr;
213 : 1351 : WalSummarizerCtl->lsn_is_exact = false;
214 : 1351 : WalSummarizerCtl->summarizer_pgprocno = INVALID_PROC_NUMBER;
215 : 1351 : WalSummarizerCtl->pending_lsn = InvalidXLogRecPtr;
216 : 1351 : ConditionVariableInit(&WalSummarizerCtl->summary_file_cv);
217 : 1351 : }
218 : :
219 : : /*
220 : : * Entry point for walsummarizer process.
221 : : */
222 : : void
223 : 10 : WalSummarizerMain(const void *startup_data, size_t startup_data_len)
224 : : {
225 : : sigjmp_buf local_sigjmp_buf;
226 : : MemoryContext context;
227 : :
228 : : /*
229 : : * Within this function, 'current_lsn' and 'current_tli' refer to the
230 : : * point from which the next WAL summary file should start. 'exact' is
231 : : * true if 'current_lsn' is known to be the start of a WAL record or WAL
232 : : * segment, and false if it might be in the middle of a record someplace.
233 : : *
234 : : * 'switch_lsn', is the LSN at which we need to switch to a new timeline.
235 : : * If not set, we either haven't figured out the answer yet or we're
236 : : * already on the latest timeline. 'descendant_tlis' stores an array of
237 : : * future timeline IDs to which we know we'll need to switch, and
238 : : * 'num_descendant_tlis' is the length of that array. The first element of
239 : : * the array is the first timeline to which we will need to switch.
240 : : */
241 : : XLogRecPtr current_lsn;
242 : : TimeLineID current_tli;
243 : : bool exact;
244 : 10 : XLogRecPtr switch_lsn = InvalidXLogRecPtr;
245 : 10 : int num_descendant_tlis = 0;
246 : 10 : TimeLineID *descendant_tlis = NULL;
247 : :
248 : : Assert(startup_data_len == 0);
249 : :
250 : 10 : AuxiliaryProcessMainCommon();
251 : :
252 [ - + ]: 10 : ereport(DEBUG1,
253 : : (errmsg_internal("WAL summarizer started")));
254 : :
255 : : /*
256 : : * Properly accept or ignore signals the postmaster might send us
257 : : */
258 : 10 : pqsignal(SIGHUP, SignalHandlerForConfigReload);
259 : 10 : pqsignal(SIGINT, PG_SIG_IGN); /* no query to cancel */
260 : 10 : pqsignal(SIGTERM, SignalHandlerForShutdownRequest);
261 : : /* SIGQUIT handler was already set up by InitPostmasterChild */
262 : 10 : pqsignal(SIGALRM, PG_SIG_IGN);
263 : 10 : pqsignal(SIGPIPE, PG_SIG_IGN);
264 : 10 : pqsignal(SIGUSR1, procsignal_sigusr1_handler);
265 : 10 : pqsignal(SIGUSR2, PG_SIG_IGN); /* not used */
266 : :
267 : : /* Advertise ourselves. */
268 : 10 : on_shmem_exit(WalSummarizerShutdown, (Datum) 0);
269 : 10 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
270 : 10 : WalSummarizerCtl->summarizer_pgprocno = MyProcNumber;
271 : 10 : LWLockRelease(WALSummarizerLock);
272 : :
273 : : /* Create and switch to a memory context that we can reset on error. */
274 : 10 : context = AllocSetContextCreate(TopMemoryContext,
275 : : "Wal Summarizer",
276 : : ALLOCSET_DEFAULT_SIZES);
277 : 10 : MemoryContextSwitchTo(context);
278 : :
279 : : /*
280 : : * If an exception is encountered, processing resumes here.
281 : : */
282 [ - + ]: 10 : if (sigsetjmp(local_sigjmp_buf, 1) != 0)
283 : : {
284 : : /* Since not using PG_TRY, must reset error stack by hand */
285 : 0 : error_context_stack = NULL;
286 : :
287 : : /* Prevent interrupts while cleaning up */
288 : 0 : HOLD_INTERRUPTS();
289 : :
290 : : /* Report the error to the server log */
291 : 0 : EmitErrorReport();
292 : :
293 : : /* Release resources we might have acquired. */
294 : 0 : LWLockReleaseAll();
295 : 0 : ConditionVariableCancelSleep();
296 : 0 : pgstat_report_wait_end();
297 : 0 : pgaio_error_cleanup();
298 : 0 : ReleaseAuxProcessResources(false);
299 : 0 : AtEOXact_Files(false);
300 : 0 : AtEOXact_HashTables(false);
301 : :
302 : : /*
303 : : * Now return to normal top-level context and clear ErrorContext for
304 : : * next time.
305 : : */
306 : 0 : MemoryContextSwitchTo(context);
307 : 0 : FlushErrorState();
308 : :
309 : : /* Flush any leaked data in the top-level context */
310 : 0 : MemoryContextReset(context);
311 : :
312 : : /* Now we can allow interrupts again */
313 : 0 : RESUME_INTERRUPTS();
314 : :
315 : : /*
316 : : * Sleep for 10 seconds before attempting to resume operations in
317 : : * order to avoid excessive logging.
318 : : *
319 : : * Many of the likely error conditions are things that will repeat
320 : : * every time. For example, if the WAL can't be read or the summary
321 : : * can't be written, only administrator action will cure the problem.
322 : : * So a really fast retry time doesn't seem to be especially
323 : : * beneficial, and it will clutter the logs.
324 : : */
325 : 0 : (void) WaitLatch(NULL,
326 : : WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
327 : : 10000,
328 : : WAIT_EVENT_WAL_SUMMARIZER_ERROR);
329 : : }
330 : :
331 : : /* We can now handle ereport(ERROR) */
332 : 10 : PG_exception_stack = &local_sigjmp_buf;
333 : :
334 : : /*
335 : : * Unblock signals (they were blocked when the postmaster forked us)
336 : : */
337 : 10 : sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
338 : :
339 : : /*
340 : : * Fetch information about previous progress from shared memory, and ask
341 : : * GetOldestUnsummarizedLSN to reset pending_lsn to summarized_lsn. We
342 : : * might be recovering from an error, and if so, pending_lsn might have
343 : : * advanced past summarized_lsn, but any WAL we read previously has been
344 : : * lost and will need to be reread.
345 : : *
346 : : * If we discover that WAL summarization is not enabled, just exit.
347 : : */
348 : 10 : current_lsn = GetOldestUnsummarizedLSN(¤t_tli, &exact);
349 [ + - ]: 10 : if (!XLogRecPtrIsValid(current_lsn))
350 : 0 : proc_exit(0);
351 : :
352 : : /*
353 : : * Loop forever
354 : : */
355 : : for (;;)
356 : 52 : {
357 : : XLogRecPtr latest_lsn;
358 : : TimeLineID latest_tli;
359 : : XLogRecPtr maximum_lsn;
360 : : XLogRecPtr end_of_summary_lsn;
361 : :
362 : : /* Flush any leaked data in the top-level context */
363 : 62 : MemoryContextReset(context);
364 : :
365 : : /* Process any signals received recently. */
366 : 62 : ProcessWalSummarizerInterrupts();
367 : :
368 : : /* If it's time to remove any old WAL summaries, do that now. */
369 : 62 : MaybeRemoveOldWalSummaries();
370 : :
371 : : /* Find the LSN and TLI up to which we can safely summarize. */
372 : 62 : latest_lsn = GetLatestLSN(&latest_tli);
373 : :
374 : : /*
375 : : * If we're summarizing a historic timeline and we haven't yet
376 : : * computed the point at which to switch to the next timeline, do that
377 : : * now.
378 : : *
379 : : * Note that if this is a standby, what was previously the current
380 : : * timeline could become historic at any time.
381 : : *
382 : : * We could try to make this more efficient by caching the results of
383 : : * readTimeLineHistory when latest_tli has not changed, but since we
384 : : * only have to do this once per timeline switch, we probably wouldn't
385 : : * save any significant amount of work in practice.
386 : : */
387 [ + + + - ]: 62 : if (current_tli != latest_tli && !XLogRecPtrIsValid(switch_lsn))
388 : : {
389 : 1 : List *tles = readTimeLineHistory(latest_tli);
390 : : int new_num_descendant_tlis;
391 : : TimeLineID *new_descendant_tlis;
392 : :
393 : : /*
394 : : * Make sure that the array of descendant TLIs get stored into
395 : : * TopMemoryContext.
396 : : */
397 : 1 : MemoryContextSwitchTo(TopMemoryContext);
398 : 1 : switch_lsn = WalSummarizerSwitchPoint(current_tli, tles,
399 : : &new_num_descendant_tlis,
400 : : &new_descendant_tlis);
401 : 1 : MemoryContextSwitchTo(context);
402 : :
403 : : /*
404 : : * Free any old array of descendant TLIs and install the new
405 : : * values.
406 : : */
407 [ - + ]: 1 : if (descendant_tlis != NULL)
408 : 0 : pfree(descendant_tlis);
409 : 1 : num_descendant_tlis = new_num_descendant_tlis;
410 : 1 : descendant_tlis = new_descendant_tlis;
411 : :
412 : : /* Debug message. */
413 [ - + ]: 1 : ereport(DEBUG1,
414 : : errmsg_internal("switch point from TLI %u to TLI %u is at %X/%08X",
415 : : current_tli, descendant_tlis[0],
416 : : LSN_FORMAT_ARGS(switch_lsn)));
417 : : }
418 : :
419 : : /*
420 : : * If we've reached the switch LSN, we can't summarize anything else
421 : : * on this timeline. Switch to the next timeline and go around again,
422 : : * backing up to the exact switch point if we passed it.
423 : : */
424 [ + + + - ]: 62 : if (XLogRecPtrIsValid(switch_lsn) && current_lsn >= switch_lsn)
425 : : {
426 : : /* Restart summarization from switch point. */
427 : : Assert(num_descendant_tlis > 0);
428 : 1 : current_tli = descendant_tlis[0];
429 : 1 : current_lsn = switch_lsn;
430 : :
431 : : /* Switch point, if any, and future TLIs, not yet known. */
432 : 1 : switch_lsn = InvalidXLogRecPtr;
433 : 1 : num_descendant_tlis = 0;
434 : 1 : pfree(descendant_tlis);
435 : 1 : descendant_tlis = NULL;
436 : :
437 : : /* Update (really, rewind, if needed) state in shared memory. */
438 : 1 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
439 : 1 : WalSummarizerCtl->summarized_lsn = current_lsn;
440 : 1 : WalSummarizerCtl->summarized_tli = current_tli;
441 : 1 : WalSummarizerCtl->lsn_is_exact = true;
442 : 1 : WalSummarizerCtl->pending_lsn = current_lsn;
443 : 1 : LWLockRelease(WALSummarizerLock);
444 : :
445 : 1 : continue;
446 : : }
447 : :
448 : : /* Summarize WAL. */
449 [ - + ]: 61 : maximum_lsn = XLogRecPtrIsValid(switch_lsn) ? switch_lsn : latest_lsn;
450 : 61 : end_of_summary_lsn = SummarizeWAL(current_tli,
451 : : current_lsn, exact,
452 : : switch_lsn, maximum_lsn,
453 : : num_descendant_tlis, descendant_tlis);
454 : : Assert(XLogRecPtrIsValid(end_of_summary_lsn));
455 : : Assert(end_of_summary_lsn >= current_lsn);
456 : :
457 : : /*
458 : : * Update state for next loop iteration.
459 : : *
460 : : * Next summary file should start from exactly where this one ended.
461 : : */
462 : 51 : current_lsn = end_of_summary_lsn;
463 : 51 : exact = true;
464 : :
465 : : /* Update state in shared memory. */
466 : 51 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
467 : 51 : WalSummarizerCtl->summarized_lsn = end_of_summary_lsn;
468 : 51 : WalSummarizerCtl->summarized_tli = current_tli;
469 : 51 : WalSummarizerCtl->lsn_is_exact = true;
470 : 51 : WalSummarizerCtl->pending_lsn = end_of_summary_lsn;
471 : 51 : LWLockRelease(WALSummarizerLock);
472 : :
473 : : /* Wake up anyone waiting for more summary files to be written. */
474 : 51 : ConditionVariableBroadcast(&WalSummarizerCtl->summary_file_cv);
475 : : }
476 : : }
477 : :
478 : : /*
479 : : * Get information about the state of the WAL summarizer.
480 : : */
481 : : void
482 : 0 : GetWalSummarizerState(TimeLineID *summarized_tli, XLogRecPtr *summarized_lsn,
483 : : XLogRecPtr *pending_lsn, int *summarizer_pid)
484 : : {
485 : 0 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
486 [ # # ]: 0 : if (!WalSummarizerCtl->initialized)
487 : : {
488 : : /*
489 : : * If initialized is false, the rest of the structure contents are
490 : : * undefined.
491 : : */
492 : 0 : *summarized_tli = 0;
493 : 0 : *summarized_lsn = InvalidXLogRecPtr;
494 : 0 : *pending_lsn = InvalidXLogRecPtr;
495 : 0 : *summarizer_pid = -1;
496 : : }
497 : : else
498 : : {
499 : 0 : int summarizer_pgprocno = WalSummarizerCtl->summarizer_pgprocno;
500 : :
501 : 0 : *summarized_tli = WalSummarizerCtl->summarized_tli;
502 : 0 : *summarized_lsn = WalSummarizerCtl->summarized_lsn;
503 [ # # ]: 0 : if (summarizer_pgprocno == INVALID_PROC_NUMBER)
504 : : {
505 : : /*
506 : : * If the summarizer has exited, the fact that it had processed
507 : : * beyond summarized_lsn is irrelevant now.
508 : : */
509 : 0 : *pending_lsn = WalSummarizerCtl->summarized_lsn;
510 : 0 : *summarizer_pid = -1;
511 : : }
512 : : else
513 : : {
514 : 0 : *pending_lsn = WalSummarizerCtl->pending_lsn;
515 : :
516 : : /*
517 : : * We're not fussed about inexact answers here, since they could
518 : : * become stale instantly, so we don't bother taking the lock, but
519 : : * make sure that invalid PID values are normalized to -1.
520 : : */
521 : 0 : *summarizer_pid = GetPGProcByNumber(summarizer_pgprocno)->pid;
522 [ # # ]: 0 : if (*summarizer_pid <= 0)
523 : 0 : *summarizer_pid = -1;
524 : : }
525 : : }
526 : 0 : LWLockRelease(WALSummarizerLock);
527 : 0 : }
528 : :
529 : : /*
530 : : * Get the oldest LSN in this server's timeline history that has not yet been
531 : : * summarized, and update shared memory state as appropriate.
532 : : *
533 : : * If *tli != NULL, it will be set to the TLI for the LSN that is returned.
534 : : *
535 : : * If *lsn_is_exact != NULL, it will be set to true if the returned LSN is
536 : : * necessarily the start of a WAL record and false if it's just the beginning
537 : : * of a WAL segment.
538 : : */
539 : : XLogRecPtr
540 : 2664 : GetOldestUnsummarizedLSN(TimeLineID *tli, bool *lsn_is_exact)
541 : : {
542 : : TimeLineID latest_tli;
543 : : int n;
544 : : List *tles;
545 : 2664 : XLogRecPtr unsummarized_lsn = InvalidXLogRecPtr;
546 : 2664 : TimeLineID unsummarized_tli = 0;
547 : 2664 : bool should_make_exact = false;
548 : : List *existing_summaries;
549 : : ListCell *lc;
550 : 2664 : bool am_wal_summarizer = AmWalSummarizerProcess();
551 : :
552 : : /* If not summarizing WAL, do nothing. */
553 [ + + ]: 2664 : if (!summarize_wal)
554 : 2630 : return InvalidXLogRecPtr;
555 : :
556 : : /*
557 : : * If we are not the WAL summarizer process, then we normally just want to
558 : : * read the values from shared memory. However, as an exception, if shared
559 : : * memory hasn't been initialized yet, then we need to do that so that we
560 : : * can read legal values and not remove any WAL too early.
561 : : */
562 [ + + ]: 34 : if (!am_wal_summarizer)
563 : : {
564 : 24 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
565 : :
566 [ + + ]: 24 : if (WalSummarizerCtl->initialized)
567 : : {
568 : 20 : unsummarized_lsn = WalSummarizerCtl->summarized_lsn;
569 [ - + ]: 20 : if (tli != NULL)
570 : 0 : *tli = WalSummarizerCtl->summarized_tli;
571 [ - + ]: 20 : if (lsn_is_exact != NULL)
572 : 0 : *lsn_is_exact = WalSummarizerCtl->lsn_is_exact;
573 : 20 : LWLockRelease(WALSummarizerLock);
574 : 20 : return unsummarized_lsn;
575 : : }
576 : :
577 : 4 : LWLockRelease(WALSummarizerLock);
578 : : }
579 : :
580 : : /*
581 : : * Find the oldest timeline on which WAL still exists, and the earliest
582 : : * segment for which it exists.
583 : : *
584 : : * Note that we do this every time the WAL summarizer process restarts or
585 : : * recovers from an error, in case the contents of pg_wal have changed
586 : : * under us e.g. if some files were removed, either manually - which
587 : : * shouldn't really happen, but might - or by postgres itself, if
588 : : * summarize_wal was turned off and then back on again.
589 : : */
590 : 14 : (void) GetLatestLSN(&latest_tli);
591 : 14 : tles = readTimeLineHistory(latest_tli);
592 [ + - ]: 16 : for (n = list_length(tles) - 1; n >= 0; --n)
593 : : {
594 : 16 : TimeLineHistoryEntry *tle = list_nth(tles, n);
595 : : XLogSegNo oldest_segno;
596 : :
597 : 16 : oldest_segno = XLogGetOldestSegno(tle->tli);
598 [ + + ]: 16 : if (oldest_segno != 0)
599 : : {
600 : : /* Compute oldest LSN that still exists on disk. */
601 : 14 : XLogSegNoOffsetToRecPtr(oldest_segno, 0, wal_segment_size,
602 : : unsummarized_lsn);
603 : :
604 : 14 : unsummarized_tli = tle->tli;
605 : 14 : break;
606 : : }
607 : : }
608 : :
609 : : /*
610 : : * Don't try to summarize anything older than the end LSN of the newest
611 : : * summary file that exists for this timeline.
612 : : */
613 : : existing_summaries =
614 : 14 : GetWalSummaries(unsummarized_tli,
615 : : InvalidXLogRecPtr, InvalidXLogRecPtr);
616 [ - + - - : 14 : foreach(lc, existing_summaries)
- + ]
617 : : {
618 : 0 : WalSummaryFile *ws = lfirst(lc);
619 : :
620 [ # # ]: 0 : if (ws->end_lsn > unsummarized_lsn)
621 : : {
622 : 0 : unsummarized_lsn = ws->end_lsn;
623 : 0 : should_make_exact = true;
624 : : }
625 : : }
626 : :
627 : : /* It really should not be possible for us to find no WAL. */
628 [ - + ]: 14 : if (unsummarized_tli == 0)
629 [ # # ]: 0 : ereport(ERROR,
630 : : errcode(ERRCODE_INTERNAL_ERROR),
631 : : errmsg_internal("no WAL found on timeline %u", latest_tli));
632 : :
633 : : /*
634 : : * If we're the WAL summarizer, we always want to store the values we just
635 : : * computed into shared memory, because those are the values we're going
636 : : * to use to drive our operation, and so they are the authoritative
637 : : * values. Otherwise, we only store values into shared memory if shared
638 : : * memory is uninitialized. Our values are not canonical in such a case,
639 : : * but it's better to have something than nothing, to guide WAL retention.
640 : : */
641 : 14 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
642 [ + + + - ]: 14 : if (am_wal_summarizer || !WalSummarizerCtl->initialized)
643 : : {
644 : 14 : WalSummarizerCtl->initialized = true;
645 : 14 : WalSummarizerCtl->summarized_lsn = unsummarized_lsn;
646 : 14 : WalSummarizerCtl->summarized_tli = unsummarized_tli;
647 : 14 : WalSummarizerCtl->lsn_is_exact = should_make_exact;
648 : 14 : WalSummarizerCtl->pending_lsn = unsummarized_lsn;
649 : : }
650 : : else
651 : 0 : unsummarized_lsn = WalSummarizerCtl->summarized_lsn;
652 : :
653 : : /* Also return the to the caller as required. */
654 [ + + ]: 14 : if (tli != NULL)
655 : 10 : *tli = WalSummarizerCtl->summarized_tli;
656 [ + + ]: 14 : if (lsn_is_exact != NULL)
657 : 10 : *lsn_is_exact = WalSummarizerCtl->lsn_is_exact;
658 : 14 : LWLockRelease(WALSummarizerLock);
659 : :
660 : 14 : return unsummarized_lsn;
661 : : }
662 : :
663 : : /*
664 : : * Wake up the WAL summarizer process.
665 : : *
666 : : * This might not work, because there's no guarantee that the WAL summarizer
667 : : * process was successfully started, and it also might have started but
668 : : * subsequently terminated. So, under normal circumstances, this will get the
669 : : * latch set, but there's no guarantee.
670 : : */
671 : : void
672 : 1864 : WakeupWalSummarizer(void)
673 : : {
674 : : ProcNumber pgprocno;
675 : :
676 [ - + ]: 1864 : if (WalSummarizerCtl == NULL)
677 : 0 : return;
678 : :
679 : 1864 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
680 : 1864 : pgprocno = WalSummarizerCtl->summarizer_pgprocno;
681 : 1864 : LWLockRelease(WALSummarizerLock);
682 : :
683 [ + + ]: 1864 : if (pgprocno != INVALID_PROC_NUMBER)
684 : 10 : SetLatch(&GetPGProcByNumber(pgprocno)->procLatch);
685 : : }
686 : :
687 : : /*
688 : : * Wait until WAL summarization reaches the given LSN, but time out with an
689 : : * error if the summarizer seems to be stick.
690 : : *
691 : : * Returns immediately if summarize_wal is turned off while we wait. Caller
692 : : * is expected to handle this case, if necessary.
693 : : */
694 : : void
695 : 15 : WaitForWalSummarization(XLogRecPtr lsn)
696 : : {
697 : : TimestampTz initial_time,
698 : : cycle_time,
699 : : current_time;
700 : 15 : XLogRecPtr prior_pending_lsn = InvalidXLogRecPtr;
701 : 15 : int deadcycles = 0;
702 : :
703 : 15 : initial_time = cycle_time = GetCurrentTimestamp();
704 : :
705 : : while (1)
706 : 16 : {
707 : 31 : long timeout_in_ms = 10000;
708 : : XLogRecPtr summarized_lsn;
709 : : XLogRecPtr pending_lsn;
710 : :
711 [ - + ]: 31 : CHECK_FOR_INTERRUPTS();
712 : :
713 : : /* If WAL summarization is disabled while we're waiting, give up. */
714 [ - + ]: 31 : if (!summarize_wal)
715 : 0 : return;
716 : :
717 : : /*
718 : : * If the LSN summarized on disk has reached the target value, stop.
719 : : */
720 : 31 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
721 : 31 : summarized_lsn = WalSummarizerCtl->summarized_lsn;
722 : 31 : pending_lsn = WalSummarizerCtl->pending_lsn;
723 : 31 : LWLockRelease(WALSummarizerLock);
724 : :
725 : : /* If WAL summarization has progressed sufficiently, stop waiting. */
726 [ + + ]: 31 : if (summarized_lsn >= lsn)
727 : 15 : break;
728 : :
729 : : /* Recheck current time. */
730 : 16 : current_time = GetCurrentTimestamp();
731 : :
732 : : /* Have we finished the current cycle of waiting? */
733 [ - + ]: 16 : if (TimestampDifferenceMilliseconds(cycle_time,
734 : : current_time) >= timeout_in_ms)
735 : : {
736 : : long elapsed_seconds;
737 : :
738 : : /* Begin new wait cycle. */
739 : 0 : cycle_time = TimestampTzPlusMilliseconds(cycle_time,
740 : : timeout_in_ms);
741 : :
742 : : /*
743 : : * Keep track of the number of cycles during which there has been
744 : : * no progression of pending_lsn. If pending_lsn is not advancing,
745 : : * that means that not only are no new files appearing on disk,
746 : : * but we're not even incorporating new records into the in-memory
747 : : * state.
748 : : */
749 [ # # ]: 0 : if (pending_lsn > prior_pending_lsn)
750 : : {
751 : 0 : prior_pending_lsn = pending_lsn;
752 : 0 : deadcycles = 0;
753 : : }
754 : : else
755 : 0 : ++deadcycles;
756 : :
757 : : /*
758 : : * If we've managed to wait for an entire minute without the WAL
759 : : * summarizer absorbing a single WAL record, error out; probably
760 : : * something is wrong.
761 : : *
762 : : * We could consider also erroring out if the summarizer is taking
763 : : * too long to catch up, but it's not clear what rate of progress
764 : : * would be acceptable and what would be too slow. So instead, we
765 : : * just try to error out in the case where there's no progress at
766 : : * all. That seems likely to catch a reasonable number of the
767 : : * things that can go wrong in practice (e.g. the summarizer
768 : : * process is completely hung, say because somebody hooked up a
769 : : * debugger to it or something) without giving up too quickly when
770 : : * the system is just slow.
771 : : */
772 [ # # ]: 0 : if (deadcycles >= 6)
773 [ # # ]: 0 : ereport(ERROR,
774 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
775 : : errmsg("WAL summarization is not progressing"),
776 : : errdetail("Summarization is needed through %X/%08X, but is stuck at %X/%08X on disk and %X/%08X in memory.",
777 : : LSN_FORMAT_ARGS(lsn),
778 : : LSN_FORMAT_ARGS(summarized_lsn),
779 : : LSN_FORMAT_ARGS(pending_lsn))));
780 : :
781 : :
782 : : /*
783 : : * Otherwise, just let the user know what's happening.
784 : : */
785 : 0 : elapsed_seconds =
786 : 0 : TimestampDifferenceMilliseconds(initial_time,
787 : : current_time) / 1000;
788 [ # # ]: 0 : ereport(WARNING,
789 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
790 : : errmsg_plural("still waiting for WAL summarization through %X/%08X after %ld second",
791 : : "still waiting for WAL summarization through %X/%08X after %ld seconds",
792 : : elapsed_seconds,
793 : : LSN_FORMAT_ARGS(lsn),
794 : : elapsed_seconds),
795 : : errdetail("Summarization has reached %X/%08X on disk and %X/%08X in memory.",
796 : : LSN_FORMAT_ARGS(summarized_lsn),
797 : : LSN_FORMAT_ARGS(pending_lsn))));
798 : : }
799 : :
800 : : /*
801 : : * Align the wait time to prevent drift. This doesn't really matter,
802 : : * but we'd like the warnings about how long we've been waiting to say
803 : : * 10 seconds, 20 seconds, 30 seconds, 40 seconds ... without ever
804 : : * drifting to something that is not a multiple of ten.
805 : : */
806 : 16 : timeout_in_ms -=
807 : 16 : TimestampDifferenceMilliseconds(cycle_time, current_time);
808 : :
809 : : /* Wait and see. */
810 : 16 : ConditionVariableTimedSleep(&WalSummarizerCtl->summary_file_cv,
811 : : timeout_in_ms,
812 : : WAIT_EVENT_WAL_SUMMARY_READY);
813 : : }
814 : :
815 : 15 : ConditionVariableCancelSleep();
816 : : }
817 : :
818 : : /*
819 : : * On exit, update shared memory to make it clear that we're no longer
820 : : * running.
821 : : */
822 : : static void
823 : 10 : WalSummarizerShutdown(int code, Datum arg)
824 : : {
825 : 10 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
826 : 10 : WalSummarizerCtl->summarizer_pgprocno = INVALID_PROC_NUMBER;
827 : 10 : LWLockRelease(WALSummarizerLock);
828 : 10 : }
829 : :
830 : : /*
831 : : * Get the latest LSN that is eligible to be summarized, and set *tli to the
832 : : * corresponding timeline.
833 : : */
834 : : static XLogRecPtr
835 : 158 : GetLatestLSN(TimeLineID *tli)
836 : : {
837 [ + + ]: 158 : if (!RecoveryInProgress())
838 : : {
839 : : /* Don't summarize WAL before it's flushed. */
840 : 143 : return GetFlushRecPtr(tli);
841 : : }
842 : : else
843 : : {
844 : : XLogRecPtr flush_lsn;
845 : : TimeLineID flush_tli;
846 : : XLogRecPtr replay_lsn;
847 : : TimeLineID replay_tli;
848 : : TimeLineID insert_tli;
849 : :
850 : : /*
851 : : * After the insert TLI has been set and before the control file has
852 : : * been updated to show the DB in production, RecoveryInProgress()
853 : : * will return true, because it's not yet safe for all backends to
854 : : * begin writing WAL. However, replay has already ceased, so from our
855 : : * point of view, recovery is already over. We should summarize up to
856 : : * where replay stopped and then prepare to resume at the start of the
857 : : * insert timeline.
858 : : */
859 [ + + ]: 15 : if ((insert_tli = GetWALInsertionTimeLineIfSet()) != 0)
860 : : {
861 : 5 : *tli = insert_tli;
862 : 5 : return GetXLogReplayRecPtr(NULL);
863 : : }
864 : :
865 : : /*
866 : : * What we really want to know is how much WAL has been flushed to
867 : : * disk, but the only flush position available is the one provided by
868 : : * the walreceiver, which may not be running, because this could be
869 : : * crash recovery or recovery via restore_command. So use either the
870 : : * WAL receiver's flush position or the replay position, whichever is
871 : : * further ahead, on the theory that if the WAL has been replayed then
872 : : * it must also have been flushed to disk.
873 : : */
874 : 10 : flush_lsn = GetWalRcvFlushRecPtr(NULL, &flush_tli);
875 : 10 : replay_lsn = GetXLogReplayRecPtr(&replay_tli);
876 [ - + ]: 10 : if (flush_lsn > replay_lsn)
877 : : {
878 : 0 : *tli = flush_tli;
879 : 0 : return flush_lsn;
880 : : }
881 : : else
882 : : {
883 : 10 : *tli = replay_tli;
884 : 10 : return replay_lsn;
885 : : }
886 : : }
887 : : }
888 : :
889 : : /*
890 : : * Compute the LSN at which we switched from current_tli to some later timeline.
891 : : * 'tles' must be the timeline history of the latest timeline.
892 : : *
893 : : * As a side effect, we set *num_descendant_tlis to the number of later TLIs that
894 : : * appear in the timeline history, and *descendant_tlis to an array of those TLIs,
895 : : * starting with immediate successor of current_tli.
896 : : */
897 : : static XLogRecPtr
898 : 1 : WalSummarizerSwitchPoint(TimeLineID current_tli, List *tles,
899 : : int *num_descendant_tlis, TimeLineID **descendant_tlis)
900 : : {
901 : 1 : XLogRecPtr switch_lsn = InvalidXLogRecPtr;
902 : 1 : int count = 0;
903 : :
904 : : /*
905 : : * Find the switch point and, at the same time, count the number of TLIs
906 : : * in this history that are descendants of that TLI.
907 : : */
908 [ + - + - : 3 : foreach_ptr(TimeLineHistoryEntry, tle, tles)
+ + ]
909 : : {
910 [ + + ]: 2 : if (tle->tli == current_tli)
911 : : {
912 : 1 : switch_lsn = tle->end;
913 : 1 : break;
914 : : }
915 : 1 : ++count;
916 : : }
917 : :
918 : : /* Sanity checks. */
919 [ - + ]: 1 : if (!XLogRecPtrIsValid(switch_lsn))
920 [ # # ]: 0 : ereport(ERROR,
921 : : (errmsg("requested timeline %u is not in this server's history",
922 : : current_tli)));
923 [ - + ]: 1 : if (count == 0)
924 [ # # ]: 0 : elog(ERROR, "cannot compute switch point for current TLI %u", current_tli);
925 : :
926 : : /*
927 : : * Generate an array of TLIs that are part of this history and descendants
928 : : * of current_tli. The TLE list starts with the newest timeline and works
929 : : * backward toward older timelines; we want the opposite ordering.
930 : : */
931 : 1 : *num_descendant_tlis = count;
932 : 1 : *descendant_tlis = palloc_array(TimeLineID, count);
933 [ + + ]: 2 : for (int i = 0; i < count; ++i)
934 : : {
935 : : TimeLineHistoryEntry *tle;
936 : :
937 : 1 : tle = (TimeLineHistoryEntry *) list_nth(tles, count - i - 1);
938 : 1 : (*descendant_tlis)[i] = tle->tli;
939 : : }
940 : :
941 : : /* Return value is the switchpoint. */
942 : 1 : return switch_lsn;
943 : : }
944 : :
945 : : /*
946 : : * Interrupt handler for main loop of WAL summarizer process.
947 : : */
948 : : static void
949 : 130125 : ProcessWalSummarizerInterrupts(void)
950 : : {
951 [ + + ]: 130125 : if (ProcSignalBarrierPending)
952 : 1 : ProcessProcSignalBarrier();
953 : :
954 [ - + ]: 130125 : if (ConfigReloadPending)
955 : : {
956 : 0 : ConfigReloadPending = false;
957 : 0 : ProcessConfigFile(PGC_SIGHUP);
958 : : }
959 : :
960 [ + + - + ]: 130125 : if (ShutdownRequestPending || !summarize_wal)
961 : : {
962 [ - + ]: 10 : ereport(DEBUG1,
963 : : errmsg_internal("WAL summarizer shutting down"));
964 : 10 : proc_exit(0);
965 : : }
966 : :
967 : : /* Perform logging of memory contexts of this process */
968 [ - + ]: 130115 : if (LogMemoryContextPending)
969 : 0 : ProcessLogMemoryContextInterrupt();
970 : 130115 : }
971 : :
972 : : /*
973 : : * Summarize a range of WAL records on a single timeline.
974 : : *
975 : : * 'tli' is the timeline to be summarized.
976 : : *
977 : : * 'start_lsn' is the point at which we should start summarizing. If this
978 : : * value comes from the end LSN of the previous record as returned by the
979 : : * xlogreader machinery, 'exact' should be true; otherwise, 'exact' should
980 : : * be false, and this function will search forward for the start of a valid
981 : : * WAL record.
982 : : *
983 : : * 'switch_lsn' is the point at which we should switch to a later timeline,
984 : : * if we're summarizing a historic timeline.
985 : : *
986 : : * 'maximum_lsn' identifies the point beyond which we can't count on being
987 : : * able to read any more WAL. It should be the switch point when reading a
988 : : * historic timeline, or the most-recently-measured end of WAL when reading
989 : : * the current timeline.
990 : : *
991 : : * The return value is the LSN at which the WAL summary actually ends. Most
992 : : * often, a summary file ends because we notice that a checkpoint has
993 : : * occurred and reach the redo pointer of that checkpoint, but sometimes
994 : : * we stop for other reasons, such as a timeline switch.
995 : : */
996 : : static XLogRecPtr
997 : 61 : SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn, bool exact,
998 : : XLogRecPtr switch_lsn, XLogRecPtr maximum_lsn,
999 : : int num_descendant_tlis, TimeLineID *descendant_tlis)
1000 : : {
1001 : : SummarizerReadLocalXLogPrivate *private_data;
1002 : : XLogReaderState *xlogreader;
1003 : : XLogRecPtr summary_start_lsn;
1004 : 61 : XLogRecPtr summary_end_lsn = switch_lsn;
1005 : : char temp_path[MAXPGPATH];
1006 : : char final_path[MAXPGPATH];
1007 : : WalSummaryIO io;
1008 : 61 : BlockRefTable *brtab = CreateEmptyBlockRefTable();
1009 : 61 : bool fast_forward = true;
1010 : : char *errormsg;
1011 : :
1012 : : /* Initialize private data for xlogreader. */
1013 : 61 : private_data = palloc0_object(SummarizerReadLocalXLogPrivate);
1014 : 61 : private_data->tli = tli;
1015 : 61 : private_data->historic = XLogRecPtrIsValid(switch_lsn);
1016 : 61 : private_data->read_upto = maximum_lsn;
1017 : 61 : private_data->num_descendant_tlis = num_descendant_tlis;
1018 : 61 : private_data->descendant_tlis = descendant_tlis;
1019 : :
1020 : : /* Create xlogreader. */
1021 : 61 : xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
1022 : 61 : XL_ROUTINE(.page_read = &summarizer_read_local_xlog_page,
1023 : : .segment_open = &summarizer_wal_segment_open,
1024 : : .segment_close = &wal_segment_close),
1025 : : private_data);
1026 [ - + ]: 61 : if (xlogreader == NULL)
1027 [ # # ]: 0 : ereport(ERROR,
1028 : : (errcode(ERRCODE_OUT_OF_MEMORY),
1029 : : errmsg("out of memory"),
1030 : : errdetail("Failed while allocating a WAL reading processor.")));
1031 : :
1032 : : /*
1033 : : * When exact = false, we're starting from an arbitrary point in the WAL
1034 : : * and must search forward for the start of the next record.
1035 : : *
1036 : : * When exact = true, start_lsn should be either the LSN where a record
1037 : : * begins, or the LSN of a page where the page header is immediately
1038 : : * followed by the start of a new record. XLogBeginRead should tolerate
1039 : : * either case.
1040 : : *
1041 : : * We need to allow for both cases because the behavior of xlogreader
1042 : : * varies. When a record spans two or more xlog pages, the ending LSN
1043 : : * reported by xlogreader will be the starting LSN of the following
1044 : : * record, but when an xlog page boundary falls between two records, the
1045 : : * end LSN for the first will be reported as the first byte of the
1046 : : * following page. We can't know until we read that page how large the
1047 : : * header will be, but we'll have to skip over it to find the next record.
1048 : : */
1049 [ + + ]: 61 : if (exact)
1050 : : {
1051 : : /*
1052 : : * Even if start_lsn is the beginning of a page rather than the
1053 : : * beginning of the first record on that page, we should still use it
1054 : : * as the start LSN for the summary file. That's because we detect
1055 : : * missing summary files by looking for cases where the end LSN of one
1056 : : * file is less than the start LSN of the next file. When only a page
1057 : : * header is skipped, nothing has been missed.
1058 : : */
1059 : 51 : XLogBeginRead(xlogreader, start_lsn);
1060 : 51 : summary_start_lsn = start_lsn;
1061 : : }
1062 : : else
1063 : : {
1064 : 10 : summary_start_lsn = XLogFindNextRecord(xlogreader, start_lsn, &errormsg);
1065 [ - + ]: 10 : if (!XLogRecPtrIsValid(summary_start_lsn))
1066 : : {
1067 : : /*
1068 : : * If we hit end-of-WAL while trying to find the next valid
1069 : : * record, we must be on a historic timeline that has no valid
1070 : : * records that begin after start_lsn and before end of WAL.
1071 : : */
1072 [ # # ]: 0 : if (private_data->end_of_wal)
1073 : : {
1074 [ # # ]: 0 : ereport(DEBUG1,
1075 : : errmsg_internal("could not read WAL from timeline %u at %X/%08X: end of WAL at %X/%08X",
1076 : : tli,
1077 : : LSN_FORMAT_ARGS(start_lsn),
1078 : : LSN_FORMAT_ARGS(private_data->read_upto)));
1079 : :
1080 : : /*
1081 : : * The timeline ends at or after start_lsn, without containing
1082 : : * any records. Thus, we must make sure the main loop does not
1083 : : * iterate. If start_lsn is the end of the timeline, then we
1084 : : * won't actually emit an empty summary file, but otherwise,
1085 : : * we must, to capture the fact that the LSN range in question
1086 : : * contains no interesting WAL records.
1087 : : */
1088 : 0 : summary_start_lsn = start_lsn;
1089 : 0 : summary_end_lsn = private_data->read_upto;
1090 : 0 : switch_lsn = xlogreader->EndRecPtr;
1091 : : }
1092 : : else
1093 : : {
1094 [ # # ]: 0 : if (errormsg)
1095 [ # # ]: 0 : ereport(ERROR,
1096 : : errmsg("could not find a valid record after %X/%08X: %s",
1097 : : LSN_FORMAT_ARGS(start_lsn), errormsg));
1098 : : else
1099 [ # # ]: 0 : ereport(ERROR,
1100 : : errmsg("could not find a valid record after %X/%08X",
1101 : : LSN_FORMAT_ARGS(start_lsn)));
1102 : : }
1103 : : }
1104 : :
1105 : : /* We shouldn't go backward. */
1106 : : Assert(summary_start_lsn >= start_lsn);
1107 : : }
1108 : :
1109 : : /*
1110 : : * Main loop: read xlog records one by one.
1111 : : */
1112 : : while (1)
1113 : 125032 : {
1114 : : int block_id;
1115 : : XLogRecord *record;
1116 : : uint8 rmid;
1117 : :
1118 : 125093 : ProcessWalSummarizerInterrupts();
1119 : :
1120 : : /* We shouldn't go backward. */
1121 : : Assert(summary_start_lsn <= xlogreader->EndRecPtr);
1122 : :
1123 : : /* Now read the next record. */
1124 : 125092 : record = XLogReadRecord(xlogreader, &errormsg);
1125 [ + + ]: 125083 : if (record == NULL)
1126 : : {
1127 [ + - ]: 1 : if (private_data->end_of_wal)
1128 : : {
1129 : : /*
1130 : : * This timeline must be historic and must end before we were
1131 : : * able to read a complete record.
1132 : : */
1133 [ - + ]: 1 : ereport(DEBUG1,
1134 : : errmsg_internal("could not read WAL from timeline %u at %X/%08X: end of WAL at %X/%08X",
1135 : : tli,
1136 : : LSN_FORMAT_ARGS(xlogreader->EndRecPtr),
1137 : : LSN_FORMAT_ARGS(private_data->read_upto)));
1138 : : /* Summary ends at end of WAL. */
1139 : 1 : summary_end_lsn = private_data->read_upto;
1140 : 1 : break;
1141 : : }
1142 [ # # ]: 0 : if (errormsg)
1143 [ # # ]: 0 : ereport(ERROR,
1144 : : (errcode_for_file_access(),
1145 : : errmsg("could not read WAL from timeline %u at %X/%08X: %s",
1146 : : tli, LSN_FORMAT_ARGS(xlogreader->EndRecPtr),
1147 : : errormsg)));
1148 : : else
1149 [ # # ]: 0 : ereport(ERROR,
1150 : : (errcode_for_file_access(),
1151 : : errmsg("could not read WAL from timeline %u at %X/%08X",
1152 : : tli, LSN_FORMAT_ARGS(xlogreader->EndRecPtr))));
1153 : : }
1154 : :
1155 : : /* We shouldn't go backward. */
1156 : : Assert(summary_start_lsn <= xlogreader->EndRecPtr);
1157 : :
1158 [ - + ]: 125082 : if (XLogRecPtrIsValid(switch_lsn) &&
1159 [ # # ]: 0 : xlogreader->ReadRecPtr >= switch_lsn)
1160 : : {
1161 : : /*
1162 : : * Whoops! We've read a record that *starts* after the switch LSN,
1163 : : * contrary to our goal of reading only until we hit the first
1164 : : * record that ends at or after the switch LSN. Pretend we didn't
1165 : : * read it after all by bailing out of this loop right here,
1166 : : * before we do anything with this record.
1167 : : *
1168 : : * This can happen because the last record before the switch LSN
1169 : : * might be continued across multiple pages, and then we might
1170 : : * come to a page with XLP_FIRST_IS_OVERWRITE_CONTRECORD set. In
1171 : : * that case, the record that was continued across multiple pages
1172 : : * is incomplete and will be disregarded, and the read will
1173 : : * restart from the beginning of the page that is flagged
1174 : : * XLP_FIRST_IS_OVERWRITE_CONTRECORD.
1175 : : *
1176 : : * If this case occurs, we can fairly say that the current summary
1177 : : * file ends at the switch LSN exactly. The first record on the
1178 : : * page marked XLP_FIRST_IS_OVERWRITE_CONTRECORD will be
1179 : : * discovered when generating the next summary file.
1180 : : */
1181 : 0 : summary_end_lsn = switch_lsn;
1182 : 0 : break;
1183 : : }
1184 : :
1185 : : /*
1186 : : * Certain types of records require special handling. Redo points and
1187 : : * shutdown checkpoints trigger creation of new summary files and can
1188 : : * also cause us to enter or exit "fast forward" mode. Other types of
1189 : : * records can require special updates to the block reference table.
1190 : : */
1191 : 125082 : rmid = XLogRecGetRmid(xlogreader);
1192 [ + + ]: 125082 : if (rmid == RM_XLOG_ID)
1193 : : {
1194 : : bool new_fast_forward;
1195 : :
1196 : : /*
1197 : : * If we've already processed some WAL records when we hit a redo
1198 : : * point or shutdown checkpoint, then we stop summarization before
1199 : : * including this record in the current file, so that it will be
1200 : : * the first record in the next file.
1201 : : *
1202 : : * When we hit one of those record types as the first record in a
1203 : : * file, we adjust our notion of whether we're fast-forwarding.
1204 : : * Any WAL generated with wal_level=minimal must be skipped
1205 : : * without actually generating any summary file, because an
1206 : : * incremental backup that crosses such WAL would be unsafe.
1207 : : */
1208 [ + + ]: 1642 : if (SummarizeXlogRecord(xlogreader, &new_fast_forward))
1209 : : {
1210 [ + + ]: 111 : if (xlogreader->ReadRecPtr > summary_start_lsn)
1211 : : {
1212 : 50 : summary_end_lsn = xlogreader->ReadRecPtr;
1213 : 50 : break;
1214 : : }
1215 : : else
1216 : 61 : fast_forward = new_fast_forward;
1217 : : }
1218 : : }
1219 [ + - ]: 123440 : else if (!fast_forward)
1220 : : {
1221 : : /*
1222 : : * This switch handles record types that require extra updates to
1223 : : * the contents of the block reference table.
1224 : : */
1225 [ + + + + ]: 123440 : switch (rmid)
1226 : : {
1227 : 10 : case RM_DBASE_ID:
1228 : 10 : SummarizeDbaseRecord(xlogreader, brtab);
1229 : 10 : break;
1230 : 76 : case RM_SMGR_ID:
1231 : 76 : SummarizeSmgrRecord(xlogreader, brtab);
1232 : 76 : break;
1233 : 3340 : case RM_XACT_ID:
1234 : 3340 : SummarizeXactRecord(xlogreader, brtab);
1235 : 3340 : break;
1236 : : }
1237 : : }
1238 : :
1239 : : /*
1240 : : * If we're in fast-forward mode, we don't really need to do anything.
1241 : : * Otherwise, feed block references from xlog record to block
1242 : : * reference table.
1243 : : */
1244 [ + - ]: 125032 : if (!fast_forward)
1245 : : {
1246 [ + + ]: 249594 : for (block_id = 0; block_id <= XLogRecMaxBlockId(xlogreader);
1247 : 124562 : block_id++)
1248 : : {
1249 : : RelFileLocator rlocator;
1250 : : ForkNumber forknum;
1251 : : BlockNumber blocknum;
1252 : :
1253 [ + + ]: 124562 : if (!XLogRecGetBlockTagExtended(xlogreader, block_id, &rlocator,
1254 : : &forknum, &blocknum, NULL))
1255 : 89 : continue;
1256 : :
1257 : : /*
1258 : : * As we do elsewhere, ignore the FSM fork, because it's not
1259 : : * fully WAL-logged.
1260 : : */
1261 [ + + ]: 124473 : if (forknum != FSM_FORKNUM)
1262 : 123880 : BlockRefTableMarkBlockModified(brtab, &rlocator, forknum,
1263 : : blocknum);
1264 : : }
1265 : : }
1266 : :
1267 : : /* Update our notion of where this summary file ends. */
1268 : 125032 : summary_end_lsn = xlogreader->EndRecPtr;
1269 : :
1270 : : /* Also update shared memory. */
1271 : 125032 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
1272 : : Assert(summary_end_lsn >= WalSummarizerCtl->summarized_lsn);
1273 : 125032 : WalSummarizerCtl->pending_lsn = summary_end_lsn;
1274 : 125032 : LWLockRelease(WALSummarizerLock);
1275 : :
1276 : : /*
1277 : : * If we have a switch LSN and have reached it, stop before reading
1278 : : * the next record.
1279 : : */
1280 [ - + ]: 125032 : if (XLogRecPtrIsValid(switch_lsn) &&
1281 [ # # ]: 0 : xlogreader->EndRecPtr >= switch_lsn)
1282 : 0 : break;
1283 : : }
1284 : :
1285 : : /* Destroy xlogreader. */
1286 : 51 : pfree(xlogreader->private_data);
1287 : 51 : XLogReaderFree(xlogreader);
1288 : :
1289 : : /*
1290 : : * If a timeline switch occurs, we may fail to make any progress at all
1291 : : * before exiting the loop above. If that happens, we don't write a WAL
1292 : : * summary file at all. We can also skip writing a file if we're in
1293 : : * fast-forward mode.
1294 : : */
1295 [ + - + - ]: 51 : if (summary_end_lsn > summary_start_lsn && !fast_forward)
1296 : : {
1297 : : /* Generate temporary and final path name. */
1298 : 51 : snprintf(temp_path, MAXPGPATH,
1299 : : XLOGDIR "/summaries/temp.summary");
1300 : 51 : snprintf(final_path, MAXPGPATH,
1301 : : XLOGDIR "/summaries/%08X%08X%08X%08X%08X.summary",
1302 : : tli,
1303 : 51 : LSN_FORMAT_ARGS(summary_start_lsn),
1304 : 51 : LSN_FORMAT_ARGS(summary_end_lsn));
1305 : :
1306 : : /* Open the temporary file for writing. */
1307 : 51 : io.filepos = 0;
1308 : 51 : io.file = PathNameOpenFile(temp_path, O_WRONLY | O_CREAT | O_TRUNC);
1309 [ - + ]: 51 : if (io.file < 0)
1310 [ # # ]: 0 : ereport(ERROR,
1311 : : (errcode_for_file_access(),
1312 : : errmsg("could not create file \"%s\": %m", temp_path)));
1313 : :
1314 : : /* Write the data. */
1315 : 51 : WriteBlockRefTable(brtab, WriteWalSummary, &io);
1316 : :
1317 : : /* Close temporary file and shut down xlogreader. */
1318 : 51 : FileClose(io.file);
1319 : :
1320 : : /* Tell the user what we did. */
1321 [ - + ]: 51 : ereport(DEBUG1,
1322 : : errmsg_internal("summarized WAL on TLI %u from %X/%08X to %X/%08X",
1323 : : tli,
1324 : : LSN_FORMAT_ARGS(summary_start_lsn),
1325 : : LSN_FORMAT_ARGS(summary_end_lsn)));
1326 : :
1327 : : /* Durably rename the new summary into place. */
1328 : 51 : durable_rename(temp_path, final_path, ERROR);
1329 : : }
1330 : :
1331 : : /* If we skipped a non-zero amount of WAL, log a debug message. */
1332 [ + - - + ]: 51 : if (summary_end_lsn > summary_start_lsn && fast_forward)
1333 [ # # ]: 0 : ereport(DEBUG1,
1334 : : errmsg_internal("skipped summarizing WAL on TLI %u from %X/%08X to %X/%08X",
1335 : : tli,
1336 : : LSN_FORMAT_ARGS(summary_start_lsn),
1337 : : LSN_FORMAT_ARGS(summary_end_lsn)));
1338 : :
1339 : 51 : return summary_end_lsn;
1340 : : }
1341 : :
1342 : : /*
1343 : : * Special handling for WAL records with RM_DBASE_ID.
1344 : : */
1345 : : static void
1346 : 10 : SummarizeDbaseRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
1347 : : {
1348 : 10 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1349 : :
1350 : : /*
1351 : : * We use relfilenode zero for a given database OID and tablespace OID to
1352 : : * indicate that all relations with that pair of IDs have been recreated
1353 : : * if they exist at all. Effectively, we're setting a limit block of 0 for
1354 : : * all such relfilenodes.
1355 : : *
1356 : : * Technically, this special handling is only needed in the case of
1357 : : * XLOG_DBASE_CREATE_FILE_COPY, because that can create a whole bunch of
1358 : : * relation files in a directory without logging anything specific to each
1359 : : * one. If we didn't mark the whole DB OID/TS OID combination in some way,
1360 : : * then a tablespace that was dropped after the reference backup and
1361 : : * recreated using the FILE_COPY method prior to the incremental backup
1362 : : * would look just like one that was never touched at all, which would be
1363 : : * catastrophic.
1364 : : *
1365 : : * But it seems best to adopt this treatment for all records that drop or
1366 : : * create a DB OID/TS OID combination. That's similar to how we treat the
1367 : : * limit block for individual relations, and it's an extra layer of safety
1368 : : * here. We can never lose data by marking more stuff as needing to be
1369 : : * backed up in full.
1370 : : */
1371 [ + - ]: 10 : if (info == XLOG_DBASE_CREATE_FILE_COPY)
1372 : : {
1373 : : xl_dbase_create_file_copy_rec *xlrec;
1374 : : RelFileLocator rlocator;
1375 : :
1376 : 10 : xlrec =
1377 : 10 : (xl_dbase_create_file_copy_rec *) XLogRecGetData(xlogreader);
1378 : 10 : rlocator.spcOid = xlrec->tablespace_id;
1379 : 10 : rlocator.dbOid = xlrec->db_id;
1380 : 10 : rlocator.relNumber = 0;
1381 : 10 : BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1382 : : }
1383 [ # # ]: 0 : else if (info == XLOG_DBASE_CREATE_WAL_LOG)
1384 : : {
1385 : : xl_dbase_create_wal_log_rec *xlrec;
1386 : : RelFileLocator rlocator;
1387 : :
1388 : 0 : xlrec = (xl_dbase_create_wal_log_rec *) XLogRecGetData(xlogreader);
1389 : 0 : rlocator.spcOid = xlrec->tablespace_id;
1390 : 0 : rlocator.dbOid = xlrec->db_id;
1391 : 0 : rlocator.relNumber = 0;
1392 : 0 : BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1393 : : }
1394 [ # # ]: 0 : else if (info == XLOG_DBASE_DROP)
1395 : : {
1396 : : xl_dbase_drop_rec *xlrec;
1397 : : RelFileLocator rlocator;
1398 : : int i;
1399 : :
1400 : 0 : xlrec = (xl_dbase_drop_rec *) XLogRecGetData(xlogreader);
1401 : 0 : rlocator.dbOid = xlrec->db_id;
1402 : 0 : rlocator.relNumber = 0;
1403 [ # # ]: 0 : for (i = 0; i < xlrec->ntablespaces; ++i)
1404 : : {
1405 : 0 : rlocator.spcOid = xlrec->tablespace_ids[i];
1406 : 0 : BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1407 : : }
1408 : : }
1409 : 10 : }
1410 : :
1411 : : /*
1412 : : * Special handling for WAL records with RM_SMGR_ID.
1413 : : */
1414 : : static void
1415 : 76 : SummarizeSmgrRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
1416 : : {
1417 : 76 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1418 : :
1419 [ + + ]: 76 : if (info == XLOG_SMGR_CREATE)
1420 : : {
1421 : : xl_smgr_create *xlrec;
1422 : :
1423 : : /*
1424 : : * If a new relation fork is created on disk, there is no point
1425 : : * tracking anything about which blocks have been modified, because
1426 : : * the whole thing will be new. Hence, set the limit block for this
1427 : : * fork to 0.
1428 : : *
1429 : : * Ignore the FSM fork, which is not fully WAL-logged.
1430 : : */
1431 : 73 : xlrec = (xl_smgr_create *) XLogRecGetData(xlogreader);
1432 : :
1433 [ + - ]: 73 : if (xlrec->forkNum != FSM_FORKNUM)
1434 : 73 : BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1435 : : xlrec->forkNum, 0);
1436 : : }
1437 [ + - ]: 3 : else if (info == XLOG_SMGR_TRUNCATE)
1438 : : {
1439 : : xl_smgr_truncate *xlrec;
1440 : :
1441 : 3 : xlrec = (xl_smgr_truncate *) XLogRecGetData(xlogreader);
1442 : :
1443 : : /*
1444 : : * If a relation fork is truncated on disk, there is no point in
1445 : : * tracking anything about block modifications beyond the truncation
1446 : : * point.
1447 : : *
1448 : : * We ignore SMGR_TRUNCATE_FSM here because the FSM isn't fully
1449 : : * WAL-logged and thus we can't track modified blocks for it anyway.
1450 : : */
1451 [ + - ]: 3 : if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
1452 : 3 : BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1453 : : MAIN_FORKNUM, xlrec->blkno);
1454 [ + - ]: 3 : if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0)
1455 : : {
1456 : 3 : BlockNumber vmlimit = visibilitymap_truncation_length(xlrec->blkno);
1457 : :
1458 : 3 : BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1459 : : VISIBILITYMAP_FORKNUM, vmlimit);
1460 : :
1461 : : /*
1462 : : * visibilitymap_prepare_truncate() may clear trailing bits in the
1463 : : * last remaining VM page without WAL-logging the page. Mark it
1464 : : * as modified so that incremental backups include its current
1465 : : * contents. Do this even at a VM page boundary, where no bits
1466 : : * need clearing.
1467 : : */
1468 [ + - ]: 3 : if (vmlimit > 0)
1469 : 3 : BlockRefTableMarkBlockModified(brtab, &xlrec->rlocator,
1470 : : VISIBILITYMAP_FORKNUM,
1471 : : vmlimit - 1);
1472 : : }
1473 : : }
1474 : 76 : }
1475 : :
1476 : : /*
1477 : : * Special handling for WAL records with RM_XACT_ID.
1478 : : */
1479 : : static void
1480 : 3340 : SummarizeXactRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
1481 : : {
1482 : 3340 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1483 : 3340 : uint8 xact_info = info & XLOG_XACT_OPMASK;
1484 : :
1485 [ - + - - ]: 3340 : if (xact_info == XLOG_XACT_COMMIT ||
1486 : : xact_info == XLOG_XACT_COMMIT_PREPARED)
1487 : 3340 : {
1488 : 3340 : xl_xact_commit *xlrec = (xl_xact_commit *) XLogRecGetData(xlogreader);
1489 : : xl_xact_parsed_commit parsed;
1490 : : int i;
1491 : :
1492 : : /*
1493 : : * Don't track modified blocks for any relations that were removed on
1494 : : * commit.
1495 : : */
1496 : 3340 : ParseCommitRecord(XLogRecGetInfo(xlogreader), xlrec, &parsed);
1497 [ - + ]: 3340 : for (i = 0; i < parsed.nrels; ++i)
1498 : : {
1499 : : ForkNumber forknum;
1500 : :
1501 [ # # ]: 0 : for (forknum = 0; forknum <= MAX_FORKNUM; ++forknum)
1502 [ # # ]: 0 : if (forknum != FSM_FORKNUM)
1503 : 0 : BlockRefTableSetLimitBlock(brtab, &parsed.xlocators[i],
1504 : : forknum, 0);
1505 : : }
1506 : : }
1507 [ # # # # ]: 0 : else if (xact_info == XLOG_XACT_ABORT ||
1508 : : xact_info == XLOG_XACT_ABORT_PREPARED)
1509 : : {
1510 : 0 : xl_xact_abort *xlrec = (xl_xact_abort *) XLogRecGetData(xlogreader);
1511 : : xl_xact_parsed_abort parsed;
1512 : : int i;
1513 : :
1514 : : /*
1515 : : * Don't track modified blocks for any relations that were removed on
1516 : : * abort.
1517 : : */
1518 : 0 : ParseAbortRecord(XLogRecGetInfo(xlogreader), xlrec, &parsed);
1519 [ # # ]: 0 : for (i = 0; i < parsed.nrels; ++i)
1520 : : {
1521 : : ForkNumber forknum;
1522 : :
1523 [ # # ]: 0 : for (forknum = 0; forknum <= MAX_FORKNUM; ++forknum)
1524 [ # # ]: 0 : if (forknum != FSM_FORKNUM)
1525 : 0 : BlockRefTableSetLimitBlock(brtab, &parsed.xlocators[i],
1526 : : forknum, 0);
1527 : : }
1528 : : }
1529 : 3340 : }
1530 : :
1531 : : /*
1532 : : * Special handling for WAL records with RM_XLOG_ID.
1533 : : *
1534 : : * The return value is true if WAL summarization should stop before this
1535 : : * record and false otherwise. When the return value is true,
1536 : : * *new_fast_forward indicates whether future processing should be done
1537 : : * in fast forward mode (i.e. read WAL without emitting summaries) or not.
1538 : : */
1539 : : static bool
1540 : 1642 : SummarizeXlogRecord(XLogReaderState *xlogreader, bool *new_fast_forward)
1541 : : {
1542 : 1642 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1543 : : int record_wal_level;
1544 : :
1545 [ + + ]: 1642 : if (info == XLOG_CHECKPOINT_REDO)
1546 : : {
1547 : : xl_checkpoint_redo xlrec;
1548 : :
1549 : : /* Payload is wal_level at the time record was written. */
1550 : 65 : memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_checkpoint_redo));
1551 : 65 : record_wal_level = xlrec.wal_level;
1552 : : }
1553 [ + + ]: 1577 : else if (info == XLOG_CHECKPOINT_SHUTDOWN)
1554 : : {
1555 : : CheckPoint rec_ckpt;
1556 : :
1557 : : /* Extract wal_level at time record was written from payload. */
1558 : 35 : memcpy(&rec_ckpt, XLogRecGetData(xlogreader), sizeof(CheckPoint));
1559 : 35 : record_wal_level = rec_ckpt.wal_level;
1560 : : }
1561 [ + + ]: 1542 : else if (info == XLOG_PARAMETER_CHANGE)
1562 : : {
1563 : : xl_parameter_change xlrec;
1564 : :
1565 : : /* Extract wal_level at time record was written from payload. */
1566 : 10 : memcpy(&xlrec, XLogRecGetData(xlogreader),
1567 : : sizeof(xl_parameter_change));
1568 : 10 : record_wal_level = xlrec.wal_level;
1569 : : }
1570 [ + + ]: 1532 : else if (info == XLOG_END_OF_RECOVERY)
1571 : : {
1572 : : xl_end_of_recovery xlrec;
1573 : :
1574 : : /* Extract wal_level at time record was written from payload. */
1575 : 1 : memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery));
1576 : 1 : record_wal_level = xlrec.wal_level;
1577 : : }
1578 : : else
1579 : : {
1580 : : /* No special handling required. Return false. */
1581 : 1531 : return false;
1582 : : }
1583 : :
1584 : : /*
1585 : : * Redo can only begin at an XLOG_CHECKPOINT_REDO or
1586 : : * XLOG_CHECKPOINT_SHUTDOWN record, so we want WAL summarization to begin
1587 : : * at those points. Hence, when those records are encountered, return
1588 : : * true, so that we stop just before summarizing either of those records.
1589 : : *
1590 : : * We also reach here if we just saw XLOG_END_OF_RECOVERY or
1591 : : * XLOG_PARAMETER_CHANGE. These are not places where recovery can start,
1592 : : * but they're still relevant here. A new timeline can begin with
1593 : : * XLOG_END_OF_RECOVERY, so we need to confirm the WAL level at that
1594 : : * point; and a restart can provoke XLOG_PARAMETER_CHANGE after an
1595 : : * intervening change to postgresql.conf, which might force us to stop
1596 : : * summarizing.
1597 : : */
1598 : 111 : *new_fast_forward = (record_wal_level == WAL_LEVEL_MINIMAL);
1599 : 111 : return true;
1600 : : }
1601 : :
1602 : : /*
1603 : : * Similar to wal_segment_open, but checks for a file on any descendant timelines
1604 : : * known to us if no file is found on the requested timeline.
1605 : : */
1606 : : static void
1607 : 79 : summarizer_wal_segment_open(XLogReaderState *state, XLogSegNo nextSegNo,
1608 : : TimeLineID *tli_p)
1609 : : {
1610 : 79 : SummarizerReadLocalXLogPrivate *private_data = state->private_data;
1611 : 79 : int count = 0;
1612 : 79 : TimeLineID tli = *tli_p;
1613 : : char path[MAXPGPATH];
1614 : :
1615 : : for (;;)
1616 : : {
1617 : 79 : XLogFilePath(path, tli, nextSegNo, state->segcxt.ws_segsize);
1618 : 79 : state->seg.ws_file = BasicOpenFile(path, O_RDONLY | PG_BINARY);
1619 [ + - ]: 79 : if (state->seg.ws_file >= 0)
1620 : : {
1621 : 79 : XLogReaderRegisterResetCallback(state);
1622 : 79 : *tli_p = tli;
1623 : 79 : return;
1624 : : }
1625 : :
1626 : : /*
1627 : : * If the error is anything other than file-not-found, complain at
1628 : : * once.
1629 : : */
1630 [ # # ]: 0 : if (errno != ENOENT)
1631 [ # # ]: 0 : ereport(ERROR,
1632 : : (errcode_for_file_access(),
1633 : : errmsg("could not open file \"%s\": %m",
1634 : : path)));
1635 : :
1636 : : /* Try other timelines, if any remain. */
1637 [ # # ]: 0 : if (count >= private_data->num_descendant_tlis)
1638 : 0 : break;
1639 : 0 : tli = private_data->descendant_tlis[count];
1640 : 0 : ++count;
1641 : : }
1642 : :
1643 : : /* Complain about the originally requested filename. */
1644 : 0 : XLogFilePath(path, *tli_p, nextSegNo, state->segcxt.ws_segsize);
1645 [ # # ]: 0 : ereport(ERROR,
1646 : : (errcode_for_file_access(),
1647 : : errmsg("requested WAL segment %s has already been removed",
1648 : : path)));
1649 : : }
1650 : :
1651 : : /*
1652 : : * Similar to read_local_xlog_page, but limited to read from one particular
1653 : : * timeline. If the end of WAL is reached, it will wait for more if reading
1654 : : * from the current timeline, or give up if reading from a historic timeline.
1655 : : * In the latter case, it will also set private_data->end_of_wal = true.
1656 : : *
1657 : : * Caller must set private_data->tli to the TLI of interest,
1658 : : * private_data->read_upto to the lowest LSN that is not known to be safe
1659 : : * to read on that timeline, and private_data->historic to true if and only
1660 : : * if the timeline is not the current timeline. This function will update
1661 : : * private_data->read_upto and private_data->historic if more WAL appears
1662 : : * on the current timeline or if the current timeline becomes historic.
1663 : : */
1664 : : static int
1665 : 4791 : summarizer_read_local_xlog_page(XLogReaderState *state,
1666 : : XLogRecPtr targetPagePtr, int reqLen,
1667 : : XLogRecPtr targetRecPtr, char *cur_page)
1668 : : {
1669 : : int count;
1670 : : WALReadError errinfo;
1671 : : SummarizerReadLocalXLogPrivate *private_data;
1672 : :
1673 : 4791 : ProcessWalSummarizerInterrupts();
1674 : :
1675 : 4790 : private_data = (SummarizerReadLocalXLogPrivate *)
1676 : : state->private_data;
1677 : :
1678 : : while (1)
1679 : : {
1680 [ + + ]: 4872 : if (targetPagePtr + XLOG_BLCKSZ <= private_data->read_upto)
1681 : : {
1682 : : /*
1683 : : * more than one block available; read only that block, have
1684 : : * caller come back if they need more.
1685 : : */
1686 : 4716 : count = XLOG_BLCKSZ;
1687 : 4716 : break;
1688 : : }
1689 [ + + ]: 156 : else if (targetPagePtr + reqLen > private_data->read_upto)
1690 : : {
1691 : : /* We don't seem to have enough data. */
1692 [ + + ]: 91 : if (private_data->historic)
1693 : : {
1694 : : /*
1695 : : * This is a historic timeline, so there will never be any
1696 : : * more data than we have currently.
1697 : : */
1698 : 1 : private_data->end_of_wal = true;
1699 : 1 : return -1;
1700 : : }
1701 : : else
1702 : : {
1703 : : XLogRecPtr latest_lsn;
1704 : : TimeLineID latest_tli;
1705 : :
1706 : : /*
1707 : : * This is - or at least was up until very recently - the
1708 : : * current timeline, so more data might show up. Delay here
1709 : : * so we don't tight-loop.
1710 : : */
1711 : 90 : ProcessWalSummarizerInterrupts();
1712 : 82 : summarizer_wait_for_wal();
1713 : :
1714 : : /* Recheck end-of-WAL. */
1715 : 82 : latest_lsn = GetLatestLSN(&latest_tli);
1716 [ + + ]: 82 : if (private_data->tli == latest_tli)
1717 : : {
1718 : : /* Still the current timeline, update max LSN. */
1719 : : Assert(latest_lsn >= private_data->read_upto);
1720 : 81 : private_data->read_upto = latest_lsn;
1721 : : }
1722 : : else
1723 : : {
1724 : 1 : List *tles = readTimeLineHistory(latest_tli);
1725 : : XLogRecPtr switchpoint;
1726 : :
1727 : : /*
1728 : : * The timeline we're scanning is no longer the latest
1729 : : * one. Figure out when it ended.
1730 : : */
1731 : 1 : private_data->historic = true;
1732 : 1 : switchpoint = tliSwitchPoint(private_data->tli, tles,
1733 : : NULL);
1734 : :
1735 : : /*
1736 : : * Allow reads up to exactly the switch point.
1737 : : *
1738 : : * It's possible that this will cause read_upto to move
1739 : : * backwards, because we might have been promoted before
1740 : : * reaching the end of the previous timeline. In that
1741 : : * case, the next loop iteration will likely conclude that
1742 : : * we've reached end of WAL.
1743 : : */
1744 : 1 : private_data->read_upto = switchpoint;
1745 : :
1746 : : /* Debugging output. */
1747 [ - + ]: 1 : ereport(DEBUG1,
1748 : : errmsg_internal("timeline %u became historic, can read up to %X/%08X",
1749 : : private_data->tli, LSN_FORMAT_ARGS(private_data->read_upto)));
1750 : : }
1751 : :
1752 : : /* Go around and try again. */
1753 : : }
1754 : : }
1755 : : else
1756 : : {
1757 : : /* enough bytes available to satisfy the request */
1758 : 65 : count = private_data->read_upto - targetPagePtr;
1759 : 65 : break;
1760 : : }
1761 : : }
1762 : :
1763 [ - + ]: 4781 : if (!WALRead(state, cur_page, targetPagePtr, count,
1764 : : private_data->tli, &errinfo))
1765 : 0 : WALReadRaiseError(&errinfo);
1766 : :
1767 : : /* Track that we read a page, for sleep time calculation. */
1768 : 4781 : ++pages_read_since_last_sleep;
1769 : :
1770 : : /* number of valid bytes in the buffer */
1771 : 4781 : return count;
1772 : : }
1773 : :
1774 : : /*
1775 : : * Sleep for long enough that we believe it's likely that more WAL will
1776 : : * be available afterwards.
1777 : : */
1778 : : static void
1779 : 82 : summarizer_wait_for_wal(void)
1780 : : {
1781 [ + + ]: 82 : if (pages_read_since_last_sleep == 0)
1782 : : {
1783 : : /*
1784 : : * No pages were read since the last sleep, so double the sleep time,
1785 : : * but not beyond the maximum allowable value.
1786 : : */
1787 : 34 : sleep_quanta = Min(sleep_quanta * 2, MAX_SLEEP_QUANTA);
1788 : : }
1789 [ + + ]: 48 : else if (pages_read_since_last_sleep > 1)
1790 : : {
1791 : : /*
1792 : : * Multiple pages were read since the last sleep, so reduce the sleep
1793 : : * time.
1794 : : *
1795 : : * A large burst of activity should be able to quickly reduce the
1796 : : * sleep time to the minimum, but we don't want a handful of extra WAL
1797 : : * records to provoke a strong reaction. We choose to reduce the sleep
1798 : : * time by 1 quantum for each page read beyond the first, which is a
1799 : : * fairly arbitrary way of trying to be reactive without overreacting.
1800 : : */
1801 [ + + ]: 37 : if (pages_read_since_last_sleep > sleep_quanta - 1)
1802 : 36 : sleep_quanta = 1;
1803 : : else
1804 : 1 : sleep_quanta -= pages_read_since_last_sleep;
1805 : : }
1806 : :
1807 : : /* Report pending statistics to the cumulative stats system. */
1808 : 82 : pgstat_report_wal(false);
1809 : :
1810 : : /* OK, now sleep. */
1811 : 82 : (void) WaitLatch(MyLatch,
1812 : : WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
1813 : : sleep_quanta * MS_PER_SLEEP_QUANTUM,
1814 : : WAIT_EVENT_WAL_SUMMARIZER_WAL);
1815 : 82 : ResetLatch(MyLatch);
1816 : :
1817 : : /* Reset count of pages read. */
1818 : 82 : pages_read_since_last_sleep = 0;
1819 : 82 : }
1820 : :
1821 : : /*
1822 : : * Remove WAL summaries whose mtimes are older than wal_summary_keep_time.
1823 : : */
1824 : : static void
1825 : 62 : MaybeRemoveOldWalSummaries(void)
1826 : : {
1827 : 62 : XLogRecPtr redo_pointer = GetRedoRecPtr();
1828 : : List *wslist;
1829 : : time_t cutoff_time;
1830 : :
1831 : : /* If WAL summary removal is disabled, don't do anything. */
1832 [ - + ]: 62 : if (wal_summary_keep_time == 0)
1833 : 0 : return;
1834 : :
1835 : : /*
1836 : : * If the redo pointer has not advanced, don't do anything.
1837 : : *
1838 : : * This has the effect that we only try to remove old WAL summary files
1839 : : * once per checkpoint cycle.
1840 : : */
1841 [ + + ]: 62 : if (redo_pointer == redo_pointer_at_last_summary_removal)
1842 : 42 : return;
1843 : 20 : redo_pointer_at_last_summary_removal = redo_pointer;
1844 : :
1845 : : /*
1846 : : * Files should only be removed if the last modification time precedes the
1847 : : * cutoff time we compute here.
1848 : : */
1849 : 20 : cutoff_time = time(NULL) - wal_summary_keep_time * SECS_PER_MINUTE;
1850 : :
1851 : : /* Get all the summaries that currently exist. */
1852 : 20 : wslist = GetWalSummaries(0, InvalidXLogRecPtr, InvalidXLogRecPtr);
1853 : :
1854 : : /* Loop until all summaries have been considered for removal. */
1855 [ + + ]: 32 : while (wslist != NIL)
1856 : : {
1857 : : ListCell *lc;
1858 : : XLogSegNo oldest_segno;
1859 : 12 : XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
1860 : : TimeLineID selected_tli;
1861 : :
1862 : 12 : ProcessWalSummarizerInterrupts();
1863 : :
1864 : : /*
1865 : : * Pick a timeline for which some summary files still exist on disk,
1866 : : * and find the oldest LSN that still exists on disk for that
1867 : : * timeline.
1868 : : */
1869 : 12 : selected_tli = ((WalSummaryFile *) linitial(wslist))->tli;
1870 : 12 : oldest_segno = XLogGetOldestSegno(selected_tli);
1871 [ + - ]: 12 : if (oldest_segno != 0)
1872 : 12 : XLogSegNoOffsetToRecPtr(oldest_segno, 0, wal_segment_size,
1873 : : oldest_lsn);
1874 : :
1875 : :
1876 : : /* Consider each WAL file on the selected timeline in turn. */
1877 [ + + + + : 89 : foreach(lc, wslist)
+ + ]
1878 : : {
1879 : 77 : WalSummaryFile *ws = lfirst(lc);
1880 : :
1881 : 77 : ProcessWalSummarizerInterrupts();
1882 : :
1883 : : /* If it's not on this timeline, it's not time to consider it. */
1884 [ + + ]: 77 : if (selected_tli != ws->tli)
1885 : 3 : continue;
1886 : :
1887 : : /*
1888 : : * If the WAL doesn't exist any more, we can remove it if the file
1889 : : * modification time is old enough.
1890 : : */
1891 [ + - + + ]: 74 : if (!XLogRecPtrIsValid(oldest_lsn) || ws->end_lsn <= oldest_lsn)
1892 : 21 : RemoveWalSummaryIfOlderThan(ws, cutoff_time);
1893 : :
1894 : : /*
1895 : : * Whether we removed the file or not, we need not consider it
1896 : : * again.
1897 : : */
1898 : 74 : wslist = foreach_delete_current(wslist, lc);
1899 : 74 : pfree(ws);
1900 : : }
1901 : : }
1902 : : }
|