Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * walsummarizer.c
4 : : *
5 : : * Background process to perform WAL summarization, if it is enabled.
6 : : * It continuously scans the write-ahead log and periodically emits a
7 : : * summary file which indicates which blocks in which relation forks
8 : : * were modified by WAL records in the LSN range covered by the summary
9 : : * file. See walsummary.c and blkreftable.c for more details on the
10 : : * naming and contents of WAL summary files.
11 : : *
12 : : * If configured to do, this background process will also remove WAL
13 : : * summary files when the file timestamp is older than a configurable
14 : : * threshold (but only if the WAL has been removed first).
15 : : *
16 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
17 : : *
18 : : * IDENTIFICATION
19 : : * src/backend/postmaster/walsummarizer.c
20 : : *
21 : : *-------------------------------------------------------------------------
22 : : */
23 : : #include "postgres.h"
24 : :
25 : : #include "access/timeline.h"
26 : : #include "access/visibilitymap.h"
27 : : #include "access/xlog.h"
28 : : #include "access/xlog_internal.h"
29 : : #include "access/xlogrecovery.h"
30 : : #include "access/xlogutils.h"
31 : : #include "backup/walsummary.h"
32 : : #include "catalog/storage_xlog.h"
33 : : #include "commands/dbcommands_xlog.h"
34 : : #include "common/blkreftable.h"
35 : : #include "libpq/pqsignal.h"
36 : : #include "miscadmin.h"
37 : : #include "pgstat.h"
38 : : #include "postmaster/auxprocess.h"
39 : : #include "postmaster/interrupt.h"
40 : : #include "postmaster/walsummarizer.h"
41 : : #include "replication/walreceiver.h"
42 : : #include "storage/aio_subsys.h"
43 : : #include "storage/fd.h"
44 : : #include "storage/ipc.h"
45 : : #include "storage/latch.h"
46 : : #include "storage/lwlock.h"
47 : : #include "storage/proc.h"
48 : : #include "storage/procsignal.h"
49 : : #include "storage/shmem.h"
50 : : #include "storage/subsystems.h"
51 : : #include "utils/guc.h"
52 : : #include "utils/memutils.h"
53 : : #include "utils/wait_event.h"
54 : :
55 : : /*
56 : : * Data in shared memory related to WAL summarization.
57 : : */
58 : : typedef struct
59 : : {
60 : : /*
61 : : * These fields are protected by WALSummarizerLock.
62 : : *
63 : : * Until we've discovered what summary files already exist on disk and
64 : : * stored that information in shared memory, initialized is false and the
65 : : * other fields here contain no meaningful information. After that has
66 : : * been done, initialized is true.
67 : : *
68 : : * summarized_tli and summarized_lsn indicate the last LSN and TLI at
69 : : * which the next summary file will start. Normally, these are the LSN and
70 : : * TLI at which the last file ended; in such case, lsn_is_exact is true.
71 : : * If, however, the LSN is just an approximation, then lsn_is_exact is
72 : : * false. This can happen if, for example, there are no existing WAL
73 : : * summary files at startup. In that case, we have to derive the position
74 : : * at which to start summarizing from the WAL files that exist on disk,
75 : : * and so the LSN might point to the start of the next file even though
76 : : * that might happen to be in the middle of a WAL record.
77 : : *
78 : : * summarizer_pgprocno is the proc number of the summarizer process, if
79 : : * one is running, or else INVALID_PROC_NUMBER.
80 : : *
81 : : * pending_lsn is used by the summarizer to advertise the ending LSN of a
82 : : * record it has recently read. It shouldn't ever be less than
83 : : * summarized_lsn, but might be greater, because the summarizer buffers
84 : : * data for a range of LSNs in memory before writing out a new file.
85 : : */
86 : : bool initialized;
87 : : TimeLineID summarized_tli;
88 : : XLogRecPtr summarized_lsn;
89 : : bool lsn_is_exact;
90 : : ProcNumber summarizer_pgprocno;
91 : : XLogRecPtr pending_lsn;
92 : :
93 : : /*
94 : : * This field handles its own synchronization.
95 : : */
96 : : ConditionVariable summary_file_cv;
97 : : } WalSummarizerData;
98 : :
99 : : /*
100 : : * Private data for our xlogreader's page read callback.
101 : : */
102 : : typedef struct
103 : : {
104 : : TimeLineID tli;
105 : : bool historic;
106 : : XLogRecPtr read_upto;
107 : : bool end_of_wal;
108 : : } SummarizerReadLocalXLogPrivate;
109 : :
110 : : /* Pointer to shared memory state. */
111 : : static WalSummarizerData *WalSummarizerCtl;
112 : :
113 : : static void WalSummarizerShmemRequest(void *arg);
114 : : static void WalSummarizerShmemInit(void *arg);
115 : :
116 : : const ShmemCallbacks WalSummarizerShmemCallbacks = {
117 : : .request_fn = WalSummarizerShmemRequest,
118 : : .init_fn = WalSummarizerShmemInit,
119 : : };
120 : :
121 : : /*
122 : : * When we reach end of WAL and need to read more, we sleep for a number of
123 : : * milliseconds that is an integer multiple of MS_PER_SLEEP_QUANTUM. This is
124 : : * the multiplier. It should vary between 1 and MAX_SLEEP_QUANTA, depending
125 : : * on system activity. See summarizer_wait_for_wal() for how we adjust this.
126 : : */
127 : : static long sleep_quanta = 1;
128 : :
129 : : /*
130 : : * The sleep time will always be a multiple of 200ms and will not exceed
131 : : * thirty seconds (150 * 200 = 30 * 1000). Note that the timeout here needs
132 : : * to be substantially less than the maximum amount of time for which an
133 : : * incremental backup will wait for this process to catch up. Otherwise, an
134 : : * incremental backup might time out on an idle system just because we sleep
135 : : * for too long.
136 : : */
137 : : #define MAX_SLEEP_QUANTA 150
138 : : #define MS_PER_SLEEP_QUANTUM 200
139 : :
140 : : /*
141 : : * This is a count of the number of pages of WAL that we've read since the
142 : : * last time we waited for more WAL to appear.
143 : : */
144 : : static long pages_read_since_last_sleep = 0;
145 : :
146 : : /*
147 : : * Most recent RedoRecPtr value observed by MaybeRemoveOldWalSummaries.
148 : : */
149 : : static XLogRecPtr redo_pointer_at_last_summary_removal = InvalidXLogRecPtr;
150 : :
151 : : /*
152 : : * GUC parameters
153 : : */
154 : : bool summarize_wal = false;
155 : : int wal_summary_keep_time = 10 * HOURS_PER_DAY * MINS_PER_HOUR;
156 : :
157 : : static void WalSummarizerShutdown(int code, Datum arg);
158 : : static XLogRecPtr GetLatestLSN(TimeLineID *tli);
159 : : static void ProcessWalSummarizerInterrupts(void);
160 : : static XLogRecPtr SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn,
161 : : bool exact, XLogRecPtr switch_lsn,
162 : : XLogRecPtr maximum_lsn);
163 : : static void SummarizeDbaseRecord(XLogReaderState *xlogreader,
164 : : BlockRefTable *brtab);
165 : : static void SummarizeSmgrRecord(XLogReaderState *xlogreader,
166 : : BlockRefTable *brtab);
167 : : static void SummarizeXactRecord(XLogReaderState *xlogreader,
168 : : BlockRefTable *brtab);
169 : : static bool SummarizeXlogRecord(XLogReaderState *xlogreader,
170 : : bool *new_fast_forward);
171 : : static int summarizer_read_local_xlog_page(XLogReaderState *state,
172 : : XLogRecPtr targetPagePtr,
173 : : int reqLen,
174 : : XLogRecPtr targetRecPtr,
175 : : char *cur_page);
176 : : static void summarizer_wait_for_wal(void);
177 : : static void MaybeRemoveOldWalSummaries(void);
178 : :
179 : : /*
180 : : * Register shared memory space needed by this module.
181 : : */
182 : : static void
110 heikki.linnakangas@i 183 :CBC 1225 : WalSummarizerShmemRequest(void *arg)
184 : : {
185 : 1225 : ShmemRequestStruct(.name = "Wal Summarizer Ctl",
186 : : .size = sizeof(WalSummarizerData),
187 : : .ptr = (void **) &WalSummarizerCtl,
188 : : );
948 rhaas@postgresql.org 189 : 1225 : }
190 : :
191 : : /*
192 : : * Initialize shared memory for this module.
193 : : */
194 : : static void
110 heikki.linnakangas@i 195 : 1222 : WalSummarizerShmemInit(void *arg)
196 : : {
197 : : /*
198 : : * We're just filling in dummy values here -- the real initialization will
199 : : * happen when GetOldestUnsummarizedLSN() is called for the first time.
200 : : */
201 : 1222 : WalSummarizerCtl->initialized = false;
202 : 1222 : WalSummarizerCtl->summarized_tli = 0;
203 : 1222 : WalSummarizerCtl->summarized_lsn = InvalidXLogRecPtr;
204 : 1222 : WalSummarizerCtl->lsn_is_exact = false;
205 : 1222 : WalSummarizerCtl->summarizer_pgprocno = INVALID_PROC_NUMBER;
206 : 1222 : WalSummarizerCtl->pending_lsn = InvalidXLogRecPtr;
207 : 1222 : ConditionVariableInit(&WalSummarizerCtl->summary_file_cv);
948 rhaas@postgresql.org 208 : 1222 : }
209 : :
210 : : /*
211 : : * Entry point for walsummarizer process.
212 : : */
213 : : void
519 peter@eisentraut.org 214 : 5 : WalSummarizerMain(const void *startup_data, size_t startup_data_len)
215 : : {
216 : : sigjmp_buf local_sigjmp_buf;
217 : : MemoryContext context;
218 : :
219 : : /*
220 : : * Within this function, 'current_lsn' and 'current_tli' refer to the
221 : : * point from which the next WAL summary file should start. 'exact' is
222 : : * true if 'current_lsn' is known to be the start of a WAL record or WAL
223 : : * segment, and false if it might be in the middle of a record someplace.
224 : : *
225 : : * 'switch_lsn' and 'switch_tli', if set, are the LSN at which we need to
226 : : * switch to a new timeline and the timeline to which we need to switch.
227 : : * If not set, we either haven't figured out the answers yet or we're
228 : : * already on the latest timeline.
229 : : */
230 : : XLogRecPtr current_lsn;
231 : : TimeLineID current_tli;
232 : : bool exact;
948 rhaas@postgresql.org 233 : 5 : XLogRecPtr switch_lsn = InvalidXLogRecPtr;
234 : 5 : TimeLineID switch_tli = 0;
235 : :
859 heikki.linnakangas@i 236 [ - + ]: 5 : Assert(startup_data_len == 0);
237 : :
238 : 5 : AuxiliaryProcessMainCommon();
239 : :
948 rhaas@postgresql.org 240 [ - + ]: 5 : ereport(DEBUG1,
241 : : (errmsg_internal("WAL summarizer started")));
242 : :
243 : : /*
244 : : * Properly accept or ignore signals the postmaster might send us
245 : : */
246 : 5 : pqsignal(SIGHUP, SignalHandlerForConfigReload);
102 andrew@dunslane.net 247 : 5 : pqsignal(SIGINT, PG_SIG_IGN); /* no query to cancel */
948 rhaas@postgresql.org 248 : 5 : pqsignal(SIGTERM, SignalHandlerForShutdownRequest);
249 : : /* SIGQUIT handler was already set up by InitPostmasterChild */
102 andrew@dunslane.net 250 : 5 : pqsignal(SIGALRM, PG_SIG_IGN);
251 : 5 : pqsignal(SIGPIPE, PG_SIG_IGN);
948 rhaas@postgresql.org 252 : 5 : pqsignal(SIGUSR1, procsignal_sigusr1_handler);
102 andrew@dunslane.net 253 : 5 : pqsignal(SIGUSR2, PG_SIG_IGN); /* not used */
254 : :
255 : : /* Advertise ourselves. */
926 rhaas@postgresql.org 256 : 5 : on_shmem_exit(WalSummarizerShutdown, (Datum) 0);
948 257 : 5 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
884 heikki.linnakangas@i 258 : 5 : WalSummarizerCtl->summarizer_pgprocno = MyProcNumber;
948 rhaas@postgresql.org 259 : 5 : LWLockRelease(WALSummarizerLock);
260 : :
261 : : /* Create and switch to a memory context that we can reset on error. */
262 : 5 : context = AllocSetContextCreate(TopMemoryContext,
263 : : "Wal Summarizer",
264 : : ALLOCSET_DEFAULT_SIZES);
265 : 5 : MemoryContextSwitchTo(context);
266 : :
267 : : /*
268 : : * Reset some signals that are accepted by postmaster but not here
269 : : */
102 andrew@dunslane.net 270 : 5 : pqsignal(SIGCHLD, PG_SIG_DFL);
271 : :
272 : : /*
273 : : * If an exception is encountered, processing resumes here.
274 : : */
948 rhaas@postgresql.org 275 [ - + ]: 5 : if (sigsetjmp(local_sigjmp_buf, 1) != 0)
276 : : {
277 : : /* Since not using PG_TRY, must reset error stack by hand */
948 rhaas@postgresql.org 278 :UBC 0 : error_context_stack = NULL;
279 : :
280 : : /* Prevent interrupts while cleaning up */
281 : 0 : HOLD_INTERRUPTS();
282 : :
283 : : /* Report the error to the server log */
284 : 0 : EmitErrorReport();
285 : :
286 : : /* Release resources we might have acquired. */
287 : 0 : LWLockReleaseAll();
288 : 0 : ConditionVariableCancelSleep();
289 : 0 : pgstat_report_wait_end();
495 andres@anarazel.de 290 : 0 : pgaio_error_cleanup();
948 rhaas@postgresql.org 291 : 0 : ReleaseAuxProcessResources(false);
292 : 0 : AtEOXact_Files(false);
293 : 0 : AtEOXact_HashTables(false);
294 : :
295 : : /*
296 : : * Now return to normal top-level context and clear ErrorContext for
297 : : * next time.
298 : : */
299 : 0 : MemoryContextSwitchTo(context);
300 : 0 : FlushErrorState();
301 : :
302 : : /* Flush any leaked data in the top-level context */
303 : 0 : MemoryContextReset(context);
304 : :
305 : : /* Now we can allow interrupts again */
306 [ # # ]: 0 : RESUME_INTERRUPTS();
307 : :
308 : : /*
309 : : * Sleep for 10 seconds before attempting to resume operations in
310 : : * order to avoid excessive logging.
311 : : *
312 : : * Many of the likely error conditions are things that will repeat
313 : : * every time. For example, if the WAL can't be read or the summary
314 : : * can't be written, only administrator action will cure the problem.
315 : : * So a really fast retry time doesn't seem to be especially
316 : : * beneficial, and it will clutter the logs.
317 : : */
658 heikki.linnakangas@i 318 : 0 : (void) WaitLatch(NULL,
319 : : WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
320 : : 10000,
321 : : WAIT_EVENT_WAL_SUMMARIZER_ERROR);
322 : : }
323 : :
324 : : /* We can now handle ereport(ERROR) */
948 rhaas@postgresql.org 325 :CBC 5 : PG_exception_stack = &local_sigjmp_buf;
326 : :
327 : : /*
328 : : * Unblock signals (they were blocked when the postmaster forked us)
329 : : */
330 : 5 : sigprocmask(SIG_SETMASK, &UnBlockSig, NULL);
331 : :
332 : : /*
333 : : * Fetch information about previous progress from shared memory, and ask
334 : : * GetOldestUnsummarizedLSN to reset pending_lsn to summarized_lsn. We
335 : : * might be recovering from an error, and if so, pending_lsn might have
336 : : * advanced past summarized_lsn, but any WAL we read previously has been
337 : : * lost and will need to be reread.
338 : : *
339 : : * If we discover that WAL summarization is not enabled, just exit.
340 : : */
760 341 : 5 : current_lsn = GetOldestUnsummarizedLSN(¤t_tli, &exact);
261 alvherre@kurilemu.de 342 [ + - ]: 5 : if (!XLogRecPtrIsValid(current_lsn))
948 rhaas@postgresql.org 343 :UBC 0 : proc_exit(0);
344 : :
345 : : /*
346 : : * Loop forever
347 : : */
348 : : for (;;)
948 rhaas@postgresql.org 349 :CBC 28 : {
350 : : XLogRecPtr latest_lsn;
351 : : TimeLineID latest_tli;
352 : : XLogRecPtr maximum_lsn;
353 : : XLogRecPtr end_of_summary_lsn;
354 : :
355 : : /* Flush any leaked data in the top-level context */
356 : 33 : MemoryContextReset(context);
357 : :
358 : : /* Process any signals received recently. */
507 heikki.linnakangas@i 359 : 33 : ProcessWalSummarizerInterrupts();
360 : :
361 : : /* If it's time to remove any old WAL summaries, do that now. */
948 rhaas@postgresql.org 362 : 33 : MaybeRemoveOldWalSummaries();
363 : :
364 : : /* Find the LSN and TLI up to which we can safely summarize. */
365 : 33 : latest_lsn = GetLatestLSN(&latest_tli);
366 : :
367 : : /*
368 : : * If we're summarizing a historic timeline and we haven't yet
369 : : * computed the point at which to switch to the next timeline, do that
370 : : * now.
371 : : *
372 : : * Note that if this is a standby, what was previously the current
373 : : * timeline could become historic at any time.
374 : : *
375 : : * We could try to make this more efficient by caching the results of
376 : : * readTimeLineHistory when latest_tli has not changed, but since we
377 : : * only have to do this once per timeline switch, we probably wouldn't
378 : : * save any significant amount of work in practice.
379 : : */
261 alvherre@kurilemu.de 380 [ - + - - ]: 33 : if (current_tli != latest_tli && !XLogRecPtrIsValid(switch_lsn))
381 : : {
948 rhaas@postgresql.org 382 :UBC 0 : List *tles = readTimeLineHistory(latest_tli);
383 : :
384 : 0 : switch_lsn = tliSwitchPoint(current_tli, tles, &switch_tli);
385 [ # # ]: 0 : ereport(DEBUG1,
386 : : errmsg_internal("switch point from TLI %u to TLI %u is at %X/%08X",
387 : : current_tli, switch_tli, LSN_FORMAT_ARGS(switch_lsn)));
388 : : }
389 : :
390 : : /*
391 : : * If we've reached the switch LSN, we can't summarize anything else
392 : : * on this timeline. Switch to the next timeline and go around again,
393 : : * backing up to the exact switch point if we passed it.
394 : : */
261 alvherre@kurilemu.de 395 [ - + - - ]:CBC 33 : if (XLogRecPtrIsValid(switch_lsn) && current_lsn >= switch_lsn)
396 : : {
397 : : /* Restart summarization from switch point. */
948 rhaas@postgresql.org 398 :UBC 0 : current_tli = switch_tli;
729 399 : 0 : current_lsn = switch_lsn;
400 : :
401 : : /* Next timeline and switch point, if any, not yet known. */
948 402 : 0 : switch_lsn = InvalidXLogRecPtr;
403 : 0 : switch_tli = 0;
404 : :
405 : : /* Update (really, rewind, if needed) state in shared memory. */
729 406 : 0 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
407 : 0 : WalSummarizerCtl->summarized_lsn = current_lsn;
408 : 0 : WalSummarizerCtl->summarized_tli = current_tli;
409 : 0 : WalSummarizerCtl->lsn_is_exact = true;
410 : 0 : WalSummarizerCtl->pending_lsn = current_lsn;
411 : 0 : LWLockRelease(WALSummarizerLock);
412 : :
948 413 : 0 : continue;
414 : : }
415 : :
416 : : /* Summarize WAL. */
3 rhaas@postgresql.org 417 [ - + ]:CBC 33 : maximum_lsn = XLogRecPtrIsValid(switch_lsn) ? switch_lsn : latest_lsn;
948 418 : 33 : end_of_summary_lsn = SummarizeWAL(current_tli,
419 : : current_lsn, exact,
420 : : switch_lsn, maximum_lsn);
261 alvherre@kurilemu.de 421 [ - + ]: 28 : Assert(XLogRecPtrIsValid(end_of_summary_lsn));
948 rhaas@postgresql.org 422 [ - + ]: 28 : Assert(end_of_summary_lsn >= current_lsn);
423 : :
424 : : /*
425 : : * Update state for next loop iteration.
426 : : *
427 : : * Next summary file should start from exactly where this one ended.
428 : : */
429 : 28 : current_lsn = end_of_summary_lsn;
430 : 28 : exact = true;
431 : :
432 : : /* Update state in shared memory. */
433 : 28 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
434 : 28 : WalSummarizerCtl->summarized_lsn = end_of_summary_lsn;
435 : 28 : WalSummarizerCtl->summarized_tli = current_tli;
436 : 28 : WalSummarizerCtl->lsn_is_exact = true;
437 : 28 : WalSummarizerCtl->pending_lsn = end_of_summary_lsn;
438 : 28 : LWLockRelease(WALSummarizerLock);
439 : :
440 : : /* Wake up anyone waiting for more summary files to be written. */
441 : 28 : ConditionVariableBroadcast(&WalSummarizerCtl->summary_file_cv);
442 : : }
443 : : }
444 : :
445 : : /*
446 : : * Get information about the state of the WAL summarizer.
447 : : */
448 : : void
926 rhaas@postgresql.org 449 :UBC 0 : GetWalSummarizerState(TimeLineID *summarized_tli, XLogRecPtr *summarized_lsn,
450 : : XLogRecPtr *pending_lsn, int *summarizer_pid)
451 : : {
452 : 0 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
453 [ # # ]: 0 : if (!WalSummarizerCtl->initialized)
454 : : {
455 : : /*
456 : : * If initialized is false, the rest of the structure contents are
457 : : * undefined.
458 : : */
459 : 0 : *summarized_tli = 0;
460 : 0 : *summarized_lsn = InvalidXLogRecPtr;
461 : 0 : *pending_lsn = InvalidXLogRecPtr;
462 : 0 : *summarizer_pid = -1;
463 : : }
464 : : else
465 : : {
466 : 0 : int summarizer_pgprocno = WalSummarizerCtl->summarizer_pgprocno;
467 : :
468 : 0 : *summarized_tli = WalSummarizerCtl->summarized_tli;
469 : 0 : *summarized_lsn = WalSummarizerCtl->summarized_lsn;
874 heikki.linnakangas@i 470 [ # # ]: 0 : if (summarizer_pgprocno == INVALID_PROC_NUMBER)
471 : : {
472 : : /*
473 : : * If the summarizer has exited, the fact that it had processed
474 : : * beyond summarized_lsn is irrelevant now.
475 : : */
926 rhaas@postgresql.org 476 : 0 : *pending_lsn = WalSummarizerCtl->summarized_lsn;
477 : 0 : *summarizer_pid = -1;
478 : : }
479 : : else
480 : : {
481 : 0 : *pending_lsn = WalSummarizerCtl->pending_lsn;
482 : :
483 : : /*
484 : : * We're not fussed about inexact answers here, since they could
485 : : * become stale instantly, so we don't bother taking the lock, but
486 : : * make sure that invalid PID values are normalized to -1.
487 : : */
488 : 0 : *summarizer_pid = GetPGProcByNumber(summarizer_pgprocno)->pid;
489 [ # # ]: 0 : if (*summarizer_pid <= 0)
490 : 0 : *summarizer_pid = -1;
491 : : }
492 : : }
493 : 0 : LWLockRelease(WALSummarizerLock);
494 : 0 : }
495 : :
496 : : /*
497 : : * Get the oldest LSN in this server's timeline history that has not yet been
498 : : * summarized, and update shared memory state as appropriate.
499 : : *
500 : : * If *tli != NULL, it will be set to the TLI for the LSN that is returned.
501 : : *
502 : : * If *lsn_is_exact != NULL, it will be set to true if the returned LSN is
503 : : * necessarily the start of a WAL record and false if it's just the beginning
504 : : * of a WAL segment.
505 : : */
506 : : XLogRecPtr
760 rhaas@postgresql.org 507 :CBC 2540 : GetOldestUnsummarizedLSN(TimeLineID *tli, bool *lsn_is_exact)
508 : : {
509 : : TimeLineID latest_tli;
510 : : int n;
511 : : List *tles;
941 512 : 2540 : XLogRecPtr unsummarized_lsn = InvalidXLogRecPtr;
948 513 : 2540 : TimeLineID unsummarized_tli = 0;
514 : 2540 : bool should_make_exact = false;
515 : : List *existing_summaries;
516 : : ListCell *lc;
760 517 : 2540 : bool am_wal_summarizer = AmWalSummarizerProcess();
518 : :
519 : : /* If not summarizing WAL, do nothing. */
948 520 [ + + ]: 2540 : if (!summarize_wal)
521 : 2523 : return InvalidXLogRecPtr;
522 : :
523 : : /*
524 : : * If we are not the WAL summarizer process, then we normally just want to
525 : : * read the values from shared memory. However, as an exception, if shared
526 : : * memory hasn't been initialized yet, then we need to do that so that we
527 : : * can read legal values and not remove any WAL too early.
528 : : */
760 529 [ + + ]: 17 : if (!am_wal_summarizer)
530 : : {
531 : 12 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
532 : :
948 533 [ + + ]: 12 : if (WalSummarizerCtl->initialized)
534 : : {
535 : 10 : unsummarized_lsn = WalSummarizerCtl->summarized_lsn;
536 [ - + ]: 10 : if (tli != NULL)
948 rhaas@postgresql.org 537 :UBC 0 : *tli = WalSummarizerCtl->summarized_tli;
948 rhaas@postgresql.org 538 [ - + ]:CBC 10 : if (lsn_is_exact != NULL)
948 rhaas@postgresql.org 539 :UBC 0 : *lsn_is_exact = WalSummarizerCtl->lsn_is_exact;
948 rhaas@postgresql.org 540 :CBC 10 : LWLockRelease(WALSummarizerLock);
541 : 10 : return unsummarized_lsn;
542 : : }
543 : :
544 : 2 : LWLockRelease(WALSummarizerLock);
545 : : }
546 : :
547 : : /*
548 : : * Find the oldest timeline on which WAL still exists, and the earliest
549 : : * segment for which it exists.
550 : : *
551 : : * Note that we do this every time the WAL summarizer process restarts or
552 : : * recovers from an error, in case the contents of pg_wal have changed
553 : : * under us e.g. if some files were removed, either manually - which
554 : : * shouldn't really happen, but might - or by postgres itself, if
555 : : * summarize_wal was turned off and then back on again.
556 : : */
557 : 7 : (void) GetLatestLSN(&latest_tli);
558 : 7 : tles = readTimeLineHistory(latest_tli);
559 [ + - ]: 7 : for (n = list_length(tles) - 1; n >= 0; --n)
560 : : {
561 : 7 : TimeLineHistoryEntry *tle = list_nth(tles, n);
562 : : XLogSegNo oldest_segno;
563 : :
564 : 7 : oldest_segno = XLogGetOldestSegno(tle->tli);
565 [ + - ]: 7 : if (oldest_segno != 0)
566 : : {
567 : : /* Compute oldest LSN that still exists on disk. */
568 : 7 : XLogSegNoOffsetToRecPtr(oldest_segno, 0, wal_segment_size,
569 : : unsummarized_lsn);
570 : :
571 : 7 : unsummarized_tli = tle->tli;
572 : 7 : break;
573 : : }
574 : : }
575 : :
576 : : /*
577 : : * Don't try to summarize anything older than the end LSN of the newest
578 : : * summary file that exists for this timeline.
579 : : */
580 : : existing_summaries =
581 : 7 : GetWalSummaries(unsummarized_tli,
582 : : InvalidXLogRecPtr, InvalidXLogRecPtr);
583 [ - + - - : 7 : foreach(lc, existing_summaries)
- + ]
584 : : {
948 rhaas@postgresql.org 585 :UBC 0 : WalSummaryFile *ws = lfirst(lc);
586 : :
587 [ # # ]: 0 : if (ws->end_lsn > unsummarized_lsn)
588 : : {
589 : 0 : unsummarized_lsn = ws->end_lsn;
590 : 0 : should_make_exact = true;
591 : : }
592 : : }
593 : :
594 : : /* It really should not be possible for us to find no WAL. */
760 rhaas@postgresql.org 595 [ - + ]:CBC 7 : if (unsummarized_tli == 0)
760 rhaas@postgresql.org 596 [ # # ]:UBC 0 : ereport(ERROR,
597 : : errcode(ERRCODE_INTERNAL_ERROR),
598 : : errmsg_internal("no WAL found on timeline %u", latest_tli));
599 : :
600 : : /*
601 : : * If we're the WAL summarizer, we always want to store the values we just
602 : : * computed into shared memory, because those are the values we're going
603 : : * to use to drive our operation, and so they are the authoritative
604 : : * values. Otherwise, we only store values into shared memory if shared
605 : : * memory is uninitialized. Our values are not canonical in such a case,
606 : : * but it's better to have something than nothing, to guide WAL retention.
607 : : */
760 rhaas@postgresql.org 608 :CBC 7 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
757 609 [ + + + - ]: 7 : if (am_wal_summarizer || !WalSummarizerCtl->initialized)
610 : : {
760 611 : 7 : WalSummarizerCtl->initialized = true;
612 : 7 : WalSummarizerCtl->summarized_lsn = unsummarized_lsn;
613 : 7 : WalSummarizerCtl->summarized_tli = unsummarized_tli;
614 : 7 : WalSummarizerCtl->lsn_is_exact = should_make_exact;
615 : 7 : WalSummarizerCtl->pending_lsn = unsummarized_lsn;
616 : : }
617 : : else
760 rhaas@postgresql.org 618 :UBC 0 : unsummarized_lsn = WalSummarizerCtl->summarized_lsn;
619 : :
620 : : /* Also return the to the caller as required. */
948 rhaas@postgresql.org 621 [ + + ]:CBC 7 : if (tli != NULL)
622 : 5 : *tli = WalSummarizerCtl->summarized_tli;
623 [ + + ]: 7 : if (lsn_is_exact != NULL)
624 : 5 : *lsn_is_exact = WalSummarizerCtl->lsn_is_exact;
625 : 7 : LWLockRelease(WALSummarizerLock);
626 : :
627 : 7 : return unsummarized_lsn;
628 : : }
629 : :
630 : : /*
631 : : * Wake up the WAL summarizer process.
632 : : *
633 : : * This might not work, because there's no guarantee that the WAL summarizer
634 : : * process was successfully started, and it also might have started but
635 : : * subsequently terminated. So, under normal circumstances, this will get the
636 : : * latch set, but there's no guarantee.
637 : : */
638 : : void
631 heikki.linnakangas@i 639 : 1711 : WakeupWalSummarizer(void)
640 : : {
641 : : ProcNumber pgprocno;
642 : :
948 rhaas@postgresql.org 643 [ - + ]: 1711 : if (WalSummarizerCtl == NULL)
948 rhaas@postgresql.org 644 :UBC 0 : return;
645 : :
331 msawada@postgresql.o 646 :CBC 1711 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
948 rhaas@postgresql.org 647 : 1711 : pgprocno = WalSummarizerCtl->summarizer_pgprocno;
648 : 1711 : LWLockRelease(WALSummarizerLock);
649 : :
874 heikki.linnakangas@i 650 [ + + ]: 1711 : if (pgprocno != INVALID_PROC_NUMBER)
201 drowley@postgresql.o 651 : 5 : SetLatch(&GetPGProcByNumber(pgprocno)->procLatch);
652 : : }
653 : :
654 : : /*
655 : : * Wait until WAL summarization reaches the given LSN, but time out with an
656 : : * error if the summarizer seems to be stick.
657 : : *
658 : : * Returns immediately if summarize_wal is turned off while we wait. Caller
659 : : * is expected to handle this case, if necessary.
660 : : */
661 : : void
729 rhaas@postgresql.org 662 : 13 : WaitForWalSummarization(XLogRecPtr lsn)
663 : : {
664 : : TimestampTz initial_time,
665 : : cycle_time,
666 : : current_time;
667 : 13 : XLogRecPtr prior_pending_lsn = InvalidXLogRecPtr;
668 : 13 : int deadcycles = 0;
669 : :
670 : 13 : initial_time = cycle_time = GetCurrentTimestamp();
671 : :
672 : : while (1)
948 673 : 12 : {
729 674 : 25 : long timeout_in_ms = 10000;
675 : : XLogRecPtr summarized_lsn;
676 : : XLogRecPtr pending_lsn;
677 : :
678 [ - + ]: 25 : CHECK_FOR_INTERRUPTS();
679 : :
680 : : /* If WAL summarization is disabled while we're waiting, give up. */
681 [ - + ]: 25 : if (!summarize_wal)
729 rhaas@postgresql.org 682 :UBC 0 : return;
683 : :
684 : : /*
685 : : * If the LSN summarized on disk has reached the target value, stop.
686 : : */
331 msawada@postgresql.o 687 :CBC 25 : LWLockAcquire(WALSummarizerLock, LW_SHARED);
948 rhaas@postgresql.org 688 : 25 : summarized_lsn = WalSummarizerCtl->summarized_lsn;
729 689 : 25 : pending_lsn = WalSummarizerCtl->pending_lsn;
948 690 : 25 : LWLockRelease(WALSummarizerLock);
691 : :
692 : : /* If WAL summarization has progressed sufficiently, stop waiting. */
693 [ + + ]: 25 : if (summarized_lsn >= lsn)
694 : 13 : break;
695 : :
696 : : /* Recheck current time. */
729 697 : 12 : current_time = GetCurrentTimestamp();
698 : :
699 : : /* Have we finished the current cycle of waiting? */
700 [ - + ]: 12 : if (TimestampDifferenceMilliseconds(cycle_time,
701 : : current_time) >= timeout_in_ms)
702 : : {
703 : : long elapsed_seconds;
704 : :
705 : : /* Begin new wait cycle. */
729 rhaas@postgresql.org 706 :UBC 0 : cycle_time = TimestampTzPlusMilliseconds(cycle_time,
707 : : timeout_in_ms);
708 : :
709 : : /*
710 : : * Keep track of the number of cycles during which there has been
711 : : * no progression of pending_lsn. If pending_lsn is not advancing,
712 : : * that means that not only are no new files appearing on disk,
713 : : * but we're not even incorporating new records into the in-memory
714 : : * state.
715 : : */
716 [ # # ]: 0 : if (pending_lsn > prior_pending_lsn)
717 : : {
718 : 0 : prior_pending_lsn = pending_lsn;
719 : 0 : deadcycles = 0;
720 : : }
721 : : else
722 : 0 : ++deadcycles;
723 : :
724 : : /*
725 : : * If we've managed to wait for an entire minute without the WAL
726 : : * summarizer absorbing a single WAL record, error out; probably
727 : : * something is wrong.
728 : : *
729 : : * We could consider also erroring out if the summarizer is taking
730 : : * too long to catch up, but it's not clear what rate of progress
731 : : * would be acceptable and what would be too slow. So instead, we
732 : : * just try to error out in the case where there's no progress at
733 : : * all. That seems likely to catch a reasonable number of the
734 : : * things that can go wrong in practice (e.g. the summarizer
735 : : * process is completely hung, say because somebody hooked up a
736 : : * debugger to it or something) without giving up too quickly when
737 : : * the system is just slow.
738 : : */
739 [ # # ]: 0 : if (deadcycles >= 6)
740 [ # # ]: 0 : ereport(ERROR,
741 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
742 : : errmsg("WAL summarization is not progressing"),
743 : : errdetail("Summarization is needed through %X/%08X, but is stuck at %X/%08X on disk and %X/%08X in memory.",
744 : : LSN_FORMAT_ARGS(lsn),
745 : : LSN_FORMAT_ARGS(summarized_lsn),
746 : : LSN_FORMAT_ARGS(pending_lsn))));
747 : :
748 : :
749 : : /*
750 : : * Otherwise, just let the user know what's happening.
751 : : */
752 : 0 : elapsed_seconds =
753 : 0 : TimestampDifferenceMilliseconds(initial_time,
754 : : current_time) / 1000;
755 [ # # ]: 0 : ereport(WARNING,
756 : : (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
757 : : errmsg_plural("still waiting for WAL summarization through %X/%08X after %ld second",
758 : : "still waiting for WAL summarization through %X/%08X after %ld seconds",
759 : : elapsed_seconds,
760 : : LSN_FORMAT_ARGS(lsn),
761 : : elapsed_seconds),
762 : : errdetail("Summarization has reached %X/%08X on disk and %X/%08X in memory.",
763 : : LSN_FORMAT_ARGS(summarized_lsn),
764 : : LSN_FORMAT_ARGS(pending_lsn))));
765 : : }
766 : :
767 : : /*
768 : : * Align the wait time to prevent drift. This doesn't really matter,
769 : : * but we'd like the warnings about how long we've been waiting to say
770 : : * 10 seconds, 20 seconds, 30 seconds, 40 seconds ... without ever
771 : : * drifting to something that is not a multiple of ten.
772 : : */
729 rhaas@postgresql.org 773 :CBC 12 : timeout_in_ms -=
774 : 12 : TimestampDifferenceMilliseconds(cycle_time, current_time);
775 : :
776 : : /* Wait and see. */
948 777 : 12 : ConditionVariableTimedSleep(&WalSummarizerCtl->summary_file_cv,
778 : : timeout_in_ms,
779 : : WAIT_EVENT_WAL_SUMMARY_READY);
780 : : }
781 : :
738 782 : 13 : ConditionVariableCancelSleep();
783 : : }
784 : :
785 : : /*
786 : : * On exit, update shared memory to make it clear that we're no longer
787 : : * running.
788 : : */
789 : : static void
926 790 : 5 : WalSummarizerShutdown(int code, Datum arg)
791 : : {
792 : 5 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
874 heikki.linnakangas@i 793 : 5 : WalSummarizerCtl->summarizer_pgprocno = INVALID_PROC_NUMBER;
926 rhaas@postgresql.org 794 : 5 : LWLockRelease(WALSummarizerLock);
795 : 5 : }
796 : :
797 : : /*
798 : : * Get the latest LSN that is eligible to be summarized, and set *tli to the
799 : : * corresponding timeline.
800 : : */
801 : : static XLogRecPtr
948 802 : 80 : GetLatestLSN(TimeLineID *tli)
803 : : {
804 [ + + ]: 80 : if (!RecoveryInProgress())
805 : : {
806 : : /* Don't summarize WAL before it's flushed. */
807 : 78 : return GetFlushRecPtr(tli);
808 : : }
809 : : else
810 : : {
811 : : XLogRecPtr flush_lsn;
812 : : TimeLineID flush_tli;
813 : : XLogRecPtr replay_lsn;
814 : : TimeLineID replay_tli;
815 : : TimeLineID insert_tli;
816 : :
817 : : /*
818 : : * After the insert TLI has been set and before the control file has
819 : : * been updated to show the DB in production, RecoveryInProgress()
820 : : * will return true, because it's not yet safe for all backends to
821 : : * begin writing WAL. However, replay has already ceased, so from our
822 : : * point of view, recovery is already over. We should summarize up to
823 : : * where replay stopped and then prepare to resume at the start of the
824 : : * insert timeline.
825 : : */
729 826 [ + - ]: 2 : if ((insert_tli = GetWALInsertionTimeLineIfSet()) != 0)
827 : : {
828 : 2 : *tli = insert_tli;
829 : 2 : return GetXLogReplayRecPtr(NULL);
830 : : }
831 : :
832 : : /*
833 : : * What we really want to know is how much WAL has been flushed to
834 : : * disk, but the only flush position available is the one provided by
835 : : * the walreceiver, which may not be running, because this could be
836 : : * crash recovery or recovery via restore_command. So use either the
837 : : * WAL receiver's flush position or the replay position, whichever is
838 : : * further ahead, on the theory that if the WAL has been replayed then
839 : : * it must also have been flushed to disk.
840 : : */
948 rhaas@postgresql.org 841 :UBC 0 : flush_lsn = GetWalRcvFlushRecPtr(NULL, &flush_tli);
842 : 0 : replay_lsn = GetXLogReplayRecPtr(&replay_tli);
843 [ # # ]: 0 : if (flush_lsn > replay_lsn)
844 : : {
845 : 0 : *tli = flush_tli;
846 : 0 : return flush_lsn;
847 : : }
848 : : else
849 : : {
850 : 0 : *tli = replay_tli;
851 : 0 : return replay_lsn;
852 : : }
853 : : }
854 : : }
855 : :
856 : : /*
857 : : * Interrupt handler for main loop of WAL summarizer process.
858 : : */
859 : : static void
507 heikki.linnakangas@i 860 :CBC 80846 : ProcessWalSummarizerInterrupts(void)
861 : : {
948 rhaas@postgresql.org 862 [ - + ]: 80846 : if (ProcSignalBarrierPending)
948 rhaas@postgresql.org 863 :UBC 0 : ProcessProcSignalBarrier();
864 : :
948 rhaas@postgresql.org 865 [ - + ]:CBC 80846 : if (ConfigReloadPending)
866 : : {
948 rhaas@postgresql.org 867 :UBC 0 : ConfigReloadPending = false;
868 : 0 : ProcessConfigFile(PGC_SIGHUP);
869 : : }
870 : :
948 rhaas@postgresql.org 871 [ + + - + ]:CBC 80846 : if (ShutdownRequestPending || !summarize_wal)
872 : : {
873 [ - + ]: 5 : ereport(DEBUG1,
874 : : errmsg_internal("WAL summarizer shutting down"));
875 : 5 : proc_exit(0);
876 : : }
877 : :
878 : : /* Perform logging of memory contexts of this process */
879 [ - + ]: 80841 : if (LogMemoryContextPending)
948 rhaas@postgresql.org 880 :UBC 0 : ProcessLogMemoryContextInterrupt();
948 rhaas@postgresql.org 881 :CBC 80841 : }
882 : :
883 : : /*
884 : : * Summarize a range of WAL records on a single timeline.
885 : : *
886 : : * 'tli' is the timeline to be summarized.
887 : : *
888 : : * 'start_lsn' is the point at which we should start summarizing. If this
889 : : * value comes from the end LSN of the previous record as returned by the
890 : : * xlogreader machinery, 'exact' should be true; otherwise, 'exact' should
891 : : * be false, and this function will search forward for the start of a valid
892 : : * WAL record.
893 : : *
894 : : * 'switch_lsn' is the point at which we should switch to a later timeline,
895 : : * if we're summarizing a historic timeline.
896 : : *
897 : : * 'maximum_lsn' identifies the point beyond which we can't count on being
898 : : * able to read any more WAL. It should be the switch point when reading a
899 : : * historic timeline, or the most-recently-measured end of WAL when reading
900 : : * the current timeline.
901 : : *
902 : : * The return value is the LSN at which the WAL summary actually ends. Most
903 : : * often, a summary file ends because we notice that a checkpoint has
904 : : * occurred and reach the redo pointer of that checkpoint, but sometimes
905 : : * we stop for other reasons, such as a timeline switch.
906 : : */
907 : : static XLogRecPtr
908 : 33 : SummarizeWAL(TimeLineID tli, XLogRecPtr start_lsn, bool exact,
909 : : XLogRecPtr switch_lsn, XLogRecPtr maximum_lsn)
910 : : {
911 : : SummarizerReadLocalXLogPrivate *private_data;
912 : : XLogReaderState *xlogreader;
913 : : XLogRecPtr summary_start_lsn;
914 : 33 : XLogRecPtr summary_end_lsn = switch_lsn;
915 : : char temp_path[MAXPGPATH];
916 : : char final_path[MAXPGPATH];
917 : : WalSummaryIO io;
918 : 33 : BlockRefTable *brtab = CreateEmptyBlockRefTable();
737 919 : 33 : bool fast_forward = true;
920 : : char *errormsg;
921 : :
922 : : /* Initialize private data for xlogreader. */
227 michael@paquier.xyz 923 : 33 : private_data = palloc0_object(SummarizerReadLocalXLogPrivate);
948 rhaas@postgresql.org 924 : 33 : private_data->tli = tli;
261 alvherre@kurilemu.de 925 : 33 : private_data->historic = XLogRecPtrIsValid(switch_lsn);
948 rhaas@postgresql.org 926 : 33 : private_data->read_upto = maximum_lsn;
927 : :
928 : : /* Create xlogreader. */
929 : 33 : xlogreader = XLogReaderAllocate(wal_segment_size, NULL,
930 : 33 : XL_ROUTINE(.page_read = &summarizer_read_local_xlog_page,
931 : : .segment_open = &wal_segment_open,
932 : : .segment_close = &wal_segment_close),
933 : : private_data);
934 [ - + ]: 33 : if (xlogreader == NULL)
948 rhaas@postgresql.org 935 [ # # ]:UBC 0 : ereport(ERROR,
936 : : (errcode(ERRCODE_OUT_OF_MEMORY),
937 : : errmsg("out of memory"),
938 : : errdetail("Failed while allocating a WAL reading processor.")));
939 : :
940 : : /*
941 : : * When exact = false, we're starting from an arbitrary point in the WAL
942 : : * and must search forward for the start of the next record.
943 : : *
944 : : * When exact = true, start_lsn should be either the LSN where a record
945 : : * begins, or the LSN of a page where the page header is immediately
946 : : * followed by the start of a new record. XLogBeginRead should tolerate
947 : : * either case.
948 : : *
949 : : * We need to allow for both cases because the behavior of xlogreader
950 : : * varies. When a record spans two or more xlog pages, the ending LSN
951 : : * reported by xlogreader will be the starting LSN of the following
952 : : * record, but when an xlog page boundary falls between two records, the
953 : : * end LSN for the first will be reported as the first byte of the
954 : : * following page. We can't know until we read that page how large the
955 : : * header will be, but we'll have to skip over it to find the next record.
956 : : */
948 rhaas@postgresql.org 957 [ + + ]:CBC 33 : if (exact)
958 : : {
959 : : /*
960 : : * Even if start_lsn is the beginning of a page rather than the
961 : : * beginning of the first record on that page, we should still use it
962 : : * as the start LSN for the summary file. That's because we detect
963 : : * missing summary files by looking for cases where the end LSN of one
964 : : * file is less than the start LSN of the next file. When only a page
965 : : * header is skipped, nothing has been missed.
966 : : */
967 : 28 : XLogBeginRead(xlogreader, start_lsn);
968 : 28 : summary_start_lsn = start_lsn;
969 : : }
970 : : else
971 : : {
123 fujii@postgresql.org 972 : 5 : summary_start_lsn = XLogFindNextRecord(xlogreader, start_lsn, &errormsg);
261 alvherre@kurilemu.de 973 [ - + ]: 5 : if (!XLogRecPtrIsValid(summary_start_lsn))
974 : : {
975 : : /*
976 : : * If we hit end-of-WAL while trying to find the next valid
977 : : * record, we must be on a historic timeline that has no valid
978 : : * records that begin after start_lsn and before end of WAL.
979 : : */
948 rhaas@postgresql.org 980 [ # # ]:UBC 0 : if (private_data->end_of_wal)
981 : : {
982 [ # # ]: 0 : ereport(DEBUG1,
983 : : errmsg_internal("could not read WAL from timeline %u at %X/%08X: end of WAL at %X/%08X",
984 : : tli,
985 : : LSN_FORMAT_ARGS(start_lsn),
986 : : LSN_FORMAT_ARGS(private_data->read_upto)));
987 : :
988 : : /*
989 : : * The timeline ends at or after start_lsn, without containing
990 : : * any records. Thus, we must make sure the main loop does not
991 : : * iterate. If start_lsn is the end of the timeline, then we
992 : : * won't actually emit an empty summary file, but otherwise,
993 : : * we must, to capture the fact that the LSN range in question
994 : : * contains no interesting WAL records.
995 : : */
996 : 0 : summary_start_lsn = start_lsn;
997 : 0 : summary_end_lsn = private_data->read_upto;
998 : 0 : switch_lsn = xlogreader->EndRecPtr;
999 : : }
1000 : : else
1001 : : {
123 fujii@postgresql.org 1002 [ # # ]: 0 : if (errormsg)
1003 [ # # ]: 0 : ereport(ERROR,
1004 : : errmsg("could not find a valid record after %X/%08X: %s",
1005 : : LSN_FORMAT_ARGS(start_lsn), errormsg));
1006 : : else
1007 [ # # ]: 0 : ereport(ERROR,
1008 : : errmsg("could not find a valid record after %X/%08X",
1009 : : LSN_FORMAT_ARGS(start_lsn)));
1010 : : }
1011 : : }
1012 : :
1013 : : /* We shouldn't go backward. */
948 rhaas@postgresql.org 1014 [ + - ]:CBC 5 : Assert(summary_start_lsn >= start_lsn);
1015 : : }
1016 : :
1017 : : /*
1018 : : * Main loop: read xlog records one by one.
1019 : : */
1020 : : while (1)
1021 : 77587 : {
1022 : : int block_id;
1023 : : XLogRecord *record;
1024 : : uint8 rmid;
1025 : :
507 heikki.linnakangas@i 1026 : 77620 : ProcessWalSummarizerInterrupts();
1027 : :
1028 : : /* We shouldn't go backward. */
948 rhaas@postgresql.org 1029 [ - + ]: 77618 : Assert(summary_start_lsn <= xlogreader->EndRecPtr);
1030 : :
1031 : : /* Now read the next record. */
1032 : 77618 : record = XLogReadRecord(xlogreader, &errormsg);
1033 [ - + ]: 77615 : if (record == NULL)
1034 : : {
948 rhaas@postgresql.org 1035 [ # # ]:UBC 0 : if (private_data->end_of_wal)
1036 : : {
1037 : : /*
1038 : : * This timeline must be historic and must end before we were
1039 : : * able to read a complete record.
1040 : : */
1041 [ # # ]: 0 : ereport(DEBUG1,
1042 : : errmsg_internal("could not read WAL from timeline %u at %X/%08X: end of WAL at %X/%08X",
1043 : : tli,
1044 : : LSN_FORMAT_ARGS(xlogreader->EndRecPtr),
1045 : : LSN_FORMAT_ARGS(private_data->read_upto)));
1046 : : /* Summary ends at end of WAL. */
1047 : 0 : summary_end_lsn = private_data->read_upto;
1048 : 0 : break;
1049 : : }
1050 [ # # ]: 0 : if (errormsg)
1051 [ # # ]: 0 : ereport(ERROR,
1052 : : (errcode_for_file_access(),
1053 : : errmsg("could not read WAL from timeline %u at %X/%08X: %s",
1054 : : tli, LSN_FORMAT_ARGS(xlogreader->EndRecPtr),
1055 : : errormsg)));
1056 : : else
1057 [ # # ]: 0 : ereport(ERROR,
1058 : : (errcode_for_file_access(),
1059 : : errmsg("could not read WAL from timeline %u at %X/%08X",
1060 : : tli, LSN_FORMAT_ARGS(xlogreader->EndRecPtr))));
1061 : : }
1062 : :
1063 : : /* We shouldn't go backward. */
948 rhaas@postgresql.org 1064 [ - + ]:CBC 77615 : Assert(summary_start_lsn <= xlogreader->EndRecPtr);
1065 : :
261 alvherre@kurilemu.de 1066 [ - + ]: 77615 : if (XLogRecPtrIsValid(switch_lsn) &&
948 rhaas@postgresql.org 1067 [ # # ]:UBC 0 : xlogreader->ReadRecPtr >= switch_lsn)
1068 : : {
1069 : : /*
1070 : : * Whoops! We've read a record that *starts* after the switch LSN,
1071 : : * contrary to our goal of reading only until we hit the first
1072 : : * record that ends at or after the switch LSN. Pretend we didn't
1073 : : * read it after all by bailing out of this loop right here,
1074 : : * before we do anything with this record.
1075 : : *
1076 : : * This can happen because the last record before the switch LSN
1077 : : * might be continued across multiple pages, and then we might
1078 : : * come to a page with XLP_FIRST_IS_OVERWRITE_CONTRECORD set. In
1079 : : * that case, the record that was continued across multiple pages
1080 : : * is incomplete and will be disregarded, and the read will
1081 : : * restart from the beginning of the page that is flagged
1082 : : * XLP_FIRST_IS_OVERWRITE_CONTRECORD.
1083 : : *
1084 : : * If this case occurs, we can fairly say that the current summary
1085 : : * file ends at the switch LSN exactly. The first record on the
1086 : : * page marked XLP_FIRST_IS_OVERWRITE_CONTRECORD will be
1087 : : * discovered when generating the next summary file.
1088 : : */
1089 : 0 : summary_end_lsn = switch_lsn;
1090 : 0 : break;
1091 : : }
1092 : :
1093 : : /*
1094 : : * Certain types of records require special handling. Redo points and
1095 : : * shutdown checkpoints trigger creation of new summary files and can
1096 : : * also cause us to enter or exit "fast forward" mode. Other types of
1097 : : * records can require special updates to the block reference table.
1098 : : */
737 rhaas@postgresql.org 1099 :CBC 77615 : rmid = XLogRecGetRmid(xlogreader);
1100 [ + + ]: 77615 : if (rmid == RM_XLOG_ID)
1101 : : {
1102 : : bool new_fast_forward;
1103 : :
1104 : : /*
1105 : : * If we've already processed some WAL records when we hit a redo
1106 : : * point or shutdown checkpoint, then we stop summarization before
1107 : : * including this record in the current file, so that it will be
1108 : : * the first record in the next file.
1109 : : *
1110 : : * When we hit one of those record types as the first record in a
1111 : : * file, we adjust our notion of whether we're fast-forwarding.
1112 : : * Any WAL generated with wal_level=minimal must be skipped
1113 : : * without actually generating any summary file, because an
1114 : : * incremental backup that crosses such WAL would be unsafe.
1115 : : */
1116 [ + + ]: 1107 : if (SummarizeXlogRecord(xlogreader, &new_fast_forward))
1117 : : {
1118 [ + + ]: 61 : if (xlogreader->ReadRecPtr > summary_start_lsn)
1119 : : {
1120 : 28 : summary_end_lsn = xlogreader->ReadRecPtr;
1121 : 28 : break;
1122 : : }
1123 : : else
1124 : 33 : fast_forward = new_fast_forward;
1125 : : }
1126 : : }
1127 [ + - ]: 76508 : else if (!fast_forward)
1128 : : {
1129 : : /*
1130 : : * This switch handles record types that require extra updates to
1131 : : * the contents of the block reference table.
1132 : : */
1133 [ + + + + ]: 76508 : switch (rmid)
1134 : : {
1135 : 6 : case RM_DBASE_ID:
1136 : 6 : SummarizeDbaseRecord(xlogreader, brtab);
1137 : 6 : break;
1138 : 48 : case RM_SMGR_ID:
1139 : 48 : SummarizeSmgrRecord(xlogreader, brtab);
1140 : 48 : break;
1141 : 2098 : case RM_XACT_ID:
1142 : 2098 : SummarizeXactRecord(xlogreader, brtab);
1143 : 2098 : break;
1144 : : }
1145 : : }
1146 : :
1147 : : /*
1148 : : * If we're in fast-forward mode, we don't really need to do anything.
1149 : : * Otherwise, feed block references from xlog record to block
1150 : : * reference table.
1151 : : */
1152 [ + - ]: 77587 : if (!fast_forward)
1153 : : {
1154 [ + + ]: 154704 : for (block_id = 0; block_id <= XLogRecMaxBlockId(xlogreader);
1155 : 77117 : block_id++)
1156 : : {
1157 : : RelFileLocator rlocator;
1158 : : ForkNumber forknum;
1159 : : BlockNumber blocknum;
1160 : :
1161 [ + + ]: 77117 : if (!XLogRecGetBlockTagExtended(xlogreader, block_id, &rlocator,
1162 : : &forknum, &blocknum, NULL))
1163 : 50 : continue;
1164 : :
1165 : : /*
1166 : : * As we do elsewhere, ignore the FSM fork, because it's not
1167 : : * fully WAL-logged.
1168 : : */
1169 [ + + ]: 77067 : if (forknum != FSM_FORKNUM)
1170 : 76620 : BlockRefTableMarkBlockModified(brtab, &rlocator, forknum,
1171 : : blocknum);
1172 : : }
1173 : : }
1174 : :
1175 : : /* Update our notion of where this summary file ends. */
948 1176 : 77587 : summary_end_lsn = xlogreader->EndRecPtr;
1177 : :
1178 : : /* Also update shared memory. */
1179 : 77587 : LWLockAcquire(WALSummarizerLock, LW_EXCLUSIVE);
1180 [ - + ]: 77587 : Assert(summary_end_lsn >= WalSummarizerCtl->summarized_lsn);
1181 : 77587 : WalSummarizerCtl->pending_lsn = summary_end_lsn;
1182 : 77587 : LWLockRelease(WALSummarizerLock);
1183 : :
1184 : : /*
1185 : : * If we have a switch LSN and have reached it, stop before reading
1186 : : * the next record.
1187 : : */
261 alvherre@kurilemu.de 1188 [ - + ]: 77587 : if (XLogRecPtrIsValid(switch_lsn) &&
948 rhaas@postgresql.org 1189 [ # # ]:UBC 0 : xlogreader->EndRecPtr >= switch_lsn)
1190 : 0 : break;
1191 : : }
1192 : :
1193 : : /* Destroy xlogreader. */
948 rhaas@postgresql.org 1194 :CBC 28 : pfree(xlogreader->private_data);
1195 : 28 : XLogReaderFree(xlogreader);
1196 : :
1197 : : /*
1198 : : * If a timeline switch occurs, we may fail to make any progress at all
1199 : : * before exiting the loop above. If that happens, we don't write a WAL
1200 : : * summary file at all. We can also skip writing a file if we're in
1201 : : * fast-forward mode.
1202 : : */
737 1203 [ + - + - ]: 28 : if (summary_end_lsn > summary_start_lsn && !fast_forward)
1204 : : {
1205 : : /* Generate temporary and final path name. */
948 1206 : 28 : snprintf(temp_path, MAXPGPATH,
1207 : : XLOGDIR "/summaries/temp.summary");
1208 : 28 : snprintf(final_path, MAXPGPATH,
1209 : : XLOGDIR "/summaries/%08X%08X%08X%08X%08X.summary",
1210 : : tli,
1211 : 28 : LSN_FORMAT_ARGS(summary_start_lsn),
1212 : 28 : LSN_FORMAT_ARGS(summary_end_lsn));
1213 : :
1214 : : /* Open the temporary file for writing. */
1215 : 28 : io.filepos = 0;
1216 : 28 : io.file = PathNameOpenFile(temp_path, O_WRONLY | O_CREAT | O_TRUNC);
1217 [ - + ]: 28 : if (io.file < 0)
948 rhaas@postgresql.org 1218 [ # # ]:UBC 0 : ereport(ERROR,
1219 : : (errcode_for_file_access(),
1220 : : errmsg("could not create file \"%s\": %m", temp_path)));
1221 : :
1222 : : /* Write the data. */
948 rhaas@postgresql.org 1223 :CBC 28 : WriteBlockRefTable(brtab, WriteWalSummary, &io);
1224 : :
1225 : : /* Close temporary file and shut down xlogreader. */
1226 : 28 : FileClose(io.file);
1227 : :
1228 : : /* Tell the user what we did. */
1229 [ - + ]: 28 : ereport(DEBUG1,
1230 : : errmsg_internal("summarized WAL on TLI %u from %X/%08X to %X/%08X",
1231 : : tli,
1232 : : LSN_FORMAT_ARGS(summary_start_lsn),
1233 : : LSN_FORMAT_ARGS(summary_end_lsn)));
1234 : :
1235 : : /* Durably rename the new summary into place. */
1236 : 28 : durable_rename(temp_path, final_path, ERROR);
1237 : : }
1238 : :
1239 : : /* If we skipped a non-zero amount of WAL, log a debug message. */
737 1240 [ + - - + ]: 28 : if (summary_end_lsn > summary_start_lsn && fast_forward)
737 rhaas@postgresql.org 1241 [ # # ]:UBC 0 : ereport(DEBUG1,
1242 : : errmsg_internal("skipped summarizing WAL on TLI %u from %X/%08X to %X/%08X",
1243 : : tli,
1244 : : LSN_FORMAT_ARGS(summary_start_lsn),
1245 : : LSN_FORMAT_ARGS(summary_end_lsn)));
1246 : :
948 rhaas@postgresql.org 1247 :CBC 28 : return summary_end_lsn;
1248 : : }
1249 : :
1250 : : /*
1251 : : * Special handling for WAL records with RM_DBASE_ID.
1252 : : */
1253 : : static void
873 1254 : 6 : SummarizeDbaseRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
1255 : : {
1256 : 6 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1257 : :
1258 : : /*
1259 : : * We use relfilenode zero for a given database OID and tablespace OID to
1260 : : * indicate that all relations with that pair of IDs have been recreated
1261 : : * if they exist at all. Effectively, we're setting a limit block of 0 for
1262 : : * all such relfilenodes.
1263 : : *
1264 : : * Technically, this special handling is only needed in the case of
1265 : : * XLOG_DBASE_CREATE_FILE_COPY, because that can create a whole bunch of
1266 : : * relation files in a directory without logging anything specific to each
1267 : : * one. If we didn't mark the whole DB OID/TS OID combination in some way,
1268 : : * then a tablespace that was dropped after the reference backup and
1269 : : * recreated using the FILE_COPY method prior to the incremental backup
1270 : : * would look just like one that was never touched at all, which would be
1271 : : * catastrophic.
1272 : : *
1273 : : * But it seems best to adopt this treatment for all records that drop or
1274 : : * create a DB OID/TS OID combination. That's similar to how we treat the
1275 : : * limit block for individual relations, and it's an extra layer of safety
1276 : : * here. We can never lose data by marking more stuff as needing to be
1277 : : * backed up in full.
1278 : : */
1279 [ + - ]: 6 : if (info == XLOG_DBASE_CREATE_FILE_COPY)
1280 : : {
1281 : : xl_dbase_create_file_copy_rec *xlrec;
1282 : : RelFileLocator rlocator;
1283 : :
1284 : 6 : xlrec =
1285 : 6 : (xl_dbase_create_file_copy_rec *) XLogRecGetData(xlogreader);
1286 : 6 : rlocator.spcOid = xlrec->tablespace_id;
1287 : 6 : rlocator.dbOid = xlrec->db_id;
1288 : 6 : rlocator.relNumber = 0;
1289 : 6 : BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1290 : : }
873 rhaas@postgresql.org 1291 [ # # ]:UBC 0 : else if (info == XLOG_DBASE_CREATE_WAL_LOG)
1292 : : {
1293 : : xl_dbase_create_wal_log_rec *xlrec;
1294 : : RelFileLocator rlocator;
1295 : :
1296 : 0 : xlrec = (xl_dbase_create_wal_log_rec *) XLogRecGetData(xlogreader);
1297 : 0 : rlocator.spcOid = xlrec->tablespace_id;
1298 : 0 : rlocator.dbOid = xlrec->db_id;
1299 : 0 : rlocator.relNumber = 0;
1300 : 0 : BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1301 : : }
1302 [ # # ]: 0 : else if (info == XLOG_DBASE_DROP)
1303 : : {
1304 : : xl_dbase_drop_rec *xlrec;
1305 : : RelFileLocator rlocator;
1306 : : int i;
1307 : :
1308 : 0 : xlrec = (xl_dbase_drop_rec *) XLogRecGetData(xlogreader);
1309 : 0 : rlocator.dbOid = xlrec->db_id;
1310 : 0 : rlocator.relNumber = 0;
1311 [ # # ]: 0 : for (i = 0; i < xlrec->ntablespaces; ++i)
1312 : : {
1313 : 0 : rlocator.spcOid = xlrec->tablespace_ids[i];
1314 : 0 : BlockRefTableSetLimitBlock(brtab, &rlocator, MAIN_FORKNUM, 0);
1315 : : }
1316 : : }
873 rhaas@postgresql.org 1317 :CBC 6 : }
1318 : :
1319 : : /*
1320 : : * Special handling for WAL records with RM_SMGR_ID.
1321 : : */
1322 : : static void
948 1323 : 48 : SummarizeSmgrRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
1324 : : {
1325 : 48 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1326 : :
1327 [ + + ]: 48 : if (info == XLOG_SMGR_CREATE)
1328 : : {
1329 : : xl_smgr_create *xlrec;
1330 : :
1331 : : /*
1332 : : * If a new relation fork is created on disk, there is no point
1333 : : * tracking anything about which blocks have been modified, because
1334 : : * the whole thing will be new. Hence, set the limit block for this
1335 : : * fork to 0.
1336 : : *
1337 : : * Ignore the FSM fork, which is not fully WAL-logged.
1338 : : */
1339 : 47 : xlrec = (xl_smgr_create *) XLogRecGetData(xlogreader);
1340 : :
1341 [ + - ]: 47 : if (xlrec->forkNum != FSM_FORKNUM)
1342 : 47 : BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1343 : : xlrec->forkNum, 0);
1344 : : }
1345 [ + - ]: 1 : else if (info == XLOG_SMGR_TRUNCATE)
1346 : : {
1347 : : xl_smgr_truncate *xlrec;
1348 : :
1349 : 1 : xlrec = (xl_smgr_truncate *) XLogRecGetData(xlogreader);
1350 : :
1351 : : /*
1352 : : * If a relation fork is truncated on disk, there is no point in
1353 : : * tracking anything about block modifications beyond the truncation
1354 : : * point.
1355 : : *
1356 : : * We ignore SMGR_TRUNCATE_FSM here because the FSM isn't fully
1357 : : * WAL-logged and thus we can't track modified blocks for it anyway.
1358 : : */
1359 [ + - ]: 1 : if ((xlrec->flags & SMGR_TRUNCATE_HEAP) != 0)
1360 : 1 : BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1361 : : MAIN_FORKNUM, xlrec->blkno);
1362 [ + - ]: 1 : if ((xlrec->flags & SMGR_TRUNCATE_VM) != 0)
1363 : 1 : BlockRefTableSetLimitBlock(brtab, &xlrec->rlocator,
1364 : : VISIBILITYMAP_FORKNUM,
1365 : : visibilitymap_truncation_length(xlrec->blkno));
1366 : : }
1367 : 48 : }
1368 : :
1369 : : /*
1370 : : * Special handling for WAL records with RM_XACT_ID.
1371 : : */
1372 : : static void
1373 : 2098 : SummarizeXactRecord(XLogReaderState *xlogreader, BlockRefTable *brtab)
1374 : : {
1375 : 2098 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1376 : 2098 : uint8 xact_info = info & XLOG_XACT_OPMASK;
1377 : :
1378 [ - + - - ]: 2098 : if (xact_info == XLOG_XACT_COMMIT ||
1379 : : xact_info == XLOG_XACT_COMMIT_PREPARED)
1380 : 2098 : {
1381 : 2098 : xl_xact_commit *xlrec = (xl_xact_commit *) XLogRecGetData(xlogreader);
1382 : : xl_xact_parsed_commit parsed;
1383 : : int i;
1384 : :
1385 : : /*
1386 : : * Don't track modified blocks for any relations that were removed on
1387 : : * commit.
1388 : : */
1389 : 2098 : ParseCommitRecord(XLogRecGetInfo(xlogreader), xlrec, &parsed);
1390 [ - + ]: 2098 : for (i = 0; i < parsed.nrels; ++i)
1391 : : {
1392 : : ForkNumber forknum;
1393 : :
948 rhaas@postgresql.org 1394 [ # # ]:UBC 0 : for (forknum = 0; forknum <= MAX_FORKNUM; ++forknum)
1395 [ # # ]: 0 : if (forknum != FSM_FORKNUM)
1396 : 0 : BlockRefTableSetLimitBlock(brtab, &parsed.xlocators[i],
1397 : : forknum, 0);
1398 : : }
1399 : : }
1400 [ # # # # ]: 0 : else if (xact_info == XLOG_XACT_ABORT ||
1401 : : xact_info == XLOG_XACT_ABORT_PREPARED)
1402 : : {
1403 : 0 : xl_xact_abort *xlrec = (xl_xact_abort *) XLogRecGetData(xlogreader);
1404 : : xl_xact_parsed_abort parsed;
1405 : : int i;
1406 : :
1407 : : /*
1408 : : * Don't track modified blocks for any relations that were removed on
1409 : : * abort.
1410 : : */
1411 : 0 : ParseAbortRecord(XLogRecGetInfo(xlogreader), xlrec, &parsed);
1412 [ # # ]: 0 : for (i = 0; i < parsed.nrels; ++i)
1413 : : {
1414 : : ForkNumber forknum;
1415 : :
1416 [ # # ]: 0 : for (forknum = 0; forknum <= MAX_FORKNUM; ++forknum)
1417 [ # # ]: 0 : if (forknum != FSM_FORKNUM)
1418 : 0 : BlockRefTableSetLimitBlock(brtab, &parsed.xlocators[i],
1419 : : forknum, 0);
1420 : : }
1421 : : }
948 rhaas@postgresql.org 1422 :CBC 2098 : }
1423 : :
1424 : : /*
1425 : : * Special handling for WAL records with RM_XLOG_ID.
1426 : : *
1427 : : * The return value is true if WAL summarization should stop before this
1428 : : * record and false otherwise. When the return value is true,
1429 : : * *new_fast_forward indicates whether future processing should be done
1430 : : * in fast forward mode (i.e. read WAL without emitting summaries) or not.
1431 : : */
1432 : : static bool
737 1433 : 1107 : SummarizeXlogRecord(XLogReaderState *xlogreader, bool *new_fast_forward)
1434 : : {
948 1435 : 1107 : uint8 info = XLogRecGetInfo(xlogreader) & ~XLR_INFO_MASK;
1436 : : int record_wal_level;
1437 : :
737 1438 [ + + ]: 1107 : if (info == XLOG_CHECKPOINT_REDO)
1439 : : {
1440 : : xl_checkpoint_redo xlrec;
1441 : :
1442 : : /* Payload is wal_level at the time record was written. */
116 dgustafsson@postgres 1443 : 36 : memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_checkpoint_redo));
1444 : 36 : record_wal_level = xlrec.wal_level;
1445 : : }
737 rhaas@postgresql.org 1446 [ + + ]: 1071 : else if (info == XLOG_CHECKPOINT_SHUTDOWN)
1447 : : {
1448 : : CheckPoint rec_ckpt;
1449 : :
1450 : : /* Extract wal_level at time record was written from payload. */
1451 : 19 : memcpy(&rec_ckpt, XLogRecGetData(xlogreader), sizeof(CheckPoint));
1452 : 19 : record_wal_level = rec_ckpt.wal_level;
1453 : : }
1454 [ + + ]: 1052 : else if (info == XLOG_PARAMETER_CHANGE)
1455 : : {
1456 : : xl_parameter_change xlrec;
1457 : :
1458 : : /* Extract wal_level at time record was written from payload. */
1459 : 6 : memcpy(&xlrec, XLogRecGetData(xlogreader),
1460 : : sizeof(xl_parameter_change));
1461 : 6 : record_wal_level = xlrec.wal_level;
1462 : : }
1463 [ - + ]: 1046 : else if (info == XLOG_END_OF_RECOVERY)
1464 : : {
1465 : : xl_end_of_recovery xlrec;
1466 : :
1467 : : /* Extract wal_level at time record was written from payload. */
737 rhaas@postgresql.org 1468 :UBC 0 : memcpy(&xlrec, XLogRecGetData(xlogreader), sizeof(xl_end_of_recovery));
1469 : 0 : record_wal_level = xlrec.wal_level;
1470 : : }
1471 : : else
1472 : : {
1473 : : /* No special handling required. Return false. */
737 rhaas@postgresql.org 1474 :CBC 1046 : return false;
1475 : : }
1476 : :
1477 : : /*
1478 : : * Redo can only begin at an XLOG_CHECKPOINT_REDO or
1479 : : * XLOG_CHECKPOINT_SHUTDOWN record, so we want WAL summarization to begin
1480 : : * at those points. Hence, when those records are encountered, return
1481 : : * true, so that we stop just before summarizing either of those records.
1482 : : *
1483 : : * We also reach here if we just saw XLOG_END_OF_RECOVERY or
1484 : : * XLOG_PARAMETER_CHANGE. These are not places where recovery can start,
1485 : : * but they're still relevant here. A new timeline can begin with
1486 : : * XLOG_END_OF_RECOVERY, so we need to confirm the WAL level at that
1487 : : * point; and a restart can provoke XLOG_PARAMETER_CHANGE after an
1488 : : * intervening change to postgresql.conf, which might force us to stop
1489 : : * summarizing.
1490 : : */
1491 : 61 : *new_fast_forward = (record_wal_level == WAL_LEVEL_MINIMAL);
1492 : 61 : return true;
1493 : : }
1494 : :
1495 : : /*
1496 : : * Similar to read_local_xlog_page, but limited to read from one particular
1497 : : * timeline. If the end of WAL is reached, it will wait for more if reading
1498 : : * from the current timeline, or give up if reading from a historic timeline.
1499 : : * In the latter case, it will also set private_data->end_of_wal = true.
1500 : : *
1501 : : * Caller must set private_data->tli to the TLI of interest,
1502 : : * private_data->read_upto to the lowest LSN that is not known to be safe
1503 : : * to read on that timeline, and private_data->historic to true if and only
1504 : : * if the timeline is not the current timeline. This function will update
1505 : : * private_data->read_upto and private_data->historic if more WAL appears
1506 : : * on the current timeline or if the current timeline becomes historic.
1507 : : */
1508 : : static int
948 1509 : 3103 : summarizer_read_local_xlog_page(XLogReaderState *state,
1510 : : XLogRecPtr targetPagePtr, int reqLen,
1511 : : XLogRecPtr targetRecPtr, char *cur_page)
1512 : : {
1513 : : int count;
1514 : : WALReadError errinfo;
1515 : : SummarizerReadLocalXLogPrivate *private_data;
1516 : :
507 heikki.linnakangas@i 1517 : 3103 : ProcessWalSummarizerInterrupts();
1518 : :
948 rhaas@postgresql.org 1519 : 3103 : private_data = (SummarizerReadLocalXLogPrivate *)
1520 : : state->private_data;
1521 : :
1522 : : while (1)
1523 : : {
1524 [ + + ]: 3143 : if (targetPagePtr + XLOG_BLCKSZ <= private_data->read_upto)
1525 : : {
1526 : : /*
1527 : : * more than one block available; read only that block, have
1528 : : * caller come back if they need more.
1529 : : */
1530 : 3071 : count = XLOG_BLCKSZ;
1531 : 3071 : break;
1532 : : }
1533 [ + + ]: 72 : else if (targetPagePtr + reqLen > private_data->read_upto)
1534 : : {
1535 : : /* We don't seem to have enough data. */
1536 [ - + ]: 43 : if (private_data->historic)
1537 : : {
1538 : : /*
1539 : : * This is a historic timeline, so there will never be any
1540 : : * more data than we have currently.
1541 : : */
948 rhaas@postgresql.org 1542 :UBC 0 : private_data->end_of_wal = true;
1543 : 0 : return -1;
1544 : : }
1545 : : else
1546 : : {
1547 : : XLogRecPtr latest_lsn;
1548 : : TimeLineID latest_tli;
1549 : :
1550 : : /*
1551 : : * This is - or at least was up until very recently - the
1552 : : * current timeline, so more data might show up. Delay here
1553 : : * so we don't tight-loop.
1554 : : */
507 heikki.linnakangas@i 1555 :CBC 43 : ProcessWalSummarizerInterrupts();
948 rhaas@postgresql.org 1556 : 40 : summarizer_wait_for_wal();
1557 : :
1558 : : /* Recheck end-of-WAL. */
1559 : 40 : latest_lsn = GetLatestLSN(&latest_tli);
1560 [ + - ]: 40 : if (private_data->tli == latest_tli)
1561 : : {
1562 : : /* Still the current timeline, update max LSN. */
1563 [ - + ]: 40 : Assert(latest_lsn >= private_data->read_upto);
1564 : 40 : private_data->read_upto = latest_lsn;
1565 : : }
1566 : : else
1567 : : {
948 rhaas@postgresql.org 1568 :UBC 0 : List *tles = readTimeLineHistory(latest_tli);
1569 : : XLogRecPtr switchpoint;
1570 : :
1571 : : /*
1572 : : * The timeline we're scanning is no longer the latest
1573 : : * one. Figure out when it ended.
1574 : : */
1575 : 0 : private_data->historic = true;
1576 : 0 : switchpoint = tliSwitchPoint(private_data->tli, tles,
1577 : : NULL);
1578 : :
1579 : : /*
1580 : : * Allow reads up to exactly the switch point.
1581 : : *
1582 : : * It's possible that this will cause read_upto to move
1583 : : * backwards, because we might have been promoted before
1584 : : * reaching the end of the previous timeline. In that
1585 : : * case, the next loop iteration will likely conclude that
1586 : : * we've reached end of WAL.
1587 : : */
1588 : 0 : private_data->read_upto = switchpoint;
1589 : :
1590 : : /* Debugging output. */
1591 [ # # ]: 0 : ereport(DEBUG1,
1592 : : errmsg_internal("timeline %u became historic, can read up to %X/%08X",
1593 : : private_data->tli, LSN_FORMAT_ARGS(private_data->read_upto)));
1594 : : }
1595 : :
1596 : : /* Go around and try again. */
1597 : : }
1598 : : }
1599 : : else
1600 : : {
1601 : : /* enough bytes available to satisfy the request */
948 rhaas@postgresql.org 1602 :CBC 29 : count = private_data->read_upto - targetPagePtr;
1603 : 29 : break;
1604 : : }
1605 : : }
1606 : :
890 jdavis@postgresql.or 1607 [ - + ]: 3100 : if (!WALRead(state, cur_page, targetPagePtr, count,
1608 : : private_data->tli, &errinfo))
948 rhaas@postgresql.org 1609 :UBC 0 : WALReadRaiseError(&errinfo);
1610 : :
1611 : : /* Track that we read a page, for sleep time calculation. */
948 rhaas@postgresql.org 1612 :CBC 3100 : ++pages_read_since_last_sleep;
1613 : :
1614 : : /* number of valid bytes in the buffer */
1615 : 3100 : return count;
1616 : : }
1617 : :
1618 : : /*
1619 : : * Sleep for long enough that we believe it's likely that more WAL will
1620 : : * be available afterwards.
1621 : : */
1622 : : static void
1623 : 40 : summarizer_wait_for_wal(void)
1624 : : {
1625 [ + + ]: 40 : if (pages_read_since_last_sleep == 0)
1626 : : {
1627 : : /*
1628 : : * No pages were read since the last sleep, so double the sleep time,
1629 : : * but not beyond the maximum allowable value.
1630 : : */
1631 : 15 : sleep_quanta = Min(sleep_quanta * 2, MAX_SLEEP_QUANTA);
1632 : : }
1633 [ + + ]: 25 : else if (pages_read_since_last_sleep > 1)
1634 : : {
1635 : : /*
1636 : : * Multiple pages were read since the last sleep, so reduce the sleep
1637 : : * time.
1638 : : *
1639 : : * A large burst of activity should be able to quickly reduce the
1640 : : * sleep time to the minimum, but we don't want a handful of extra WAL
1641 : : * records to provoke a strong reaction. We choose to reduce the sleep
1642 : : * time by 1 quantum for each page read beyond the first, which is a
1643 : : * fairly arbitrary way of trying to be reactive without overreacting.
1644 : : */
1645 [ + - ]: 21 : if (pages_read_since_last_sleep > sleep_quanta - 1)
1646 : 21 : sleep_quanta = 1;
1647 : : else
948 rhaas@postgresql.org 1648 :UBC 0 : sleep_quanta -= pages_read_since_last_sleep;
1649 : : }
1650 : :
1651 : : /* Report pending statistics to the cumulative stats system. */
507 michael@paquier.xyz 1652 :CBC 40 : pgstat_report_wal(false);
1653 : :
1654 : : /* OK, now sleep. */
948 rhaas@postgresql.org 1655 : 40 : (void) WaitLatch(MyLatch,
1656 : : WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH,
1657 : : sleep_quanta * MS_PER_SLEEP_QUANTUM,
1658 : : WAIT_EVENT_WAL_SUMMARIZER_WAL);
1659 : 40 : ResetLatch(MyLatch);
1660 : :
1661 : : /* Reset count of pages read. */
1662 : 40 : pages_read_since_last_sleep = 0;
1663 : 40 : }
1664 : :
1665 : : /*
1666 : : * Remove WAL summaries whose mtimes are older than wal_summary_keep_time.
1667 : : */
1668 : : static void
1669 : 33 : MaybeRemoveOldWalSummaries(void)
1670 : : {
1671 : 33 : XLogRecPtr redo_pointer = GetRedoRecPtr();
1672 : : List *wslist;
1673 : : time_t cutoff_time;
1674 : :
1675 : : /* If WAL summary removal is disabled, don't do anything. */
1676 [ - + ]: 33 : if (wal_summary_keep_time == 0)
948 rhaas@postgresql.org 1677 :UBC 0 : return;
1678 : :
1679 : : /*
1680 : : * If the redo pointer has not advanced, don't do anything.
1681 : : *
1682 : : * This has the effect that we only try to remove old WAL summary files
1683 : : * once per checkpoint cycle.
1684 : : */
948 rhaas@postgresql.org 1685 [ + + ]:CBC 33 : if (redo_pointer == redo_pointer_at_last_summary_removal)
1686 : 23 : return;
1687 : 10 : redo_pointer_at_last_summary_removal = redo_pointer;
1688 : :
1689 : : /*
1690 : : * Files should only be removed if the last modification time precedes the
1691 : : * cutoff time we compute here.
1692 : : */
857 nathan@postgresql.or 1693 : 10 : cutoff_time = time(NULL) - wal_summary_keep_time * SECS_PER_MINUTE;
1694 : :
1695 : : /* Get all the summaries that currently exist. */
948 rhaas@postgresql.org 1696 : 10 : wslist = GetWalSummaries(0, InvalidXLogRecPtr, InvalidXLogRecPtr);
1697 : :
1698 : : /* Loop until all summaries have been considered for removal. */
1699 [ + + ]: 15 : while (wslist != NIL)
1700 : : {
1701 : : ListCell *lc;
1702 : : XLogSegNo oldest_segno;
1703 : 5 : XLogRecPtr oldest_lsn = InvalidXLogRecPtr;
1704 : : TimeLineID selected_tli;
1705 : :
507 heikki.linnakangas@i 1706 : 5 : ProcessWalSummarizerInterrupts();
1707 : :
1708 : : /*
1709 : : * Pick a timeline for which some summary files still exist on disk,
1710 : : * and find the oldest LSN that still exists on disk for that
1711 : : * timeline.
1712 : : */
948 rhaas@postgresql.org 1713 : 5 : selected_tli = ((WalSummaryFile *) linitial(wslist))->tli;
1714 : 5 : oldest_segno = XLogGetOldestSegno(selected_tli);
1715 [ + - ]: 5 : if (oldest_segno != 0)
1716 : 5 : XLogSegNoOffsetToRecPtr(oldest_segno, 0, wal_segment_size,
1717 : : oldest_lsn);
1718 : :
1719 : :
1720 : : /* Consider each WAL file on the selected timeline in turn. */
1721 [ + + + - : 47 : foreach(lc, wslist)
+ + ]
1722 : : {
1723 : 42 : WalSummaryFile *ws = lfirst(lc);
1724 : :
507 heikki.linnakangas@i 1725 : 42 : ProcessWalSummarizerInterrupts();
1726 : :
1727 : : /* If it's not on this timeline, it's not time to consider it. */
948 rhaas@postgresql.org 1728 [ - + ]: 42 : if (selected_tli != ws->tli)
948 rhaas@postgresql.org 1729 :UBC 0 : continue;
1730 : :
1731 : : /*
1732 : : * If the WAL doesn't exist any more, we can remove it if the file
1733 : : * modification time is old enough.
1734 : : */
261 alvherre@kurilemu.de 1735 [ + - + + ]:CBC 42 : if (!XLogRecPtrIsValid(oldest_lsn) || ws->end_lsn <= oldest_lsn)
948 rhaas@postgresql.org 1736 : 7 : RemoveWalSummaryIfOlderThan(ws, cutoff_time);
1737 : :
1738 : : /*
1739 : : * Whether we removed the file or not, we need not consider it
1740 : : * again.
1741 : : */
1742 : 42 : wslist = foreach_delete_current(wslist, lc);
1743 : 42 : pfree(ws);
1744 : : }
1745 : : }
1746 : : }
|
