Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * bufmgr.c
4 : * buffer manager interface routines
5 : *
6 : * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
7 : * Portions Copyright (c) 1994, Regents of the University of California
8 : *
9 : *
10 : * IDENTIFICATION
11 : * src/backend/storage/buffer/bufmgr.c
12 : *
13 : *-------------------------------------------------------------------------
14 : */
15 : /*
16 : * Principal entry points:
17 : *
18 : * ReadBuffer() -- find or create a buffer holding the requested page,
19 : * and pin it so that no one can destroy it while this process
20 : * is using it.
21 : *
22 : * ReleaseBuffer() -- unpin a buffer
23 : *
24 : * MarkBufferDirty() -- mark a pinned buffer's contents as "dirty".
25 : * The disk write is delayed until buffer replacement or checkpoint.
26 : *
27 : * See also these files:
28 : * freelist.c -- chooses victim for buffer replacement
29 : * buf_table.c -- manages the buffer lookup table
30 : */
31 : #include "postgres.h"
32 :
33 : #include <sys/file.h>
34 : #include <unistd.h>
35 :
36 : #include "access/tableam.h"
37 : #include "access/xlog.h"
38 : #include "catalog/catalog.h"
39 : #include "catalog/storage.h"
40 : #include "executor/instrument.h"
41 : #include "lib/binaryheap.h"
42 : #include "miscadmin.h"
43 : #include "pg_trace.h"
44 : #include "pgstat.h"
45 : #include "postmaster/bgwriter.h"
46 : #include "storage/buf_internals.h"
47 : #include "storage/bufmgr.h"
48 : #include "storage/ipc.h"
49 : #include "storage/proc.h"
50 : #include "storage/smgr.h"
51 : #include "storage/standby.h"
52 : #include "utils/rel.h"
53 : #include "utils/resowner_private.h"
54 : #include "utils/timestamp.h"
55 :
56 :
57 : /* Note: these two macros only work on shared buffers, not local ones! */
58 : #define BufHdrGetBlock(bufHdr) ((Block) (BufferBlocks + ((Size) (bufHdr)->buf_id) * BLCKSZ))
59 : #define BufferGetLSN(bufHdr) (PageGetLSN(BufHdrGetBlock(bufHdr)))
60 :
61 : /* Note: this macro only works on local buffers, not shared ones! */
62 : #define LocalBufHdrGetBlock(bufHdr) \
63 : LocalBufferBlockPointers[-((bufHdr)->buf_id + 2)]
64 :
65 : /* Bits in SyncOneBuffer's return value */
66 : #define BUF_WRITTEN 0x01
67 : #define BUF_REUSABLE 0x02
68 :
69 : #define DROP_RELS_BSEARCH_THRESHOLD 20
70 :
71 : typedef struct PrivateRefCountEntry
72 : {
73 : Buffer buffer;
74 : int32 refcount;
75 : } PrivateRefCountEntry;
76 :
77 : /* 64 bytes, about the size of a cache line on common systems */
78 : #define REFCOUNT_ARRAY_ENTRIES 8
79 :
80 : /*
81 : * Status of buffers to checkpoint for a particular tablespace, used
82 : * internally in BufferSync.
83 : */
84 : typedef struct CkptTsStatus
85 : {
86 : /* oid of the tablespace */
87 : Oid tsId;
88 :
89 : /*
90 : * Checkpoint progress for this tablespace. To make progress comparable
91 : * between tablespaces the progress is, for each tablespace, measured as a
92 : * number between 0 and the total number of to-be-checkpointed pages. Each
93 : * page checkpointed in this tablespace increments this space's progress
94 : * by progress_slice.
95 : */
96 : float8 progress;
97 : float8 progress_slice;
98 :
99 : /* number of to-be checkpointed pages in this tablespace */
100 : int num_to_scan;
101 : /* already processed pages in this tablespace */
102 : int num_scanned;
103 :
104 : /* current offset in CkptBufferIds for this tablespace */
105 : int index;
106 : } CkptTsStatus;
107 :
108 : /* GUC variables */
109 : bool zero_damaged_pages = false;
110 : int bgwriter_lru_maxpages = 100;
111 : double bgwriter_lru_multiplier = 2.0;
112 : bool track_io_timing = false;
113 : int effective_io_concurrency = 0;
114 :
115 : /*
116 : * GUC variables about triggering kernel writeback for buffers written; OS
117 : * dependent defaults are set via the GUC mechanism.
118 : */
119 : int checkpoint_flush_after = 0;
120 : int bgwriter_flush_after = 0;
121 : int backend_flush_after = 0;
122 :
123 : /*
124 : * How many buffers PrefetchBuffer callers should try to stay ahead of their
125 : * ReadBuffer calls by. This is maintained by the assign hook for
126 : * effective_io_concurrency. Zero means "never prefetch". This value is
127 : * only used for buffers not belonging to tablespaces that have their
128 : * effective_io_concurrency parameter set.
129 : */
130 : int target_prefetch_pages = 0;
131 :
132 : /* local state for StartBufferIO and related functions */
133 : static BufferDesc *InProgressBuf = NULL;
134 : static bool IsForInput;
135 :
136 : /* local state for LockBufferForCleanup */
137 : static BufferDesc *PinCountWaitBuf = NULL;
138 :
139 : /*
140 : * Backend-Private refcount management:
141 : *
142 : * Each buffer also has a private refcount that keeps track of the number of
143 : * times the buffer is pinned in the current process. This is so that the
144 : * shared refcount needs to be modified only once if a buffer is pinned more
145 : * than once by an individual backend. It's also used to check that no buffers
146 : * are still pinned at the end of transactions and when exiting.
147 : *
148 : *
149 : * To avoid - as we used to - requiring an array with NBuffers entries to keep
150 : * track of local buffers, we use a small sequentially searched array
151 : * (PrivateRefCountArray) and an overflow hash table (PrivateRefCountHash) to
152 : * keep track of backend local pins.
153 : *
154 : * Until no more than REFCOUNT_ARRAY_ENTRIES buffers are pinned at once, all
155 : * refcounts are kept track of in the array; after that, new array entries
156 : * displace old ones into the hash table. That way a frequently used entry
157 : * can't get "stuck" in the hashtable while infrequent ones clog the array.
158 : *
159 : * Note that in most scenarios the number of pinned buffers will not exceed
160 : * REFCOUNT_ARRAY_ENTRIES.
161 : *
162 : *
163 : * To enter a buffer into the refcount tracking mechanism first reserve a free
164 : * entry using ReservePrivateRefCountEntry() and then later, if necessary,
165 : * fill it with NewPrivateRefCountEntry(). That split lets us avoid doing
166 : * memory allocations in NewPrivateRefCountEntry() which can be important
167 : * because in some scenarios it's called with a spinlock held...
168 : */
169 : static struct PrivateRefCountEntry PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES];
170 : static HTAB *PrivateRefCountHash = NULL;
171 : static int32 PrivateRefCountOverflowed = 0;
172 : static uint32 PrivateRefCountClock = 0;
173 : static PrivateRefCountEntry *ReservedRefCountEntry = NULL;
174 :
175 : static void ReservePrivateRefCountEntry(void);
176 : static PrivateRefCountEntry *NewPrivateRefCountEntry(Buffer buffer);
177 : static PrivateRefCountEntry *GetPrivateRefCountEntry(Buffer buffer, bool do_move);
178 : static inline int32 GetPrivateRefCount(Buffer buffer);
179 : static void ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref);
180 :
181 : /*
182 : * Ensure that the PrivateRefCountArray has sufficient space to store one more
183 : * entry. This has to be called before using NewPrivateRefCountEntry() to fill
184 : * a new entry - but it's perfectly fine to not use a reserved entry.
185 : */
186 : static void
187 79081504 : ReservePrivateRefCountEntry(void)
188 : {
189 : /* Already reserved (or freed), nothing to do */
190 79081504 : if (ReservedRefCountEntry != NULL)
191 76181198 : return;
192 :
193 : /*
194 : * First search for a free entry the array, that'll be sufficient in the
195 : * majority of cases.
196 : */
197 : {
198 : int i;
199 :
200 6086806 : for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
201 : {
202 : PrivateRefCountEntry *res;
203 :
204 6079464 : res = &PrivateRefCountArray[i];
205 :
206 6079464 : if (res->buffer == InvalidBuffer)
207 : {
208 2892964 : ReservedRefCountEntry = res;
209 2892964 : return;
210 : }
211 : }
212 : }
213 :
214 : /*
215 : * No luck. All array entries are full. Move one array entry into the hash
216 : * table.
217 : */
218 : {
219 : /*
220 : * Move entry from the current clock position in the array into the
221 : * hashtable. Use that slot.
222 : */
223 : PrivateRefCountEntry *hashent;
224 : bool found;
225 :
226 : /* select victim slot */
227 7342 : ReservedRefCountEntry =
228 7342 : &PrivateRefCountArray[PrivateRefCountClock++ % REFCOUNT_ARRAY_ENTRIES];
229 :
230 : /* Better be used, otherwise we shouldn't get here. */
231 : Assert(ReservedRefCountEntry->buffer != InvalidBuffer);
232 :
233 : /* enter victim array entry into hashtable */
234 7342 : hashent = hash_search(PrivateRefCountHash,
235 7342 : (void *) &(ReservedRefCountEntry->buffer),
236 : HASH_ENTER,
237 : &found);
238 : Assert(!found);
239 7342 : hashent->refcount = ReservedRefCountEntry->refcount;
240 :
241 : /* clear the now free array slot */
242 7342 : ReservedRefCountEntry->buffer = InvalidBuffer;
243 7342 : ReservedRefCountEntry->refcount = 0;
244 :
245 7342 : PrivateRefCountOverflowed++;
246 : }
247 : }
248 :
249 : /*
250 : * Fill a previously reserved refcount entry.
251 : */
252 : static PrivateRefCountEntry *
253 78607460 : NewPrivateRefCountEntry(Buffer buffer)
254 : {
255 : PrivateRefCountEntry *res;
256 :
257 : /* only allowed to be called when a reservation has been made */
258 : Assert(ReservedRefCountEntry != NULL);
259 :
260 : /* use up the reserved entry */
261 78607460 : res = ReservedRefCountEntry;
262 78607460 : ReservedRefCountEntry = NULL;
263 :
264 : /* and fill it */
265 78607460 : res->buffer = buffer;
266 78607460 : res->refcount = 0;
267 :
268 78607460 : return res;
269 : }
270 :
271 : /*
272 : * Return the PrivateRefCount entry for the passed buffer.
273 : *
274 : * Returns NULL if a buffer doesn't have a refcount entry. Otherwise, if
275 : * do_move is true, and the entry resides in the hashtable the entry is
276 : * optimized for frequent access by moving it to the array.
277 : */
278 : static PrivateRefCountEntry *
279 185770180 : GetPrivateRefCountEntry(Buffer buffer, bool do_move)
280 : {
281 : PrivateRefCountEntry *res;
282 : int i;
283 :
284 : Assert(BufferIsValid(buffer));
285 : Assert(!BufferIsLocal(buffer));
286 :
287 : /*
288 : * First search for references in the array, that'll be sufficient in the
289 : * majority of cases.
290 : */
291 904388300 : for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
292 : {
293 827038990 : res = &PrivateRefCountArray[i];
294 :
295 827038990 : if (res->buffer == buffer)
296 108420870 : return res;
297 : }
298 :
299 : /*
300 : * By here we know that the buffer, if already pinned, isn't residing in
301 : * the array.
302 : *
303 : * Only look up the buffer in the hashtable if we've previously overflowed
304 : * into it.
305 : */
306 77349310 : if (PrivateRefCountOverflowed == 0)
307 77333790 : return NULL;
308 :
309 15520 : res = hash_search(PrivateRefCountHash,
310 : (void *) &buffer,
311 : HASH_FIND,
312 : NULL);
313 :
314 15520 : if (res == NULL)
315 8160 : return NULL;
316 7360 : else if (!do_move)
317 : {
318 : /* caller doesn't want us to move the hash entry into the array */
319 7354 : return res;
320 : }
321 : else
322 : {
323 : /* move buffer from hashtable into the free array slot */
324 : bool found;
325 : PrivateRefCountEntry *free;
326 :
327 : /* Ensure there's a free array slot */
328 6 : ReservePrivateRefCountEntry();
329 :
330 : /* Use up the reserved slot */
331 : Assert(ReservedRefCountEntry != NULL);
332 6 : free = ReservedRefCountEntry;
333 6 : ReservedRefCountEntry = NULL;
334 : Assert(free->buffer == InvalidBuffer);
335 :
336 : /* and fill it */
337 6 : free->buffer = buffer;
338 6 : free->refcount = res->refcount;
339 :
340 : /* delete from hashtable */
341 6 : hash_search(PrivateRefCountHash,
342 : (void *) &buffer,
343 : HASH_REMOVE,
344 : &found);
345 : Assert(found);
346 : Assert(PrivateRefCountOverflowed > 0);
347 6 : PrivateRefCountOverflowed--;
348 :
349 6 : return free;
350 : }
351 : }
352 :
353 : /*
354 : * Returns how many times the passed buffer is pinned by this backend.
355 : *
356 : * Only works for shared memory buffers!
357 : */
358 : static inline int32
359 366206 : GetPrivateRefCount(Buffer buffer)
360 : {
361 : PrivateRefCountEntry *ref;
362 :
363 : Assert(BufferIsValid(buffer));
364 : Assert(!BufferIsLocal(buffer));
365 :
366 : /*
367 : * Not moving the entry - that's ok for the current users, but we might
368 : * want to change this one day.
369 : */
370 366206 : ref = GetPrivateRefCountEntry(buffer, false);
371 :
372 366206 : if (ref == NULL)
373 0 : return 0;
374 366206 : return ref->refcount;
375 : }
376 :
377 : /*
378 : * Release resources used to track the reference count of a buffer which we no
379 : * longer have pinned and don't want to pin again immediately.
380 : */
381 : static void
382 78607460 : ForgetPrivateRefCountEntry(PrivateRefCountEntry *ref)
383 : {
384 : Assert(ref->refcount == 0);
385 :
386 78607460 : if (ref >= &PrivateRefCountArray[0] &&
387 : ref < &PrivateRefCountArray[REFCOUNT_ARRAY_ENTRIES])
388 : {
389 78600124 : ref->buffer = InvalidBuffer;
390 :
391 : /*
392 : * Mark the just used entry as reserved - in many scenarios that
393 : * allows us to avoid ever having to search the array/hash for free
394 : * entries.
395 : */
396 78600124 : ReservedRefCountEntry = ref;
397 : }
398 : else
399 : {
400 : bool found;
401 7336 : Buffer buffer = ref->buffer;
402 :
403 7336 : hash_search(PrivateRefCountHash,
404 : (void *) &buffer,
405 : HASH_REMOVE,
406 : &found);
407 : Assert(found);
408 : Assert(PrivateRefCountOverflowed > 0);
409 7336 : PrivateRefCountOverflowed--;
410 : }
411 78607460 : }
412 :
413 : /*
414 : * BufferIsPinned
415 : * True iff the buffer is pinned (also checks for valid buffer number).
416 : *
417 : * NOTE: what we check here is that *this* backend holds a pin on
418 : * the buffer. We do not care whether some other backend does.
419 : */
420 : #define BufferIsPinned(bufnum) \
421 : ( \
422 : !BufferIsValid(bufnum) ? \
423 : false \
424 : : \
425 : BufferIsLocal(bufnum) ? \
426 : (LocalRefCount[-(bufnum) - 1] > 0) \
427 : : \
428 : (GetPrivateRefCount(bufnum) > 0) \
429 : )
430 :
431 :
432 : static Buffer ReadBuffer_common(SMgrRelation reln, char relpersistence,
433 : ForkNumber forkNum, BlockNumber blockNum,
434 : ReadBufferMode mode, BufferAccessStrategy strategy,
435 : bool *hit);
436 : static bool PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy);
437 : static void PinBuffer_Locked(BufferDesc *buf);
438 : static void UnpinBuffer(BufferDesc *buf, bool fixOwner);
439 : static void BufferSync(int flags);
440 : static uint32 WaitBufHdrUnlocked(BufferDesc *buf);
441 : static int SyncOneBuffer(int buf_id, bool skip_recently_used,
442 : WritebackContext *wb_context);
443 : static void WaitIO(BufferDesc *buf);
444 : static bool StartBufferIO(BufferDesc *buf, bool forInput);
445 : static void TerminateBufferIO(BufferDesc *buf, bool clear_dirty,
446 : uint32 set_flag_bits);
447 : static void shared_buffer_write_error_callback(void *arg);
448 : static void local_buffer_write_error_callback(void *arg);
449 : static BufferDesc *BufferAlloc(SMgrRelation smgr,
450 : char relpersistence,
451 : ForkNumber forkNum,
452 : BlockNumber blockNum,
453 : BufferAccessStrategy strategy,
454 : bool *foundPtr);
455 : static void FlushBuffer(BufferDesc *buf, SMgrRelation reln);
456 : static void AtProcExit_Buffers(int code, Datum arg);
457 : static void CheckForBufferLeaks(void);
458 : static int rnode_comparator(const void *p1, const void *p2);
459 : static int buffertag_comparator(const void *p1, const void *p2);
460 : static int ckpt_buforder_comparator(const void *pa, const void *pb);
461 : static int ts_ckpt_progress_comparator(Datum a, Datum b, void *arg);
462 :
463 :
464 : /*
465 : * ComputeIoConcurrency -- get the number of pages to prefetch for a given
466 : * number of spindles.
467 : */
468 : bool
469 2294 : ComputeIoConcurrency(int io_concurrency, double *target)
470 : {
471 2294 : double new_prefetch_pages = 0.0;
472 : int i;
473 :
474 : /*
475 : * Make sure the io_concurrency value is within valid range; it may have
476 : * been forced with a manual pg_tablespace update.
477 : */
478 2294 : io_concurrency = Min(Max(io_concurrency, 0), MAX_IO_CONCURRENCY);
479 :
480 : /*----------
481 : * The user-visible GUC parameter is the number of drives (spindles),
482 : * which we need to translate to a number-of-pages-to-prefetch target.
483 : * The target value is stashed in *extra and then assigned to the actual
484 : * variable by assign_effective_io_concurrency.
485 : *
486 : * The expected number of prefetch pages needed to keep N drives busy is:
487 : *
488 : * drives | I/O requests
489 : * -------+----------------
490 : * 1 | 1
491 : * 2 | 2/1 + 2/2 = 3
492 : * 3 | 3/1 + 3/2 + 3/3 = 5 1/2
493 : * 4 | 4/1 + 4/2 + 4/3 + 4/4 = 8 1/3
494 : * n | n * H(n)
495 : *
496 : * This is called the "coupon collector problem" and H(n) is called the
497 : * harmonic series. This could be approximated by n * ln(n), but for
498 : * reasonable numbers of drives we might as well just compute the series.
499 : *
500 : * Alternatively we could set the target to the number of pages necessary
501 : * so that the expected number of active spindles is some arbitrary
502 : * percentage of the total. This sounds the same but is actually slightly
503 : * different. The result ends up being ln(1-P)/ln((n-1)/n) where P is
504 : * that desired fraction.
505 : *
506 : * Experimental results show that both of these formulas aren't aggressive
507 : * enough, but we don't really have any better proposals.
508 : *
509 : * Note that if io_concurrency = 0 (disabled), we must set target = 0.
510 : *----------
511 : */
512 :
513 23992 : for (i = 1; i <= io_concurrency; i++)
514 21698 : new_prefetch_pages += (double) io_concurrency / (double) i;
515 :
516 2294 : *target = new_prefetch_pages;
517 :
518 : /* This range check shouldn't fail, but let's be paranoid */
519 2294 : return (new_prefetch_pages >= 0.0 && new_prefetch_pages < (double) INT_MAX);
520 : }
521 :
522 : /*
523 : * PrefetchBuffer -- initiate asynchronous read of a block of a relation
524 : *
525 : * This is named by analogy to ReadBuffer but doesn't actually allocate a
526 : * buffer. Instead it tries to ensure that a future ReadBuffer for the given
527 : * block will not be delayed by the I/O. Prefetching is optional.
528 : * No-op if prefetching isn't compiled in.
529 : */
530 : void
531 120832 : PrefetchBuffer(Relation reln, ForkNumber forkNum, BlockNumber blockNum)
532 : {
533 : #ifdef USE_PREFETCH
534 : Assert(RelationIsValid(reln));
535 : Assert(BlockNumberIsValid(blockNum));
536 :
537 : /* Open it at the smgr level if not already done */
538 120832 : RelationOpenSmgr(reln);
539 :
540 120832 : if (RelationUsesLocalBuffers(reln))
541 : {
542 : /* see comments in ReadBufferExtended */
543 212 : if (RELATION_IS_OTHER_TEMP(reln))
544 0 : ereport(ERROR,
545 : (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
546 : errmsg("cannot access temporary tables of other sessions")));
547 :
548 : /* pass it off to localbuf.c */
549 212 : LocalPrefetchBuffer(reln->rd_smgr, forkNum, blockNum);
550 : }
551 : else
552 : {
553 : BufferTag newTag; /* identity of requested block */
554 : uint32 newHash; /* hash value for newTag */
555 : LWLock *newPartitionLock; /* buffer partition lock for it */
556 : int buf_id;
557 :
558 : /* create a tag so we can lookup the buffer */
559 120620 : INIT_BUFFERTAG(newTag, reln->rd_smgr->smgr_rnode.node,
560 : forkNum, blockNum);
561 :
562 : /* determine its hash code and partition lock ID */
563 120620 : newHash = BufTableHashCode(&newTag);
564 120620 : newPartitionLock = BufMappingPartitionLock(newHash);
565 :
566 : /* see if the block is in the buffer pool already */
567 120620 : LWLockAcquire(newPartitionLock, LW_SHARED);
568 120620 : buf_id = BufTableLookup(&newTag, newHash);
569 120620 : LWLockRelease(newPartitionLock);
570 :
571 : /* If not in buffers, initiate prefetch */
572 120620 : if (buf_id < 0)
573 110 : smgrprefetch(reln->rd_smgr, forkNum, blockNum);
574 :
575 : /*
576 : * If the block *is* in buffers, we do nothing. This is not really
577 : * ideal: the block might be just about to be evicted, which would be
578 : * stupid since we know we are going to need it soon. But the only
579 : * easy answer is to bump the usage_count, which does not seem like a
580 : * great solution: when the caller does ultimately touch the block,
581 : * usage_count would get bumped again, resulting in too much
582 : * favoritism for blocks that are involved in a prefetch sequence. A
583 : * real fix would involve some additional per-buffer state, and it's
584 : * not clear that there's enough of a problem to justify that.
585 : */
586 : }
587 : #endif /* USE_PREFETCH */
588 120832 : }
589 :
590 :
591 : /*
592 : * ReadBuffer -- a shorthand for ReadBufferExtended, for reading from main
593 : * fork with RBM_NORMAL mode and default strategy.
594 : */
595 : Buffer
596 57380642 : ReadBuffer(Relation reln, BlockNumber blockNum)
597 : {
598 57380642 : return ReadBufferExtended(reln, MAIN_FORKNUM, blockNum, RBM_NORMAL, NULL);
599 : }
600 :
601 : /*
602 : * ReadBufferExtended -- returns a buffer containing the requested
603 : * block of the requested relation. If the blknum
604 : * requested is P_NEW, extend the relation file and
605 : * allocate a new block. (Caller is responsible for
606 : * ensuring that only one backend tries to extend a
607 : * relation at the same time!)
608 : *
609 : * Returns: the buffer number for the buffer containing
610 : * the block read. The returned buffer has been pinned.
611 : * Does not return on error --- elog's instead.
612 : *
613 : * Assume when this function is called, that reln has been opened already.
614 : *
615 : * In RBM_NORMAL mode, the page is read from disk, and the page header is
616 : * validated. An error is thrown if the page header is not valid. (But
617 : * note that an all-zero page is considered "valid"; see PageIsVerified().)
618 : *
619 : * RBM_ZERO_ON_ERROR is like the normal mode, but if the page header is not
620 : * valid, the page is zeroed instead of throwing an error. This is intended
621 : * for non-critical data, where the caller is prepared to repair errors.
622 : *
623 : * In RBM_ZERO_AND_LOCK mode, if the page isn't in buffer cache already, it's
624 : * filled with zeros instead of reading it from disk. Useful when the caller
625 : * is going to fill the page from scratch, since this saves I/O and avoids
626 : * unnecessary failure if the page-on-disk has corrupt page headers.
627 : * The page is returned locked to ensure that the caller has a chance to
628 : * initialize the page before it's made visible to others.
629 : * Caution: do not use this mode to read a page that is beyond the relation's
630 : * current physical EOF; that is likely to cause problems in md.c when
631 : * the page is modified and written out. P_NEW is OK, though.
632 : *
633 : * RBM_ZERO_AND_CLEANUP_LOCK is the same as RBM_ZERO_AND_LOCK, but acquires
634 : * a cleanup-strength lock on the page.
635 : *
636 : * RBM_NORMAL_NO_LOG mode is treated the same as RBM_NORMAL here.
637 : *
638 : * If strategy is not NULL, a nondefault buffer access strategy is used.
639 : * See buffer/README for details.
640 : */
641 : Buffer
642 80093154 : ReadBufferExtended(Relation reln, ForkNumber forkNum, BlockNumber blockNum,
643 : ReadBufferMode mode, BufferAccessStrategy strategy)
644 : {
645 : bool hit;
646 : Buffer buf;
647 :
648 : /* Open it at the smgr level if not already done */
649 80093154 : RelationOpenSmgr(reln);
650 :
651 : /*
652 : * Reject attempts to read non-local temporary relations; we would be
653 : * likely to get wrong data since we have no visibility into the owning
654 : * session's local buffers.
655 : */
656 80093154 : if (RELATION_IS_OTHER_TEMP(reln))
657 0 : ereport(ERROR,
658 : (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
659 : errmsg("cannot access temporary tables of other sessions")));
660 :
661 : /*
662 : * Read the buffer, and update pgstat counters to reflect a cache hit or
663 : * miss.
664 : */
665 80093154 : pgstat_count_buffer_read(reln);
666 80093154 : buf = ReadBuffer_common(reln->rd_smgr, reln->rd_rel->relpersistence,
667 : forkNum, blockNum, mode, strategy, &hit);
668 80093154 : if (hit)
669 79272438 : pgstat_count_buffer_hit(reln);
670 80093154 : return buf;
671 : }
672 :
673 :
674 : /*
675 : * ReadBufferWithoutRelcache -- like ReadBufferExtended, but doesn't require
676 : * a relcache entry for the relation.
677 : *
678 : * NB: At present, this function may only be used on permanent relations, which
679 : * is OK, because we only use it during XLOG replay. If in the future we
680 : * want to use it on temporary or unlogged relations, we could pass additional
681 : * parameters.
682 : */
683 : Buffer
684 264938 : ReadBufferWithoutRelcache(RelFileNode rnode, ForkNumber forkNum,
685 : BlockNumber blockNum, ReadBufferMode mode,
686 : BufferAccessStrategy strategy)
687 : {
688 : bool hit;
689 :
690 264938 : SMgrRelation smgr = smgropen(rnode, InvalidBackendId);
691 :
692 : Assert(InRecovery);
693 :
694 264938 : return ReadBuffer_common(smgr, RELPERSISTENCE_PERMANENT, forkNum, blockNum,
695 : mode, strategy, &hit);
696 : }
697 :
698 :
699 : /*
700 : * ReadBuffer_common -- common logic for all ReadBuffer variants
701 : *
702 : * *hit is set to true if the request was satisfied from shared buffer cache.
703 : */
704 : static Buffer
705 80358092 : ReadBuffer_common(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
706 : BlockNumber blockNum, ReadBufferMode mode,
707 : BufferAccessStrategy strategy, bool *hit)
708 : {
709 : BufferDesc *bufHdr;
710 : Block bufBlock;
711 : bool found;
712 : bool isExtend;
713 80358092 : bool isLocalBuf = SmgrIsTemp(smgr);
714 :
715 80358092 : *hit = false;
716 :
717 : /* Make sure we will have room to remember the buffer pin */
718 80358092 : ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
719 :
720 80358092 : isExtend = (blockNum == P_NEW);
721 :
722 : TRACE_POSTGRESQL_BUFFER_READ_START(forkNum, blockNum,
723 : smgr->smgr_rnode.node.spcNode,
724 : smgr->smgr_rnode.node.dbNode,
725 : smgr->smgr_rnode.node.relNode,
726 : smgr->smgr_rnode.backend,
727 : isExtend);
728 :
729 : /* Substitute proper block number if caller asked for P_NEW */
730 80358092 : if (isExtend)
731 350966 : blockNum = smgrnblocks(smgr, forkNum);
732 :
733 80358092 : if (isLocalBuf)
734 : {
735 356872 : bufHdr = LocalBufferAlloc(smgr, forkNum, blockNum, &found);
736 356872 : if (found)
737 330586 : pgBufferUsage.local_blks_hit++;
738 26286 : else if (isExtend)
739 25084 : pgBufferUsage.local_blks_written++;
740 1202 : else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG ||
741 : mode == RBM_ZERO_ON_ERROR)
742 1202 : pgBufferUsage.local_blks_read++;
743 : }
744 : else
745 : {
746 : /*
747 : * lookup the buffer. IO_IN_PROGRESS is set if the requested block is
748 : * not currently in memory.
749 : */
750 80001220 : bufHdr = BufferAlloc(smgr, relpersistence, forkNum, blockNum,
751 : strategy, &found);
752 80001220 : if (found)
753 79201446 : pgBufferUsage.shared_blks_hit++;
754 799774 : else if (isExtend)
755 325882 : pgBufferUsage.shared_blks_written++;
756 473892 : else if (mode == RBM_NORMAL || mode == RBM_NORMAL_NO_LOG ||
757 : mode == RBM_ZERO_ON_ERROR)
758 471076 : pgBufferUsage.shared_blks_read++;
759 : }
760 :
761 : /* At this point we do NOT hold any locks. */
762 :
763 : /* if it was already in the buffer pool, we're done */
764 80358092 : if (found)
765 : {
766 79532032 : if (!isExtend)
767 : {
768 : /* Just need to update stats before we exit */
769 79532032 : *hit = true;
770 79532032 : VacuumPageHit++;
771 :
772 79532032 : if (VacuumCostActive)
773 107034 : VacuumCostBalance += VacuumCostPageHit;
774 :
775 : TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum,
776 : smgr->smgr_rnode.node.spcNode,
777 : smgr->smgr_rnode.node.dbNode,
778 : smgr->smgr_rnode.node.relNode,
779 : smgr->smgr_rnode.backend,
780 : isExtend,
781 : found);
782 :
783 : /*
784 : * In RBM_ZERO_AND_LOCK mode the caller expects the page to be
785 : * locked on return.
786 : */
787 79532032 : if (!isLocalBuf)
788 : {
789 79201446 : if (mode == RBM_ZERO_AND_LOCK)
790 248 : LWLockAcquire(BufferDescriptorGetContentLock(bufHdr),
791 : LW_EXCLUSIVE);
792 79201198 : else if (mode == RBM_ZERO_AND_CLEANUP_LOCK)
793 0 : LockBufferForCleanup(BufferDescriptorGetBuffer(bufHdr));
794 : }
795 :
796 79532032 : return BufferDescriptorGetBuffer(bufHdr);
797 : }
798 :
799 : /*
800 : * We get here only in the corner case where we are trying to extend
801 : * the relation but we found a pre-existing buffer marked BM_VALID.
802 : * This can happen because mdread doesn't complain about reads beyond
803 : * EOF (when zero_damaged_pages is ON) and so a previous attempt to
804 : * read a block beyond EOF could have left a "valid" zero-filled
805 : * buffer. Unfortunately, we have also seen this case occurring
806 : * because of buggy Linux kernels that sometimes return an
807 : * lseek(SEEK_END) result that doesn't account for a recent write. In
808 : * that situation, the pre-existing buffer would contain valid data
809 : * that we don't want to overwrite. Since the legitimate case should
810 : * always have left a zero-filled buffer, complain if not PageIsNew.
811 : */
812 0 : bufBlock = isLocalBuf ? LocalBufHdrGetBlock(bufHdr) : BufHdrGetBlock(bufHdr);
813 0 : if (!PageIsNew((Page) bufBlock))
814 0 : ereport(ERROR,
815 : (errmsg("unexpected data beyond EOF in block %u of relation %s",
816 : blockNum, relpath(smgr->smgr_rnode, forkNum)),
817 : errhint("This has been seen to occur with buggy kernels; consider updating your system.")));
818 :
819 : /*
820 : * We *must* do smgrextend before succeeding, else the page will not
821 : * be reserved by the kernel, and the next P_NEW call will decide to
822 : * return the same page. Clear the BM_VALID bit, do the StartBufferIO
823 : * call that BufferAlloc didn't, and proceed.
824 : */
825 0 : if (isLocalBuf)
826 : {
827 : /* Only need to adjust flags */
828 0 : uint32 buf_state = pg_atomic_read_u32(&bufHdr->state);
829 :
830 : Assert(buf_state & BM_VALID);
831 0 : buf_state &= ~BM_VALID;
832 0 : pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
833 : }
834 : else
835 : {
836 : /*
837 : * Loop to handle the very small possibility that someone re-sets
838 : * BM_VALID between our clearing it and StartBufferIO inspecting
839 : * it.
840 : */
841 : do
842 : {
843 0 : uint32 buf_state = LockBufHdr(bufHdr);
844 :
845 : Assert(buf_state & BM_VALID);
846 0 : buf_state &= ~BM_VALID;
847 0 : UnlockBufHdr(bufHdr, buf_state);
848 0 : } while (!StartBufferIO(bufHdr, true));
849 : }
850 : }
851 :
852 : /*
853 : * if we have gotten to this point, we have allocated a buffer for the
854 : * page but its contents are not yet valid. IO_IN_PROGRESS is set for it,
855 : * if it's a shared buffer.
856 : *
857 : * Note: if smgrextend fails, we will end up with a buffer that is
858 : * allocated but not marked BM_VALID. P_NEW will still select the same
859 : * block number (because the relation didn't get any longer on disk) and
860 : * so future attempts to extend the relation will find the same buffer (if
861 : * it's not been recycled) but come right back here to try smgrextend
862 : * again.
863 : */
864 : Assert(!(pg_atomic_read_u32(&bufHdr->state) & BM_VALID)); /* spinlock not needed */
865 :
866 826060 : bufBlock = isLocalBuf ? LocalBufHdrGetBlock(bufHdr) : BufHdrGetBlock(bufHdr);
867 :
868 826060 : if (isExtend)
869 : {
870 : /* new buffers are zero-filled */
871 350966 : MemSet((char *) bufBlock, 0, BLCKSZ);
872 : /* don't set checksum for all-zero page */
873 350966 : smgrextend(smgr, forkNum, blockNum, (char *) bufBlock, false);
874 :
875 : /*
876 : * NB: we're *not* doing a ScheduleBufferTagForWriteback here;
877 : * although we're essentially performing a write. At least on linux
878 : * doing so defeats the 'delayed allocation' mechanism, leading to
879 : * increased file fragmentation.
880 : */
881 : }
882 : else
883 : {
884 : /*
885 : * Read in the page, unless the caller intends to overwrite it and
886 : * just wants us to allocate a buffer.
887 : */
888 475094 : if (mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK)
889 2816 : MemSet((char *) bufBlock, 0, BLCKSZ);
890 : else
891 : {
892 : instr_time io_start,
893 : io_time;
894 :
895 472278 : if (track_io_timing)
896 0 : INSTR_TIME_SET_CURRENT(io_start);
897 :
898 472278 : smgrread(smgr, forkNum, blockNum, (char *) bufBlock);
899 :
900 472278 : if (track_io_timing)
901 : {
902 0 : INSTR_TIME_SET_CURRENT(io_time);
903 0 : INSTR_TIME_SUBTRACT(io_time, io_start);
904 0 : pgstat_count_buffer_read_time(INSTR_TIME_GET_MICROSEC(io_time));
905 0 : INSTR_TIME_ADD(pgBufferUsage.blk_read_time, io_time);
906 : }
907 :
908 : /* check for garbage data */
909 472278 : if (!PageIsVerified((Page) bufBlock, blockNum))
910 : {
911 0 : if (mode == RBM_ZERO_ON_ERROR || zero_damaged_pages)
912 : {
913 0 : ereport(WARNING,
914 : (errcode(ERRCODE_DATA_CORRUPTED),
915 : errmsg("invalid page in block %u of relation %s; zeroing out page",
916 : blockNum,
917 : relpath(smgr->smgr_rnode, forkNum))));
918 0 : MemSet((char *) bufBlock, 0, BLCKSZ);
919 : }
920 : else
921 0 : ereport(ERROR,
922 : (errcode(ERRCODE_DATA_CORRUPTED),
923 : errmsg("invalid page in block %u of relation %s",
924 : blockNum,
925 : relpath(smgr->smgr_rnode, forkNum))));
926 : }
927 : }
928 : }
929 :
930 : /*
931 : * In RBM_ZERO_AND_LOCK mode, grab the buffer content lock before marking
932 : * the page as valid, to make sure that no other backend sees the zeroed
933 : * page before the caller has had a chance to initialize it.
934 : *
935 : * Since no-one else can be looking at the page contents yet, there is no
936 : * difference between an exclusive lock and a cleanup-strength lock. (Note
937 : * that we cannot use LockBuffer() or LockBufferForCleanup() here, because
938 : * they assert that the buffer is already valid.)
939 : */
940 1095098 : if ((mode == RBM_ZERO_AND_LOCK || mode == RBM_ZERO_AND_CLEANUP_LOCK) &&
941 269038 : !isLocalBuf)
942 : {
943 245064 : LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_EXCLUSIVE);
944 : }
945 :
946 826060 : if (isLocalBuf)
947 : {
948 : /* Only need to adjust flags */
949 26286 : uint32 buf_state = pg_atomic_read_u32(&bufHdr->state);
950 :
951 26286 : buf_state |= BM_VALID;
952 26286 : pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
953 : }
954 : else
955 : {
956 : /* Set BM_VALID, terminate IO, and wake up any waiters */
957 799774 : TerminateBufferIO(bufHdr, false, BM_VALID);
958 : }
959 :
960 826060 : VacuumPageMiss++;
961 826060 : if (VacuumCostActive)
962 2290 : VacuumCostBalance += VacuumCostPageMiss;
963 :
964 : TRACE_POSTGRESQL_BUFFER_READ_DONE(forkNum, blockNum,
965 : smgr->smgr_rnode.node.spcNode,
966 : smgr->smgr_rnode.node.dbNode,
967 : smgr->smgr_rnode.node.relNode,
968 : smgr->smgr_rnode.backend,
969 : isExtend,
970 : found);
971 :
972 826060 : return BufferDescriptorGetBuffer(bufHdr);
973 : }
974 :
975 : /*
976 : * BufferAlloc -- subroutine for ReadBuffer. Handles lookup of a shared
977 : * buffer. If no buffer exists already, selects a replacement
978 : * victim and evicts the old page, but does NOT read in new page.
979 : *
980 : * "strategy" can be a buffer replacement strategy object, or NULL for
981 : * the default strategy. The selected buffer's usage_count is advanced when
982 : * using the default strategy, but otherwise possibly not (see PinBuffer).
983 : *
984 : * The returned buffer is pinned and is already marked as holding the
985 : * desired page. If it already did have the desired page, *foundPtr is
986 : * set true. Otherwise, *foundPtr is set false and the buffer is marked
987 : * as IO_IN_PROGRESS; ReadBuffer will now need to do I/O to fill it.
988 : *
989 : * *foundPtr is actually redundant with the buffer's BM_VALID flag, but
990 : * we keep it for simplicity in ReadBuffer.
991 : *
992 : * No locks are held either at entry or exit.
993 : */
994 : static BufferDesc *
995 80001220 : BufferAlloc(SMgrRelation smgr, char relpersistence, ForkNumber forkNum,
996 : BlockNumber blockNum,
997 : BufferAccessStrategy strategy,
998 : bool *foundPtr)
999 : {
1000 : BufferTag newTag; /* identity of requested block */
1001 : uint32 newHash; /* hash value for newTag */
1002 : LWLock *newPartitionLock; /* buffer partition lock for it */
1003 : BufferTag oldTag; /* previous identity of selected buffer */
1004 : uint32 oldHash; /* hash value for oldTag */
1005 : LWLock *oldPartitionLock; /* buffer partition lock for it */
1006 : uint32 oldFlags;
1007 : int buf_id;
1008 : BufferDesc *buf;
1009 : bool valid;
1010 : uint32 buf_state;
1011 :
1012 : /* create a tag so we can lookup the buffer */
1013 80001220 : INIT_BUFFERTAG(newTag, smgr->smgr_rnode.node, forkNum, blockNum);
1014 :
1015 : /* determine its hash code and partition lock ID */
1016 80001220 : newHash = BufTableHashCode(&newTag);
1017 80001220 : newPartitionLock = BufMappingPartitionLock(newHash);
1018 :
1019 : /* see if the block is in the buffer pool already */
1020 80001220 : LWLockAcquire(newPartitionLock, LW_SHARED);
1021 80001220 : buf_id = BufTableLookup(&newTag, newHash);
1022 80001220 : if (buf_id >= 0)
1023 : {
1024 : /*
1025 : * Found it. Now, pin the buffer so no one can steal it from the
1026 : * buffer pool, and check to see if the correct data has been loaded
1027 : * into the buffer.
1028 : */
1029 79201446 : buf = GetBufferDescriptor(buf_id);
1030 :
1031 79201446 : valid = PinBuffer(buf, strategy);
1032 :
1033 : /* Can release the mapping lock as soon as we've pinned it */
1034 79201446 : LWLockRelease(newPartitionLock);
1035 :
1036 79201446 : *foundPtr = true;
1037 :
1038 79201446 : if (!valid)
1039 : {
1040 : /*
1041 : * We can only get here if (a) someone else is still reading in
1042 : * the page, or (b) a previous read attempt failed. We have to
1043 : * wait for any active read attempt to finish, and then set up our
1044 : * own read attempt if the page is still not BM_VALID.
1045 : * StartBufferIO does it all.
1046 : */
1047 6 : if (StartBufferIO(buf, true))
1048 : {
1049 : /*
1050 : * If we get here, previous attempts to read the buffer must
1051 : * have failed ... but we shall bravely try again.
1052 : */
1053 0 : *foundPtr = false;
1054 : }
1055 : }
1056 :
1057 79201446 : return buf;
1058 : }
1059 :
1060 : /*
1061 : * Didn't find it in the buffer pool. We'll have to initialize a new
1062 : * buffer. Remember to unlock the mapping lock while doing the work.
1063 : */
1064 799774 : LWLockRelease(newPartitionLock);
1065 :
1066 : /* Loop here in case we have to try another victim buffer */
1067 : for (;;)
1068 : {
1069 : /*
1070 : * Ensure, while the spinlock's not yet held, that there's a free
1071 : * refcount entry.
1072 : */
1073 802182 : ReservePrivateRefCountEntry();
1074 :
1075 : /*
1076 : * Select a victim buffer. The buffer is returned with its header
1077 : * spinlock still held!
1078 : */
1079 800978 : buf = StrategyGetBuffer(strategy, &buf_state);
1080 :
1081 : Assert(BUF_STATE_GET_REFCOUNT(buf_state) == 0);
1082 :
1083 : /* Must copy buffer flags while we still hold the spinlock */
1084 800978 : oldFlags = buf_state & BUF_FLAG_MASK;
1085 :
1086 : /* Pin the buffer and then release the buffer spinlock */
1087 800978 : PinBuffer_Locked(buf);
1088 :
1089 : /*
1090 : * If the buffer was dirty, try to write it out. There is a race
1091 : * condition here, in that someone might dirty it after we released it
1092 : * above, or even while we are writing it out (since our share-lock
1093 : * won't prevent hint-bit updates). We will recheck the dirty bit
1094 : * after re-locking the buffer header.
1095 : */
1096 800978 : if (oldFlags & BM_DIRTY)
1097 : {
1098 : /*
1099 : * We need a share-lock on the buffer contents to write it out
1100 : * (else we might write invalid data, eg because someone else is
1101 : * compacting the page contents while we write). We must use a
1102 : * conditional lock acquisition here to avoid deadlock. Even
1103 : * though the buffer was not pinned (and therefore surely not
1104 : * locked) when StrategyGetBuffer returned it, someone else could
1105 : * have pinned and exclusive-locked it by the time we get here. If
1106 : * we try to get the lock unconditionally, we'd block waiting for
1107 : * them; if they later block waiting for us, deadlock ensues.
1108 : * (This has been observed to happen when two backends are both
1109 : * trying to split btree index pages, and the second one just
1110 : * happens to be trying to split the page the first one got from
1111 : * StrategyGetBuffer.)
1112 : */
1113 11510 : if (LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf),
1114 : LW_SHARED))
1115 : {
1116 : /*
1117 : * If using a nondefault strategy, and writing the buffer
1118 : * would require a WAL flush, let the strategy decide whether
1119 : * to go ahead and write/reuse the buffer or to choose another
1120 : * victim. We need lock to inspect the page LSN, so this
1121 : * can't be done inside StrategyGetBuffer.
1122 : */
1123 11510 : if (strategy != NULL)
1124 : {
1125 : XLogRecPtr lsn;
1126 :
1127 : /* Read the LSN while holding buffer header lock */
1128 4044 : buf_state = LockBufHdr(buf);
1129 4044 : lsn = BufferGetLSN(buf);
1130 4044 : UnlockBufHdr(buf, buf_state);
1131 :
1132 5408 : if (XLogNeedsFlush(lsn) &&
1133 1364 : StrategyRejectBuffer(strategy, buf))
1134 : {
1135 : /* Drop lock/pin and loop around for another buffer */
1136 1204 : LWLockRelease(BufferDescriptorGetContentLock(buf));
1137 1204 : UnpinBuffer(buf, true);
1138 1204 : continue;
1139 : }
1140 : }
1141 :
1142 : /* OK, do the I/O */
1143 : TRACE_POSTGRESQL_BUFFER_WRITE_DIRTY_START(forkNum, blockNum,
1144 : smgr->smgr_rnode.node.spcNode,
1145 : smgr->smgr_rnode.node.dbNode,
1146 : smgr->smgr_rnode.node.relNode);
1147 :
1148 10306 : FlushBuffer(buf, NULL);
1149 10306 : LWLockRelease(BufferDescriptorGetContentLock(buf));
1150 :
1151 10306 : ScheduleBufferTagForWriteback(&BackendWritebackContext,
1152 : &buf->tag);
1153 :
1154 : TRACE_POSTGRESQL_BUFFER_WRITE_DIRTY_DONE(forkNum, blockNum,
1155 : smgr->smgr_rnode.node.spcNode,
1156 : smgr->smgr_rnode.node.dbNode,
1157 : smgr->smgr_rnode.node.relNode);
1158 : }
1159 : else
1160 : {
1161 : /*
1162 : * Someone else has locked the buffer, so give it up and loop
1163 : * back to get another one.
1164 : */
1165 0 : UnpinBuffer(buf, true);
1166 0 : continue;
1167 : }
1168 : }
1169 :
1170 : /*
1171 : * To change the association of a valid buffer, we'll need to have
1172 : * exclusive lock on both the old and new mapping partitions.
1173 : */
1174 799774 : if (oldFlags & BM_TAG_VALID)
1175 : {
1176 : /*
1177 : * Need to compute the old tag's hashcode and partition lock ID.
1178 : * XXX is it worth storing the hashcode in BufferDesc so we need
1179 : * not recompute it here? Probably not.
1180 : */
1181 80012 : oldTag = buf->tag;
1182 80012 : oldHash = BufTableHashCode(&oldTag);
1183 80012 : oldPartitionLock = BufMappingPartitionLock(oldHash);
1184 :
1185 : /*
1186 : * Must lock the lower-numbered partition first to avoid
1187 : * deadlocks.
1188 : */
1189 80012 : if (oldPartitionLock < newPartitionLock)
1190 : {
1191 39794 : LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
1192 39794 : LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
1193 : }
1194 40218 : else if (oldPartitionLock > newPartitionLock)
1195 : {
1196 39562 : LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
1197 39562 : LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
1198 : }
1199 : else
1200 : {
1201 : /* only one partition, only one lock */
1202 656 : LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
1203 : }
1204 : }
1205 : else
1206 : {
1207 : /* if it wasn't valid, we need only the new partition */
1208 719762 : LWLockAcquire(newPartitionLock, LW_EXCLUSIVE);
1209 : /* remember we have no old-partition lock or tag */
1210 719762 : oldPartitionLock = NULL;
1211 : /* keep the compiler quiet about uninitialized variables */
1212 719762 : oldHash = 0;
1213 : }
1214 :
1215 : /*
1216 : * Try to make a hashtable entry for the buffer under its new tag.
1217 : * This could fail because while we were writing someone else
1218 : * allocated another buffer for the same block we want to read in.
1219 : * Note that we have not yet removed the hashtable entry for the old
1220 : * tag.
1221 : */
1222 799774 : buf_id = BufTableInsert(&newTag, newHash, buf->buf_id);
1223 :
1224 799774 : if (buf_id >= 0)
1225 : {
1226 : /*
1227 : * Got a collision. Someone has already done what we were about to
1228 : * do. We'll just handle this as if it were found in the buffer
1229 : * pool in the first place. First, give up the buffer we were
1230 : * planning to use.
1231 : */
1232 0 : UnpinBuffer(buf, true);
1233 :
1234 : /* Can give up that buffer's mapping partition lock now */
1235 0 : if (oldPartitionLock != NULL &&
1236 : oldPartitionLock != newPartitionLock)
1237 0 : LWLockRelease(oldPartitionLock);
1238 :
1239 : /* remaining code should match code at top of routine */
1240 :
1241 0 : buf = GetBufferDescriptor(buf_id);
1242 :
1243 0 : valid = PinBuffer(buf, strategy);
1244 :
1245 : /* Can release the mapping lock as soon as we've pinned it */
1246 0 : LWLockRelease(newPartitionLock);
1247 :
1248 0 : *foundPtr = true;
1249 :
1250 0 : if (!valid)
1251 : {
1252 : /*
1253 : * We can only get here if (a) someone else is still reading
1254 : * in the page, or (b) a previous read attempt failed. We
1255 : * have to wait for any active read attempt to finish, and
1256 : * then set up our own read attempt if the page is still not
1257 : * BM_VALID. StartBufferIO does it all.
1258 : */
1259 0 : if (StartBufferIO(buf, true))
1260 : {
1261 : /*
1262 : * If we get here, previous attempts to read the buffer
1263 : * must have failed ... but we shall bravely try again.
1264 : */
1265 0 : *foundPtr = false;
1266 : }
1267 : }
1268 :
1269 0 : return buf;
1270 : }
1271 :
1272 : /*
1273 : * Need to lock the buffer header too in order to change its tag.
1274 : */
1275 799774 : buf_state = LockBufHdr(buf);
1276 :
1277 : /*
1278 : * Somebody could have pinned or re-dirtied the buffer while we were
1279 : * doing the I/O and making the new hashtable entry. If so, we can't
1280 : * recycle this buffer; we must undo everything we've done and start
1281 : * over with a new victim buffer.
1282 : */
1283 799774 : oldFlags = buf_state & BUF_FLAG_MASK;
1284 799774 : if (BUF_STATE_GET_REFCOUNT(buf_state) == 1 && !(oldFlags & BM_DIRTY))
1285 799774 : break;
1286 :
1287 0 : UnlockBufHdr(buf, buf_state);
1288 0 : BufTableDelete(&newTag, newHash);
1289 0 : if (oldPartitionLock != NULL &&
1290 : oldPartitionLock != newPartitionLock)
1291 0 : LWLockRelease(oldPartitionLock);
1292 0 : LWLockRelease(newPartitionLock);
1293 0 : UnpinBuffer(buf, true);
1294 : }
1295 :
1296 : /*
1297 : * Okay, it's finally safe to rename the buffer.
1298 : *
1299 : * Clearing BM_VALID here is necessary, clearing the dirtybits is just
1300 : * paranoia. We also reset the usage_count since any recency of use of
1301 : * the old content is no longer relevant. (The usage_count starts out at
1302 : * 1 so that the buffer can survive one clock-sweep pass.)
1303 : *
1304 : * Make sure BM_PERMANENT is set for buffers that must be written at every
1305 : * checkpoint. Unlogged buffers only need to be written at shutdown
1306 : * checkpoints, except for their "init" forks, which need to be treated
1307 : * just like permanent relations.
1308 : */
1309 799774 : buf->tag = newTag;
1310 799774 : buf_state &= ~(BM_VALID | BM_DIRTY | BM_JUST_DIRTIED |
1311 : BM_CHECKPOINT_NEEDED | BM_IO_ERROR | BM_PERMANENT |
1312 : BUF_USAGECOUNT_MASK);
1313 799774 : if (relpersistence == RELPERSISTENCE_PERMANENT || forkNum == INIT_FORKNUM)
1314 791882 : buf_state |= BM_TAG_VALID | BM_PERMANENT | BUF_USAGECOUNT_ONE;
1315 : else
1316 7892 : buf_state |= BM_TAG_VALID | BUF_USAGECOUNT_ONE;
1317 :
1318 799774 : UnlockBufHdr(buf, buf_state);
1319 :
1320 799774 : if (oldPartitionLock != NULL)
1321 : {
1322 80012 : BufTableDelete(&oldTag, oldHash);
1323 80012 : if (oldPartitionLock != newPartitionLock)
1324 79356 : LWLockRelease(oldPartitionLock);
1325 : }
1326 :
1327 799774 : LWLockRelease(newPartitionLock);
1328 :
1329 : /*
1330 : * Buffer contents are currently invalid. Try to get the io_in_progress
1331 : * lock. If StartBufferIO returns false, then someone else managed to
1332 : * read it before we did, so there's nothing left for BufferAlloc() to do.
1333 : */
1334 799774 : if (StartBufferIO(buf, true))
1335 799774 : *foundPtr = false;
1336 : else
1337 0 : *foundPtr = true;
1338 :
1339 799774 : return buf;
1340 : }
1341 :
1342 : /*
1343 : * InvalidateBuffer -- mark a shared buffer invalid and return it to the
1344 : * freelist.
1345 : *
1346 : * The buffer header spinlock must be held at entry. We drop it before
1347 : * returning. (This is sane because the caller must have locked the
1348 : * buffer in order to be sure it should be dropped.)
1349 : *
1350 : * This is used only in contexts such as dropping a relation. We assume
1351 : * that no other backend could possibly be interested in using the page,
1352 : * so the only reason the buffer might be pinned is if someone else is
1353 : * trying to write it out. We have to let them finish before we can
1354 : * reclaim the buffer.
1355 : *
1356 : * The buffer could get reclaimed by someone else while we are waiting
1357 : * to acquire the necessary locks; if so, don't mess it up.
1358 : */
1359 : static void
1360 115692 : InvalidateBuffer(BufferDesc *buf)
1361 : {
1362 : BufferTag oldTag;
1363 : uint32 oldHash; /* hash value for oldTag */
1364 : LWLock *oldPartitionLock; /* buffer partition lock for it */
1365 : uint32 oldFlags;
1366 : uint32 buf_state;
1367 :
1368 : /* Save the original buffer tag before dropping the spinlock */
1369 115692 : oldTag = buf->tag;
1370 :
1371 115692 : buf_state = pg_atomic_read_u32(&buf->state);
1372 : Assert(buf_state & BM_LOCKED);
1373 115692 : UnlockBufHdr(buf, buf_state);
1374 :
1375 : /*
1376 : * Need to compute the old tag's hashcode and partition lock ID. XXX is it
1377 : * worth storing the hashcode in BufferDesc so we need not recompute it
1378 : * here? Probably not.
1379 : */
1380 115692 : oldHash = BufTableHashCode(&oldTag);
1381 115692 : oldPartitionLock = BufMappingPartitionLock(oldHash);
1382 :
1383 : retry:
1384 :
1385 : /*
1386 : * Acquire exclusive mapping lock in preparation for changing the buffer's
1387 : * association.
1388 : */
1389 115692 : LWLockAcquire(oldPartitionLock, LW_EXCLUSIVE);
1390 :
1391 : /* Re-lock the buffer header */
1392 115692 : buf_state = LockBufHdr(buf);
1393 :
1394 : /* If it's changed while we were waiting for lock, do nothing */
1395 115692 : if (!BUFFERTAGS_EQUAL(buf->tag, oldTag))
1396 : {
1397 0 : UnlockBufHdr(buf, buf_state);
1398 0 : LWLockRelease(oldPartitionLock);
1399 0 : return;
1400 : }
1401 :
1402 : /*
1403 : * We assume the only reason for it to be pinned is that someone else is
1404 : * flushing the page out. Wait for them to finish. (This could be an
1405 : * infinite loop if the refcount is messed up... it would be nice to time
1406 : * out after awhile, but there seems no way to be sure how many loops may
1407 : * be needed. Note that if the other guy has pinned the buffer but not
1408 : * yet done StartBufferIO, WaitIO will fall through and we'll effectively
1409 : * be busy-looping here.)
1410 : */
1411 115692 : if (BUF_STATE_GET_REFCOUNT(buf_state) != 0)
1412 : {
1413 0 : UnlockBufHdr(buf, buf_state);
1414 0 : LWLockRelease(oldPartitionLock);
1415 : /* safety check: should definitely not be our *own* pin */
1416 0 : if (GetPrivateRefCount(BufferDescriptorGetBuffer(buf)) > 0)
1417 0 : elog(ERROR, "buffer is pinned in InvalidateBuffer");
1418 0 : WaitIO(buf);
1419 0 : goto retry;
1420 : }
1421 :
1422 : /*
1423 : * Clear out the buffer's tag and flags. We must do this to ensure that
1424 : * linear scans of the buffer array don't think the buffer is valid.
1425 : */
1426 115692 : oldFlags = buf_state & BUF_FLAG_MASK;
1427 115692 : CLEAR_BUFFERTAG(buf->tag);
1428 115692 : buf_state &= ~(BUF_FLAG_MASK | BUF_USAGECOUNT_MASK);
1429 115692 : UnlockBufHdr(buf, buf_state);
1430 :
1431 : /*
1432 : * Remove the buffer from the lookup hashtable, if it was in there.
1433 : */
1434 115692 : if (oldFlags & BM_TAG_VALID)
1435 115692 : BufTableDelete(&oldTag, oldHash);
1436 :
1437 : /*
1438 : * Done with mapping lock.
1439 : */
1440 115692 : LWLockRelease(oldPartitionLock);
1441 :
1442 : /*
1443 : * Insert the buffer at the head of the list of free buffers.
1444 : */
1445 115692 : StrategyFreeBuffer(buf);
1446 : }
1447 :
1448 : /*
1449 : * MarkBufferDirty
1450 : *
1451 : * Marks buffer contents as dirty (actual write happens later).
1452 : *
1453 : * Buffer must be pinned and exclusive-locked. (If caller does not hold
1454 : * exclusive lock, then somebody could be in process of writing the buffer,
1455 : * leading to risk of bad data written to disk.)
1456 : */
1457 : void
1458 35584282 : MarkBufferDirty(Buffer buffer)
1459 : {
1460 : BufferDesc *bufHdr;
1461 : uint32 buf_state;
1462 : uint32 old_buf_state;
1463 :
1464 35584282 : if (!BufferIsValid(buffer))
1465 0 : elog(ERROR, "bad buffer ID: %d", buffer);
1466 :
1467 35584282 : if (BufferIsLocal(buffer))
1468 : {
1469 602380 : MarkLocalBufferDirty(buffer);
1470 602380 : return;
1471 : }
1472 :
1473 34981902 : bufHdr = GetBufferDescriptor(buffer - 1);
1474 :
1475 : Assert(BufferIsPinned(buffer));
1476 : Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
1477 : LW_EXCLUSIVE));
1478 :
1479 34981902 : old_buf_state = pg_atomic_read_u32(&bufHdr->state);
1480 : for (;;)
1481 : {
1482 34982362 : if (old_buf_state & BM_LOCKED)
1483 0 : old_buf_state = WaitBufHdrUnlocked(bufHdr);
1484 :
1485 34982132 : buf_state = old_buf_state;
1486 :
1487 : Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
1488 34982132 : buf_state |= BM_DIRTY | BM_JUST_DIRTIED;
1489 :
1490 34982132 : if (pg_atomic_compare_exchange_u32(&bufHdr->state, &old_buf_state,
1491 : buf_state))
1492 34981902 : break;
1493 : }
1494 :
1495 : /*
1496 : * If the buffer was not dirty already, do vacuum accounting.
1497 : */
1498 34981902 : if (!(old_buf_state & BM_DIRTY))
1499 : {
1500 515858 : VacuumPageDirty++;
1501 515858 : pgBufferUsage.shared_blks_dirtied++;
1502 515858 : if (VacuumCostActive)
1503 2034 : VacuumCostBalance += VacuumCostPageDirty;
1504 : }
1505 : }
1506 :
1507 : /*
1508 : * ReleaseAndReadBuffer -- combine ReleaseBuffer() and ReadBuffer()
1509 : *
1510 : * Formerly, this saved one cycle of acquiring/releasing the BufMgrLock
1511 : * compared to calling the two routines separately. Now it's mainly just
1512 : * a convenience function. However, if the passed buffer is valid and
1513 : * already contains the desired block, we just return it as-is; and that
1514 : * does save considerable work compared to a full release and reacquire.
1515 : *
1516 : * Note: it is OK to pass buffer == InvalidBuffer, indicating that no old
1517 : * buffer actually needs to be released. This case is the same as ReadBuffer,
1518 : * but can save some tests in the caller.
1519 : */
1520 : Buffer
1521 33438658 : ReleaseAndReadBuffer(Buffer buffer,
1522 : Relation relation,
1523 : BlockNumber blockNum)
1524 : {
1525 33438658 : ForkNumber forkNum = MAIN_FORKNUM;
1526 : BufferDesc *bufHdr;
1527 :
1528 33438658 : if (BufferIsValid(buffer))
1529 : {
1530 : Assert(BufferIsPinned(buffer));
1531 25941308 : if (BufferIsLocal(buffer))
1532 : {
1533 3366 : bufHdr = GetLocalBufferDescriptor(-buffer - 1);
1534 5732 : if (bufHdr->tag.blockNum == blockNum &&
1535 7098 : RelFileNodeEquals(bufHdr->tag.rnode, relation->rd_node) &&
1536 2366 : bufHdr->tag.forkNum == forkNum)
1537 2366 : return buffer;
1538 1000 : ResourceOwnerForgetBuffer(CurrentResourceOwner, buffer);
1539 1000 : LocalRefCount[-buffer - 1]--;
1540 : }
1541 : else
1542 : {
1543 25937942 : bufHdr = GetBufferDescriptor(buffer - 1);
1544 : /* we have pin, so it's ok to examine tag without spinlock */
1545 31980728 : if (bufHdr->tag.blockNum == blockNum &&
1546 18128358 : RelFileNodeEquals(bufHdr->tag.rnode, relation->rd_node) &&
1547 6042786 : bufHdr->tag.forkNum == forkNum)
1548 6042786 : return buffer;
1549 19895156 : UnpinBuffer(bufHdr, true);
1550 : }
1551 : }
1552 :
1553 27393506 : return ReadBuffer(relation, blockNum);
1554 : }
1555 :
1556 : /*
1557 : * PinBuffer -- make buffer unavailable for replacement.
1558 : *
1559 : * For the default access strategy, the buffer's usage_count is incremented
1560 : * when we first pin it; for other strategies we just make sure the usage_count
1561 : * isn't zero. (The idea of the latter is that we don't want synchronized
1562 : * heap scans to inflate the count, but we need it to not be zero to discourage
1563 : * other backends from stealing buffers from our ring. As long as we cycle
1564 : * through the ring faster than the global clock-sweep cycles, buffers in
1565 : * our ring won't be chosen as victims for replacement by other backends.)
1566 : *
1567 : * This should be applied only to shared buffers, never local ones.
1568 : *
1569 : * Since buffers are pinned/unpinned very frequently, pin buffers without
1570 : * taking the buffer header lock; instead update the state variable in loop of
1571 : * CAS operations. Hopefully it's just a single CAS.
1572 : *
1573 : * Note that ResourceOwnerEnlargeBuffers must have been done already.
1574 : *
1575 : * Returns true if buffer is BM_VALID, else false. This provision allows
1576 : * some callers to avoid an extra spinlock cycle.
1577 : */
1578 : static bool
1579 79201446 : PinBuffer(BufferDesc *buf, BufferAccessStrategy strategy)
1580 : {
1581 79201446 : Buffer b = BufferDescriptorGetBuffer(buf);
1582 : bool result;
1583 : PrivateRefCountEntry *ref;
1584 :
1585 79201446 : ref = GetPrivateRefCountEntry(b, true);
1586 :
1587 79201446 : if (ref == NULL)
1588 : {
1589 : uint32 buf_state;
1590 : uint32 old_buf_state;
1591 :
1592 77341950 : ReservePrivateRefCountEntry();
1593 77341950 : ref = NewPrivateRefCountEntry(b);
1594 :
1595 77341950 : old_buf_state = pg_atomic_read_u32(&buf->state);
1596 : for (;;)
1597 : {
1598 77381934 : if (old_buf_state & BM_LOCKED)
1599 2 : old_buf_state = WaitBufHdrUnlocked(buf);
1600 :
1601 77361942 : buf_state = old_buf_state;
1602 :
1603 : /* increase refcount */
1604 77361942 : buf_state += BUF_REFCOUNT_ONE;
1605 :
1606 77361942 : if (strategy == NULL)
1607 : {
1608 : /* Default case: increase usagecount unless already max. */
1609 76819870 : if (BUF_STATE_GET_USAGECOUNT(buf_state) < BM_MAX_USAGE_COUNT)
1610 2282508 : buf_state += BUF_USAGECOUNT_ONE;
1611 : }
1612 : else
1613 : {
1614 : /*
1615 : * Ring buffers shouldn't evict others from pool. Thus we
1616 : * don't make usagecount more than 1.
1617 : */
1618 542072 : if (BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
1619 790 : buf_state += BUF_USAGECOUNT_ONE;
1620 : }
1621 :
1622 77361942 : if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state,
1623 : buf_state))
1624 : {
1625 77341950 : result = (buf_state & BM_VALID) != 0;
1626 77341950 : break;
1627 : }
1628 : }
1629 : }
1630 : else
1631 : {
1632 : /* If we previously pinned the buffer, it must surely be valid */
1633 1859496 : result = true;
1634 : }
1635 :
1636 79201446 : ref->refcount++;
1637 : Assert(ref->refcount > 0);
1638 79201446 : ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
1639 79201446 : return result;
1640 : }
1641 :
1642 : /*
1643 : * PinBuffer_Locked -- as above, but caller already locked the buffer header.
1644 : * The spinlock is released before return.
1645 : *
1646 : * As this function is called with the spinlock held, the caller has to
1647 : * previously call ReservePrivateRefCountEntry().
1648 : *
1649 : * Currently, no callers of this function want to modify the buffer's
1650 : * usage_count at all, so there's no need for a strategy parameter.
1651 : * Also we don't bother with a BM_VALID test (the caller could check that for
1652 : * itself).
1653 : *
1654 : * Also all callers only ever use this function when it's known that the
1655 : * buffer can't have a preexisting pin by this backend. That allows us to skip
1656 : * searching the private refcount array & hash, which is a boon, because the
1657 : * spinlock is still held.
1658 : *
1659 : * Note: use of this routine is frequently mandatory, not just an optimization
1660 : * to save a spin lock/unlock cycle, because we need to pin a buffer before
1661 : * its state can change under us.
1662 : */
1663 : static void
1664 1265510 : PinBuffer_Locked(BufferDesc *buf)
1665 : {
1666 : Buffer b;
1667 : PrivateRefCountEntry *ref;
1668 : uint32 buf_state;
1669 :
1670 : /*
1671 : * As explained, We don't expect any preexisting pins. That allows us to
1672 : * manipulate the PrivateRefCount after releasing the spinlock
1673 : */
1674 : Assert(GetPrivateRefCountEntry(BufferDescriptorGetBuffer(buf), false) == NULL);
1675 :
1676 : /*
1677 : * Since we hold the buffer spinlock, we can update the buffer state and
1678 : * release the lock in one operation.
1679 : */
1680 1265510 : buf_state = pg_atomic_read_u32(&buf->state);
1681 : Assert(buf_state & BM_LOCKED);
1682 1265510 : buf_state += BUF_REFCOUNT_ONE;
1683 1265510 : UnlockBufHdr(buf, buf_state);
1684 :
1685 1265510 : b = BufferDescriptorGetBuffer(buf);
1686 :
1687 1265510 : ref = NewPrivateRefCountEntry(b);
1688 1265510 : ref->refcount++;
1689 :
1690 1265510 : ResourceOwnerRememberBuffer(CurrentResourceOwner, b);
1691 1265510 : }
1692 :
1693 : /*
1694 : * UnpinBuffer -- make buffer available for replacement.
1695 : *
1696 : * This should be applied only to shared buffers, never local ones.
1697 : *
1698 : * Most but not all callers want CurrentResourceOwner to be adjusted.
1699 : * Those that don't should pass fixOwner = false.
1700 : */
1701 : static void
1702 93334742 : UnpinBuffer(BufferDesc *buf, bool fixOwner)
1703 : {
1704 : PrivateRefCountEntry *ref;
1705 93334742 : Buffer b = BufferDescriptorGetBuffer(buf);
1706 :
1707 : /* not moving as we're likely deleting it soon anyway */
1708 93334742 : ref = GetPrivateRefCountEntry(b, false);
1709 : Assert(ref != NULL);
1710 :
1711 93334742 : if (fixOwner)
1712 93334742 : ResourceOwnerForgetBuffer(CurrentResourceOwner, b);
1713 :
1714 : Assert(ref->refcount > 0);
1715 93334742 : ref->refcount--;
1716 93334742 : if (ref->refcount == 0)
1717 : {
1718 : uint32 buf_state;
1719 : uint32 old_buf_state;
1720 :
1721 : /* I'd better not still hold any locks on the buffer */
1722 : Assert(!LWLockHeldByMe(BufferDescriptorGetContentLock(buf)));
1723 : Assert(!LWLockHeldByMe(BufferDescriptorGetIOLock(buf)));
1724 :
1725 : /*
1726 : * Decrement the shared reference count.
1727 : *
1728 : * Since buffer spinlock holder can update status using just write,
1729 : * it's not safe to use atomic decrement here; thus use a CAS loop.
1730 : */
1731 78607460 : old_buf_state = pg_atomic_read_u32(&buf->state);
1732 : for (;;)
1733 : {
1734 78652136 : if (old_buf_state & BM_LOCKED)
1735 2 : old_buf_state = WaitBufHdrUnlocked(buf);
1736 :
1737 78629798 : buf_state = old_buf_state;
1738 :
1739 78629798 : buf_state -= BUF_REFCOUNT_ONE;
1740 :
1741 78629798 : if (pg_atomic_compare_exchange_u32(&buf->state, &old_buf_state,
1742 : buf_state))
1743 78607460 : break;
1744 : }
1745 :
1746 : /* Support LockBufferForCleanup() */
1747 78607460 : if (buf_state & BM_PIN_COUNT_WAITER)
1748 : {
1749 : /*
1750 : * Acquire the buffer header lock, re-check that there's a waiter.
1751 : * Another backend could have unpinned this buffer, and already
1752 : * woken up the waiter. There's no danger of the buffer being
1753 : * replaced after we unpinned it above, as it's pinned by the
1754 : * waiter.
1755 : */
1756 0 : buf_state = LockBufHdr(buf);
1757 :
1758 0 : if ((buf_state & BM_PIN_COUNT_WAITER) &&
1759 0 : BUF_STATE_GET_REFCOUNT(buf_state) == 1)
1760 0 : {
1761 : /* we just released the last pin other than the waiter's */
1762 0 : int wait_backend_pid = buf->wait_backend_pid;
1763 :
1764 0 : buf_state &= ~BM_PIN_COUNT_WAITER;
1765 0 : UnlockBufHdr(buf, buf_state);
1766 0 : ProcSendSignal(wait_backend_pid);
1767 : }
1768 : else
1769 0 : UnlockBufHdr(buf, buf_state);
1770 : }
1771 78607460 : ForgetPrivateRefCountEntry(ref);
1772 : }
1773 93334742 : }
1774 :
1775 : /*
1776 : * BufferSync -- Write out all dirty buffers in the pool.
1777 : *
1778 : * This is called at checkpoint time to write out all dirty shared buffers.
1779 : * The checkpoint request flags should be passed in. If CHECKPOINT_IMMEDIATE
1780 : * is set, we disable delays between writes; if CHECKPOINT_IS_SHUTDOWN,
1781 : * CHECKPOINT_END_OF_RECOVERY or CHECKPOINT_FLUSH_ALL is set, we write even
1782 : * unlogged buffers, which are otherwise skipped. The remaining flags
1783 : * currently have no effect here.
1784 : */
1785 : static void
1786 2832 : BufferSync(int flags)
1787 : {
1788 : uint32 buf_state;
1789 : int buf_id;
1790 : int num_to_scan;
1791 : int num_spaces;
1792 : int num_processed;
1793 : int num_written;
1794 2832 : CkptTsStatus *per_ts_stat = NULL;
1795 : Oid last_tsid;
1796 : binaryheap *ts_heap;
1797 : int i;
1798 2832 : int mask = BM_DIRTY;
1799 : WritebackContext wb_context;
1800 :
1801 : /* Make sure we can handle the pin inside SyncOneBuffer */
1802 2832 : ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
1803 :
1804 : /*
1805 : * Unless this is a shutdown checkpoint or we have been explicitly told,
1806 : * we write only permanent, dirty buffers. But at shutdown or end of
1807 : * recovery, we write all dirty buffers.
1808 : */
1809 2832 : if (!((flags & (CHECKPOINT_IS_SHUTDOWN | CHECKPOINT_END_OF_RECOVERY |
1810 : CHECKPOINT_FLUSH_ALL))))
1811 956 : mask |= BM_PERMANENT;
1812 :
1813 : /*
1814 : * Loop over all buffers, and mark the ones that need to be written with
1815 : * BM_CHECKPOINT_NEEDED. Count them as we go (num_to_scan), so that we
1816 : * can estimate how much work needs to be done.
1817 : *
1818 : * This allows us to write only those pages that were dirty when the
1819 : * checkpoint began, and not those that get dirtied while it proceeds.
1820 : * Whenever a page with BM_CHECKPOINT_NEEDED is written out, either by us
1821 : * later in this function, or by normal backends or the bgwriter cleaning
1822 : * scan, the flag is cleared. Any buffer dirtied after this point won't
1823 : * have the flag set.
1824 : *
1825 : * Note that if we fail to write some buffer, we may leave buffers with
1826 : * BM_CHECKPOINT_NEEDED still set. This is OK since any such buffer would
1827 : * certainly need to be written for the next checkpoint attempt, too.
1828 : */
1829 2832 : num_to_scan = 0;
1830 42630704 : for (buf_id = 0; buf_id < NBuffers; buf_id++)
1831 : {
1832 42627872 : BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
1833 :
1834 : /*
1835 : * Header spinlock is enough to examine BM_DIRTY, see comment in
1836 : * SyncOneBuffer.
1837 : */
1838 42627872 : buf_state = LockBufHdr(bufHdr);
1839 :
1840 42627872 : if ((buf_state & mask) == mask)
1841 : {
1842 : CkptSortItem *item;
1843 :
1844 461782 : buf_state |= BM_CHECKPOINT_NEEDED;
1845 :
1846 461782 : item = &CkptBufferIds[num_to_scan++];
1847 461782 : item->buf_id = buf_id;
1848 461782 : item->tsId = bufHdr->tag.rnode.spcNode;
1849 461782 : item->relNode = bufHdr->tag.rnode.relNode;
1850 461782 : item->forkNum = bufHdr->tag.forkNum;
1851 461782 : item->blockNum = bufHdr->tag.blockNum;
1852 : }
1853 :
1854 42627872 : UnlockBufHdr(bufHdr, buf_state);
1855 : }
1856 :
1857 2832 : if (num_to_scan == 0)
1858 998 : return; /* nothing to do */
1859 :
1860 1834 : WritebackContextInit(&wb_context, &checkpoint_flush_after);
1861 :
1862 : TRACE_POSTGRESQL_BUFFER_SYNC_START(NBuffers, num_to_scan);
1863 :
1864 : /*
1865 : * Sort buffers that need to be written to reduce the likelihood of random
1866 : * IO. The sorting is also important for the implementation of balancing
1867 : * writes between tablespaces. Without balancing writes we'd potentially
1868 : * end up writing to the tablespaces one-by-one; possibly overloading the
1869 : * underlying system.
1870 : */
1871 1834 : qsort(CkptBufferIds, num_to_scan, sizeof(CkptSortItem),
1872 : ckpt_buforder_comparator);
1873 :
1874 1834 : num_spaces = 0;
1875 :
1876 : /*
1877 : * Allocate progress status for each tablespace with buffers that need to
1878 : * be flushed. This requires the to-be-flushed array to be sorted.
1879 : */
1880 1834 : last_tsid = InvalidOid;
1881 463616 : for (i = 0; i < num_to_scan; i++)
1882 : {
1883 : CkptTsStatus *s;
1884 : Oid cur_tsid;
1885 :
1886 461782 : cur_tsid = CkptBufferIds[i].tsId;
1887 :
1888 : /*
1889 : * Grow array of per-tablespace status structs, every time a new
1890 : * tablespace is found.
1891 : */
1892 461782 : if (last_tsid == InvalidOid || last_tsid != cur_tsid)
1893 3024 : {
1894 : Size sz;
1895 :
1896 3024 : num_spaces++;
1897 :
1898 : /*
1899 : * Not worth adding grow-by-power-of-2 logic here - even with a
1900 : * few hundred tablespaces this should be fine.
1901 : */
1902 3024 : sz = sizeof(CkptTsStatus) * num_spaces;
1903 :
1904 3024 : if (per_ts_stat == NULL)
1905 1834 : per_ts_stat = (CkptTsStatus *) palloc(sz);
1906 : else
1907 1190 : per_ts_stat = (CkptTsStatus *) repalloc(per_ts_stat, sz);
1908 :
1909 3024 : s = &per_ts_stat[num_spaces - 1];
1910 3024 : memset(s, 0, sizeof(*s));
1911 3024 : s->tsId = cur_tsid;
1912 :
1913 : /*
1914 : * The first buffer in this tablespace. As CkptBufferIds is sorted
1915 : * by tablespace all (s->num_to_scan) buffers in this tablespace
1916 : * will follow afterwards.
1917 : */
1918 3024 : s->index = i;
1919 :
1920 : /*
1921 : * progress_slice will be determined once we know how many buffers
1922 : * are in each tablespace, i.e. after this loop.
1923 : */
1924 :
1925 3024 : last_tsid = cur_tsid;
1926 : }
1927 : else
1928 : {
1929 458758 : s = &per_ts_stat[num_spaces - 1];
1930 : }
1931 :
1932 461782 : s->num_to_scan++;
1933 : }
1934 :
1935 : Assert(num_spaces > 0);
1936 :
1937 : /*
1938 : * Build a min-heap over the write-progress in the individual tablespaces,
1939 : * and compute how large a portion of the total progress a single
1940 : * processed buffer is.
1941 : */
1942 1834 : ts_heap = binaryheap_allocate(num_spaces,
1943 : ts_ckpt_progress_comparator,
1944 : NULL);
1945 :
1946 4858 : for (i = 0; i < num_spaces; i++)
1947 : {
1948 3024 : CkptTsStatus *ts_stat = &per_ts_stat[i];
1949 :
1950 3024 : ts_stat->progress_slice = (float8) num_to_scan / ts_stat->num_to_scan;
1951 :
1952 3024 : binaryheap_add_unordered(ts_heap, PointerGetDatum(ts_stat));
1953 : }
1954 :
1955 1834 : binaryheap_build(ts_heap);
1956 :
1957 : /*
1958 : * Iterate through to-be-checkpointed buffers and write the ones (still)
1959 : * marked with BM_CHECKPOINT_NEEDED. The writes are balanced between
1960 : * tablespaces; otherwise the sorting would lead to only one tablespace
1961 : * receiving writes at a time, making inefficient use of the hardware.
1962 : */
1963 1834 : num_processed = 0;
1964 1834 : num_written = 0;
1965 465450 : while (!binaryheap_empty(ts_heap))
1966 : {
1967 461782 : BufferDesc *bufHdr = NULL;
1968 461782 : CkptTsStatus *ts_stat = (CkptTsStatus *)
1969 461782 : DatumGetPointer(binaryheap_first(ts_heap));
1970 :
1971 461782 : buf_id = CkptBufferIds[ts_stat->index].buf_id;
1972 : Assert(buf_id != -1);
1973 :
1974 461782 : bufHdr = GetBufferDescriptor(buf_id);
1975 :
1976 461782 : num_processed++;
1977 :
1978 : /*
1979 : * We don't need to acquire the lock here, because we're only looking
1980 : * at a single bit. It's possible that someone else writes the buffer
1981 : * and clears the flag right after we check, but that doesn't matter
1982 : * since SyncOneBuffer will then do nothing. However, there is a
1983 : * further race condition: it's conceivable that between the time we
1984 : * examine the bit here and the time SyncOneBuffer acquires the lock,
1985 : * someone else not only wrote the buffer but replaced it with another
1986 : * page and dirtied it. In that improbable case, SyncOneBuffer will
1987 : * write the buffer though we didn't need to. It doesn't seem worth
1988 : * guarding against this, though.
1989 : */
1990 461782 : if (pg_atomic_read_u32(&bufHdr->state) & BM_CHECKPOINT_NEEDED)
1991 : {
1992 459326 : if (SyncOneBuffer(buf_id, false, &wb_context) & BUF_WRITTEN)
1993 : {
1994 : TRACE_POSTGRESQL_BUFFER_SYNC_WRITTEN(buf_id);
1995 459326 : BgWriterStats.m_buf_written_checkpoints++;
1996 459326 : num_written++;
1997 : }
1998 : }
1999 :
2000 : /*
2001 : * Measure progress independent of actually having to flush the buffer
2002 : * - otherwise writing become unbalanced.
2003 : */
2004 461782 : ts_stat->progress += ts_stat->progress_slice;
2005 461782 : ts_stat->num_scanned++;
2006 461782 : ts_stat->index++;
2007 :
2008 : /* Have all the buffers from the tablespace been processed? */
2009 461782 : if (ts_stat->num_scanned == ts_stat->num_to_scan)
2010 : {
2011 3024 : binaryheap_remove_first(ts_heap);
2012 : }
2013 : else
2014 : {
2015 : /* update heap with the new progress */
2016 458758 : binaryheap_replace_first(ts_heap, PointerGetDatum(ts_stat));
2017 : }
2018 :
2019 : /*
2020 : * Sleep to throttle our I/O rate.
2021 : */
2022 461782 : CheckpointWriteDelay(flags, (double) num_processed / num_to_scan);
2023 : }
2024 :
2025 : /* issue all pending flushes */
2026 1834 : IssuePendingWritebacks(&wb_context);
2027 :
2028 1834 : pfree(per_ts_stat);
2029 1834 : per_ts_stat = NULL;
2030 1834 : binaryheap_free(ts_heap);
2031 :
2032 : /*
2033 : * Update checkpoint statistics. As noted above, this doesn't include
2034 : * buffers written by other backends or bgwriter scan.
2035 : */
2036 1834 : CheckpointStats.ckpt_bufs_written += num_written;
2037 :
2038 : TRACE_POSTGRESQL_BUFFER_SYNC_DONE(NBuffers, num_written, num_to_scan);
2039 : }
2040 :
2041 : /*
2042 : * BgBufferSync -- Write out some dirty buffers in the pool.
2043 : *
2044 : * This is called periodically by the background writer process.
2045 : *
2046 : * Returns true if it's appropriate for the bgwriter process to go into
2047 : * low-power hibernation mode. (This happens if the strategy clock sweep
2048 : * has been "lapped" and no buffer allocations have occurred recently,
2049 : * or if the bgwriter has been effectively disabled by setting
2050 : * bgwriter_lru_maxpages to 0.)
2051 : */
2052 : bool
2053 9252 : BgBufferSync(WritebackContext *wb_context)
2054 : {
2055 : /* info obtained from freelist.c */
2056 : int strategy_buf_id;
2057 : uint32 strategy_passes;
2058 : uint32 recent_alloc;
2059 :
2060 : /*
2061 : * Information saved between calls so we can determine the strategy
2062 : * point's advance rate and avoid scanning already-cleaned buffers.
2063 : */
2064 : static bool saved_info_valid = false;
2065 : static int prev_strategy_buf_id;
2066 : static uint32 prev_strategy_passes;
2067 : static int next_to_clean;
2068 : static uint32 next_passes;
2069 :
2070 : /* Moving averages of allocation rate and clean-buffer density */
2071 : static float smoothed_alloc = 0;
2072 : static float smoothed_density = 10.0;
2073 :
2074 : /* Potentially these could be tunables, but for now, not */
2075 9252 : float smoothing_samples = 16;
2076 9252 : float scan_whole_pool_milliseconds = 120000.0;
2077 :
2078 : /* Used to compute how far we scan ahead */
2079 : long strategy_delta;
2080 : int bufs_to_lap;
2081 : int bufs_ahead;
2082 : float scans_per_alloc;
2083 : int reusable_buffers_est;
2084 : int upcoming_alloc_est;
2085 : int min_scan_buffers;
2086 :
2087 : /* Variables for the scanning loop proper */
2088 : int num_to_scan;
2089 : int num_written;
2090 : int reusable_buffers;
2091 :
2092 : /* Variables for final smoothed_density update */
2093 : long new_strategy_delta;
2094 : uint32 new_recent_alloc;
2095 :
2096 : /*
2097 : * Find out where the freelist clock sweep currently is, and how many
2098 : * buffer allocations have happened since our last call.
2099 : */
2100 9252 : strategy_buf_id = StrategySyncStart(&strategy_passes, &recent_alloc);
2101 :
2102 : /* Report buffer alloc counts to pgstat */
2103 9252 : BgWriterStats.m_buf_alloc += recent_alloc;
2104 :
2105 : /*
2106 : * If we're not running the LRU scan, just stop after doing the stats
2107 : * stuff. We mark the saved state invalid so that we can recover sanely
2108 : * if LRU scan is turned back on later.
2109 : */
2110 9252 : if (bgwriter_lru_maxpages <= 0)
2111 : {
2112 0 : saved_info_valid = false;
2113 0 : return true;
2114 : }
2115 :
2116 : /*
2117 : * Compute strategy_delta = how many buffers have been scanned by the
2118 : * clock sweep since last time. If first time through, assume none. Then
2119 : * see if we are still ahead of the clock sweep, and if so, how many
2120 : * buffers we could scan before we'd catch up with it and "lap" it. Note:
2121 : * weird-looking coding of xxx_passes comparisons are to avoid bogus
2122 : * behavior when the passes counts wrap around.
2123 : */
2124 9252 : if (saved_info_valid)
2125 : {
2126 8884 : int32 passes_delta = strategy_passes - prev_strategy_passes;
2127 :
2128 8884 : strategy_delta = strategy_buf_id - prev_strategy_buf_id;
2129 8884 : strategy_delta += (long) passes_delta * NBuffers;
2130 :
2131 : Assert(strategy_delta >= 0);
2132 :
2133 8884 : if ((int32) (next_passes - strategy_passes) > 0)
2134 : {
2135 : /* we're one pass ahead of the strategy point */
2136 852 : bufs_to_lap = strategy_buf_id - next_to_clean;
2137 : #ifdef BGW_DEBUG
2138 : elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
2139 : next_passes, next_to_clean,
2140 : strategy_passes, strategy_buf_id,
2141 : strategy_delta, bufs_to_lap);
2142 : #endif
2143 : }
2144 15746 : else if (next_passes == strategy_passes &&
2145 7714 : next_to_clean >= strategy_buf_id)
2146 : {
2147 : /* on same pass, but ahead or at least not behind */
2148 7646 : bufs_to_lap = NBuffers - (next_to_clean - strategy_buf_id);
2149 : #ifdef BGW_DEBUG
2150 : elog(DEBUG2, "bgwriter ahead: bgw %u-%u strategy %u-%u delta=%ld lap=%d",
2151 : next_passes, next_to_clean,
2152 : strategy_passes, strategy_buf_id,
2153 : strategy_delta, bufs_to_lap);
2154 : #endif
2155 : }
2156 : else
2157 : {
2158 : /*
2159 : * We're behind, so skip forward to the strategy point and start
2160 : * cleaning from there.
2161 : */
2162 : #ifdef BGW_DEBUG
2163 : elog(DEBUG2, "bgwriter behind: bgw %u-%u strategy %u-%u delta=%ld",
2164 : next_passes, next_to_clean,
2165 : strategy_passes, strategy_buf_id,
2166 : strategy_delta);
2167 : #endif
2168 386 : next_to_clean = strategy_buf_id;
2169 386 : next_passes = strategy_passes;
2170 386 : bufs_to_lap = NBuffers;
2171 : }
2172 : }
2173 : else
2174 : {
2175 : /*
2176 : * Initializing at startup or after LRU scanning had been off. Always
2177 : * start at the strategy point.
2178 : */
2179 : #ifdef BGW_DEBUG
2180 : elog(DEBUG2, "bgwriter initializing: strategy %u-%u",
2181 : strategy_passes, strategy_buf_id);
2182 : #endif
2183 368 : strategy_delta = 0;
2184 368 : next_to_clean = strategy_buf_id;
2185 368 : next_passes = strategy_passes;
2186 368 : bufs_to_lap = NBuffers;
2187 : }
2188 :
2189 : /* Update saved info for next time */
2190 9252 : prev_strategy_buf_id = strategy_buf_id;
2191 9252 : prev_strategy_passes = strategy_passes;
2192 9252 : saved_info_valid = true;
2193 :
2194 : /*
2195 : * Compute how many buffers had to be scanned for each new allocation, ie,
2196 : * 1/density of reusable buffers, and track a moving average of that.
2197 : *
2198 : * If the strategy point didn't move, we don't update the density estimate
2199 : */
2200 9252 : if (strategy_delta > 0 && recent_alloc > 0)
2201 : {
2202 598 : scans_per_alloc = (float) strategy_delta / (float) recent_alloc;
2203 598 : smoothed_density += (scans_per_alloc - smoothed_density) /
2204 : smoothing_samples;
2205 : }
2206 :
2207 : /*
2208 : * Estimate how many reusable buffers there are between the current
2209 : * strategy point and where we've scanned ahead to, based on the smoothed
2210 : * density estimate.
2211 : */
2212 9252 : bufs_ahead = NBuffers - bufs_to_lap;
2213 9252 : reusable_buffers_est = (float) bufs_ahead / smoothed_density;
2214 :
2215 : /*
2216 : * Track a moving average of recent buffer allocations. Here, rather than
2217 : * a true average we want a fast-attack, slow-decline behavior: we
2218 : * immediately follow any increase.
2219 : */
2220 9252 : if (smoothed_alloc <= (float) recent_alloc)
2221 1814 : smoothed_alloc = recent_alloc;
2222 : else
2223 7438 : smoothed_alloc += ((float) recent_alloc - smoothed_alloc) /
2224 : smoothing_samples;
2225 :
2226 : /* Scale the estimate by a GUC to allow more aggressive tuning. */
2227 9252 : upcoming_alloc_est = (int) (smoothed_alloc * bgwriter_lru_multiplier);
2228 :
2229 : /*
2230 : * If recent_alloc remains at zero for many cycles, smoothed_alloc will
2231 : * eventually underflow to zero, and the underflows produce annoying
2232 : * kernel warnings on some platforms. Once upcoming_alloc_est has gone to
2233 : * zero, there's no point in tracking smaller and smaller values of
2234 : * smoothed_alloc, so just reset it to exactly zero to avoid this
2235 : * syndrome. It will pop back up as soon as recent_alloc increases.
2236 : */
2237 9252 : if (upcoming_alloc_est == 0)
2238 658 : smoothed_alloc = 0;
2239 :
2240 : /*
2241 : * Even in cases where there's been little or no buffer allocation
2242 : * activity, we want to make a small amount of progress through the buffer
2243 : * cache so that as many reusable buffers as possible are clean after an
2244 : * idle period.
2245 : *
2246 : * (scan_whole_pool_milliseconds / BgWriterDelay) computes how many times
2247 : * the BGW will be called during the scan_whole_pool time; slice the
2248 : * buffer pool into that many sections.
2249 : */
2250 9252 : min_scan_buffers = (int) (NBuffers / (scan_whole_pool_milliseconds / BgWriterDelay));
2251 :
2252 9252 : if (upcoming_alloc_est < (min_scan_buffers + reusable_buffers_est))
2253 : {
2254 : #ifdef BGW_DEBUG
2255 : elog(DEBUG2, "bgwriter: alloc_est=%d too small, using min=%d + reusable_est=%d",
2256 : upcoming_alloc_est, min_scan_buffers, reusable_buffers_est);
2257 : #endif
2258 6634 : upcoming_alloc_est = min_scan_buffers + reusable_buffers_est;
2259 : }
2260 :
2261 : /*
2262 : * Now write out dirty reusable buffers, working forward from the
2263 : * next_to_clean point, until we have lapped the strategy scan, or cleaned
2264 : * enough buffers to match our estimate of the next cycle's allocation
2265 : * requirements, or hit the bgwriter_lru_maxpages limit.
2266 : */
2267 :
2268 : /* Make sure we can handle the pin inside SyncOneBuffer */
2269 9252 : ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
2270 :
2271 9252 : num_to_scan = bufs_to_lap;
2272 9252 : num_written = 0;
2273 9252 : reusable_buffers = reusable_buffers_est;
2274 :
2275 : /* Execute the LRU scan */
2276 493894 : while (num_to_scan > 0 && reusable_buffers < upcoming_alloc_est)
2277 : {
2278 475390 : int sync_state = SyncOneBuffer(next_to_clean, true,
2279 : wb_context);
2280 :
2281 475390 : if (++next_to_clean >= NBuffers)
2282 : {
2283 524 : next_to_clean = 0;
2284 524 : next_passes++;
2285 : }
2286 475390 : num_to_scan--;
2287 :
2288 475390 : if (sync_state & BUF_WRITTEN)
2289 : {
2290 1466 : reusable_buffers++;
2291 1466 : if (++num_written >= bgwriter_lru_maxpages)
2292 : {
2293 0 : BgWriterStats.m_maxwritten_clean++;
2294 0 : break;
2295 : }
2296 : }
2297 473924 : else if (sync_state & BUF_REUSABLE)
2298 384128 : reusable_buffers++;
2299 : }
2300 :
2301 9252 : BgWriterStats.m_buf_written_clean += num_written;
2302 :
2303 : #ifdef BGW_DEBUG
2304 : elog(DEBUG1, "bgwriter: recent_alloc=%u smoothed=%.2f delta=%ld ahead=%d density=%.2f reusable_est=%d upcoming_est=%d scanned=%d wrote=%d reusable=%d",
2305 : recent_alloc, smoothed_alloc, strategy_delta, bufs_ahead,
2306 : smoothed_density, reusable_buffers_est, upcoming_alloc_est,
2307 : bufs_to_lap - num_to_scan,
2308 : num_written,
2309 : reusable_buffers - reusable_buffers_est);
2310 : #endif
2311 :
2312 : /*
2313 : * Consider the above scan as being like a new allocation scan.
2314 : * Characterize its density and update the smoothed one based on it. This
2315 : * effectively halves the moving average period in cases where both the
2316 : * strategy and the background writer are doing some useful scanning,
2317 : * which is helpful because a long memory isn't as desirable on the
2318 : * density estimates.
2319 : */
2320 9252 : new_strategy_delta = bufs_to_lap - num_to_scan;
2321 9252 : new_recent_alloc = reusable_buffers - reusable_buffers_est;
2322 9252 : if (new_strategy_delta > 0 && new_recent_alloc > 0)
2323 : {
2324 7984 : scans_per_alloc = (float) new_strategy_delta / (float) new_recent_alloc;
2325 7984 : smoothed_density += (scans_per_alloc - smoothed_density) /
2326 : smoothing_samples;
2327 :
2328 : #ifdef BGW_DEBUG
2329 : elog(DEBUG2, "bgwriter: cleaner density alloc=%u scan=%ld density=%.2f new smoothed=%.2f",
2330 : new_recent_alloc, new_strategy_delta,
2331 : scans_per_alloc, smoothed_density);
2332 : #endif
2333 : }
2334 :
2335 : /* Return true if OK to hibernate */
2336 9252 : return (bufs_to_lap == 0 && recent_alloc == 0);
2337 : }
2338 :
2339 : /*
2340 : * SyncOneBuffer -- process a single buffer during syncing.
2341 : *
2342 : * If skip_recently_used is true, we don't write currently-pinned buffers, nor
2343 : * buffers marked recently used, as these are not replacement candidates.
2344 : *
2345 : * Returns a bitmask containing the following flag bits:
2346 : * BUF_WRITTEN: we wrote the buffer.
2347 : * BUF_REUSABLE: buffer is available for replacement, ie, it has
2348 : * pin count 0 and usage count 0.
2349 : *
2350 : * (BUF_WRITTEN could be set in error if FlushBuffer finds the buffer clean
2351 : * after locking it, but we don't care all that much.)
2352 : *
2353 : * Note: caller must have done ResourceOwnerEnlargeBuffers.
2354 : */
2355 : static int
2356 934716 : SyncOneBuffer(int buf_id, bool skip_recently_used, WritebackContext *wb_context)
2357 : {
2358 934716 : BufferDesc *bufHdr = GetBufferDescriptor(buf_id);
2359 934716 : int result = 0;
2360 : uint32 buf_state;
2361 : BufferTag tag;
2362 :
2363 934716 : ReservePrivateRefCountEntry();
2364 :
2365 : /*
2366 : * Check whether buffer needs writing.
2367 : *
2368 : * We can make this check without taking the buffer content lock so long
2369 : * as we mark pages dirty in access methods *before* logging changes with
2370 : * XLogInsert(): if someone marks the buffer dirty just after our check we
2371 : * don't worry because our checkpoint.redo points before log record for
2372 : * upcoming changes and so we are not required to write such dirty buffer.
2373 : */
2374 934716 : buf_state = LockBufHdr(bufHdr);
2375 :
2376 1869222 : if (BUF_STATE_GET_REFCOUNT(buf_state) == 0 &&
2377 934506 : BUF_STATE_GET_USAGECOUNT(buf_state) == 0)
2378 : {
2379 385924 : result |= BUF_REUSABLE;
2380 : }
2381 548792 : else if (skip_recently_used)
2382 : {
2383 : /* Caller told us not to write recently-used buffers */
2384 89796 : UnlockBufHdr(bufHdr, buf_state);
2385 89796 : return result;
2386 : }
2387 :
2388 844920 : if (!(buf_state & BM_VALID) || !(buf_state & BM_DIRTY))
2389 : {
2390 : /* It's clean, so nothing to do */
2391 384128 : UnlockBufHdr(bufHdr, buf_state);
2392 384128 : return result;
2393 : }
2394 :
2395 : /*
2396 : * Pin it, share-lock it, write it. (FlushBuffer will do nothing if the
2397 : * buffer is clean by the time we've locked it.)
2398 : */
2399 460792 : PinBuffer_Locked(bufHdr);
2400 460792 : LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
2401 :
2402 460792 : FlushBuffer(bufHdr, NULL);
2403 :
2404 460792 : LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
2405 :
2406 460792 : tag = bufHdr->tag;
2407 :
2408 460792 : UnpinBuffer(bufHdr, true);
2409 :
2410 460792 : ScheduleBufferTagForWriteback(wb_context, &tag);
2411 :
2412 460792 : return result | BUF_WRITTEN;
2413 : }
2414 :
2415 : /*
2416 : * AtEOXact_Buffers - clean up at end of transaction.
2417 : *
2418 : * As of PostgreSQL 8.0, buffer pins should get released by the
2419 : * ResourceOwner mechanism. This routine is just a debugging
2420 : * cross-check that no pins remain.
2421 : */
2422 : void
2423 458438 : AtEOXact_Buffers(bool isCommit)
2424 : {
2425 458438 : CheckForBufferLeaks();
2426 :
2427 458438 : AtEOXact_LocalBuffers(isCommit);
2428 :
2429 : Assert(PrivateRefCountOverflowed == 0);
2430 458438 : }
2431 :
2432 : /*
2433 : * Initialize access to shared buffer pool
2434 : *
2435 : * This is called during backend startup (whether standalone or under the
2436 : * postmaster). It sets up for this backend's access to the already-existing
2437 : * buffer pool.
2438 : *
2439 : * NB: this is called before InitProcess(), so we do not have a PGPROC and
2440 : * cannot do LWLockAcquire; hence we can't actually access stuff in
2441 : * shared memory yet. We are only initializing local data here.
2442 : * (See also InitBufferPoolBackend)
2443 : */
2444 : void
2445 12342 : InitBufferPoolAccess(void)
2446 : {
2447 : HASHCTL hash_ctl;
2448 :
2449 12342 : memset(&PrivateRefCountArray, 0, sizeof(PrivateRefCountArray));
2450 :
2451 12342 : MemSet(&hash_ctl, 0, sizeof(hash_ctl));
2452 12342 : hash_ctl.keysize = sizeof(int32);
2453 12342 : hash_ctl.entrysize = sizeof(PrivateRefCountEntry);
2454 :
2455 12342 : PrivateRefCountHash = hash_create("PrivateRefCount", 100, &hash_ctl,
2456 : HASH_ELEM | HASH_BLOBS);
2457 12342 : }
2458 :
2459 : /*
2460 : * InitBufferPoolBackend --- second-stage initialization of a new backend
2461 : *
2462 : * This is called after we have acquired a PGPROC and so can safely get
2463 : * LWLocks. We don't currently need to do anything at this stage ...
2464 : * except register a shmem-exit callback. AtProcExit_Buffers needs LWLock
2465 : * access, and thereby has to be called at the corresponding phase of
2466 : * backend shutdown.
2467 : */
2468 : void
2469 11680 : InitBufferPoolBackend(void)
2470 : {
2471 11680 : on_shmem_exit(AtProcExit_Buffers, 0);
2472 11680 : }
2473 :
2474 : /*
2475 : * During backend exit, ensure that we released all shared-buffer locks and
2476 : * assert that we have no remaining pins.
2477 : */
2478 : static void
2479 11680 : AtProcExit_Buffers(int code, Datum arg)
2480 : {
2481 11680 : AbortBufferIO();
2482 11680 : UnlockBuffers();
2483 :
2484 11680 : CheckForBufferLeaks();
2485 :
2486 : /* localbuf.c needs a chance too */
2487 11680 : AtProcExit_LocalBuffers();
2488 11680 : }
2489 :
2490 : /*
2491 : * CheckForBufferLeaks - ensure this backend holds no buffer pins
2492 : *
2493 : * As of PostgreSQL 8.0, buffer pins should get released by the
2494 : * ResourceOwner mechanism. This routine is just a debugging
2495 : * cross-check that no pins remain.
2496 : */
2497 : static void
2498 470118 : CheckForBufferLeaks(void)
2499 : {
2500 : #ifdef USE_ASSERT_CHECKING
2501 : int RefCountErrors = 0;
2502 : PrivateRefCountEntry *res;
2503 : int i;
2504 :
2505 : /* check the array */
2506 : for (i = 0; i < REFCOUNT_ARRAY_ENTRIES; i++)
2507 : {
2508 : res = &PrivateRefCountArray[i];
2509 :
2510 : if (res->buffer != InvalidBuffer)
2511 : {
2512 : PrintBufferLeakWarning(res->buffer);
2513 : RefCountErrors++;
2514 : }
2515 : }
2516 :
2517 : /* if necessary search the hash */
2518 : if (PrivateRefCountOverflowed)
2519 : {
2520 : HASH_SEQ_STATUS hstat;
2521 :
2522 : hash_seq_init(&hstat, PrivateRefCountHash);
2523 : while ((res = (PrivateRefCountEntry *) hash_seq_search(&hstat)) != NULL)
2524 : {
2525 : PrintBufferLeakWarning(res->buffer);
2526 : RefCountErrors++;
2527 : }
2528 :
2529 : }
2530 :
2531 : Assert(RefCountErrors == 0);
2532 : #endif
2533 470118 : }
2534 :
2535 : /*
2536 : * Helper routine to issue warnings when a buffer is unexpectedly pinned
2537 : */
2538 : void
2539 0 : PrintBufferLeakWarning(Buffer buffer)
2540 : {
2541 : BufferDesc *buf;
2542 : int32 loccount;
2543 : char *path;
2544 : BackendId backend;
2545 : uint32 buf_state;
2546 :
2547 : Assert(BufferIsValid(buffer));
2548 0 : if (BufferIsLocal(buffer))
2549 : {
2550 0 : buf = GetLocalBufferDescriptor(-buffer - 1);
2551 0 : loccount = LocalRefCount[-buffer - 1];
2552 0 : backend = MyBackendId;
2553 : }
2554 : else
2555 : {
2556 0 : buf = GetBufferDescriptor(buffer - 1);
2557 0 : loccount = GetPrivateRefCount(buffer);
2558 0 : backend = InvalidBackendId;
2559 : }
2560 :
2561 : /* theoretically we should lock the bufhdr here */
2562 0 : path = relpathbackend(buf->tag.rnode, backend, buf->tag.forkNum);
2563 0 : buf_state = pg_atomic_read_u32(&buf->state);
2564 0 : elog(WARNING,
2565 : "buffer refcount leak: [%03d] "
2566 : "(rel=%s, blockNum=%u, flags=0x%x, refcount=%u %d)",
2567 : buffer, path,
2568 : buf->tag.blockNum, buf_state & BUF_FLAG_MASK,
2569 : BUF_STATE_GET_REFCOUNT(buf_state), loccount);
2570 0 : pfree(path);
2571 0 : }
2572 :
2573 : /*
2574 : * CheckPointBuffers
2575 : *
2576 : * Flush all dirty blocks in buffer pool to disk at checkpoint time.
2577 : *
2578 : * Note: temporary relations do not participate in checkpoints, so they don't
2579 : * need to be flushed.
2580 : */
2581 : void
2582 2832 : CheckPointBuffers(int flags)
2583 : {
2584 : TRACE_POSTGRESQL_BUFFER_CHECKPOINT_START(flags);
2585 2832 : CheckpointStats.ckpt_write_t = GetCurrentTimestamp();
2586 2832 : BufferSync(flags);
2587 2832 : CheckpointStats.ckpt_sync_t = GetCurrentTimestamp();
2588 : TRACE_POSTGRESQL_BUFFER_CHECKPOINT_SYNC_START();
2589 2832 : ProcessSyncRequests();
2590 2832 : CheckpointStats.ckpt_sync_end_t = GetCurrentTimestamp();
2591 : TRACE_POSTGRESQL_BUFFER_CHECKPOINT_DONE();
2592 2832 : }
2593 :
2594 :
2595 : /*
2596 : * Do whatever is needed to prepare for commit at the bufmgr and smgr levels
2597 : */
2598 : void
2599 246378 : BufmgrCommit(void)
2600 : {
2601 : /* Nothing to do in bufmgr anymore... */
2602 246378 : }
2603 :
2604 : /*
2605 : * BufferGetBlockNumber
2606 : * Returns the block number associated with a buffer.
2607 : *
2608 : * Note:
2609 : * Assumes that the buffer is valid and pinned, else the
2610 : * value may be obsolete immediately...
2611 : */
2612 : BlockNumber
2613 158651114 : BufferGetBlockNumber(Buffer buffer)
2614 : {
2615 : BufferDesc *bufHdr;
2616 :
2617 : Assert(BufferIsPinned(buffer));
2618 :
2619 158651114 : if (BufferIsLocal(buffer))
2620 3957890 : bufHdr = GetLocalBufferDescriptor(-buffer - 1);
2621 : else
2622 154693224 : bufHdr = GetBufferDescriptor(buffer - 1);
2623 :
2624 : /* pinned, so OK to read tag without spinlock */
2625 158651114 : return bufHdr->tag.blockNum;
2626 : }
2627 :
2628 : /*
2629 : * BufferGetTag
2630 : * Returns the relfilenode, fork number and block number associated with
2631 : * a buffer.
2632 : */
2633 : void
2634 32402696 : BufferGetTag(Buffer buffer, RelFileNode *rnode, ForkNumber *forknum,
2635 : BlockNumber *blknum)
2636 : {
2637 : BufferDesc *bufHdr;
2638 :
2639 : /* Do the same checks as BufferGetBlockNumber. */
2640 : Assert(BufferIsPinned(buffer));
2641 :
2642 32402696 : if (BufferIsLocal(buffer))
2643 0 : bufHdr = GetLocalBufferDescriptor(-buffer - 1);
2644 : else
2645 32402696 : bufHdr = GetBufferDescriptor(buffer - 1);
2646 :
2647 : /* pinned, so OK to read tag without spinlock */
2648 32402696 : *rnode = bufHdr->tag.rnode;
2649 32402696 : *forknum = bufHdr->tag.forkNum;
2650 32402696 : *blknum = bufHdr->tag.blockNum;
2651 32402696 : }
2652 :
2653 : /*
2654 : * FlushBuffer
2655 : * Physically write out a shared buffer.
2656 : *
2657 : * NOTE: this actually just passes the buffer contents to the kernel; the
2658 : * real write to disk won't happen until the kernel feels like it. This
2659 : * is okay from our point of view since we can redo the changes from WAL.
2660 : * However, we will need to force the changes to disk via fsync before
2661 : * we can checkpoint WAL.
2662 : *
2663 : * The caller must hold a pin on the buffer and have share-locked the
2664 : * buffer contents. (Note: a share-lock does not prevent updates of
2665 : * hint bits in the buffer, so the page could change while the write
2666 : * is in progress, but we assume that that will not invalidate the data
2667 : * written.)
2668 : *
2669 : * If the caller has an smgr reference for the buffer's relation, pass it
2670 : * as the second parameter. If not, pass NULL.
2671 : */
2672 : static void
2673 474838 : FlushBuffer(BufferDesc *buf, SMgrRelation reln)
2674 : {
2675 : XLogRecPtr recptr;
2676 : ErrorContextCallback errcallback;
2677 : instr_time io_start,
2678 : io_time;
2679 : Block bufBlock;
2680 : char *bufToWrite;
2681 : uint32 buf_state;
2682 :
2683 : /*
2684 : * Acquire the buffer's io_in_progress lock. If StartBufferIO returns
2685 : * false, then someone else flushed the buffer before we could, so we need
2686 : * not do anything.
2687 : */
2688 474838 : if (!StartBufferIO(buf, false))
2689 0 : return;
2690 :
2691 : /* Setup error traceback support for ereport() */
2692 474838 : errcallback.callback = shared_buffer_write_error_callback;
2693 474838 : errcallback.arg = (void *) buf;
2694 474838 : errcallback.previous = error_context_stack;
2695 474838 : error_context_stack = &errcallback;
2696 :
2697 : /* Find smgr relation for buffer */
2698 474838 : if (reln == NULL)
2699 471098 : reln = smgropen(buf->tag.rnode, InvalidBackendId);
2700 :
2701 : TRACE_POSTGRESQL_BUFFER_FLUSH_START(buf->tag.forkNum,
2702 : buf->tag.blockNum,
2703 : reln->smgr_rnode.node.spcNode,
2704 : reln->smgr_rnode.node.dbNode,
2705 : reln->smgr_rnode.node.relNode);
2706 :
2707 474838 : buf_state = LockBufHdr(buf);
2708 :
2709 : /*
2710 : * Run PageGetLSN while holding header lock, since we don't have the
2711 : * buffer locked exclusively in all cases.
2712 : */
2713 474838 : recptr = BufferGetLSN(buf);
2714 :
2715 : /* To check if block content changes while flushing. - vadim 01/17/97 */
2716 474838 : buf_state &= ~BM_JUST_DIRTIED;
2717 474838 : UnlockBufHdr(buf, buf_state);
2718 :
2719 : /*
2720 : * Force XLOG flush up to buffer's LSN. This implements the basic WAL
2721 : * rule that log updates must hit disk before any of the data-file changes
2722 : * they describe do.
2723 : *
2724 : * However, this rule does not apply to unlogged relations, which will be
2725 : * lost after a crash anyway. Most unlogged relation pages do not bear
2726 : * LSNs since we never emit WAL records for them, and therefore flushing
2727 : * up through the buffer LSN would be useless, but harmless. However,
2728 : * GiST indexes use LSNs internally to track page-splits, and therefore
2729 : * unlogged GiST pages bear "fake" LSNs generated by
2730 : * GetFakeLSNForUnloggedRel. It is unlikely but possible that the fake
2731 : * LSN counter could advance past the WAL insertion point; and if it did
2732 : * happen, attempting to flush WAL through that location would fail, with
2733 : * disastrous system-wide consequences. To make sure that can't happen,
2734 : * skip the flush if the buffer isn't permanent.
2735 : */
2736 474838 : if (buf_state & BM_PERMANENT)
2737 470930 : XLogFlush(recptr);
2738 :
2739 : /*
2740 : * Now it's safe to write buffer to disk. Note that no one else should
2741 : * have been able to write it while we were busy with log flushing because
2742 : * we have the io_in_progress lock.
2743 : */
2744 474838 : bufBlock = BufHdrGetBlock(buf);
2745 :
2746 : /*
2747 : * Update page checksum if desired. Since we have only shared lock on the
2748 : * buffer, other processes might be updating hint bits in it, so we must
2749 : * copy the page to private storage if we do checksumming.
2750 : */
2751 474838 : bufToWrite = PageSetChecksumCopy((Page) bufBlock, buf->tag.blockNum);
2752 :
2753 474838 : if (track_io_timing)
2754 0 : INSTR_TIME_SET_CURRENT(io_start);
2755 :
2756 : /*
2757 : * bufToWrite is either the shared buffer or a copy, as appropriate.
2758 : */
2759 474838 : smgrwrite(reln,
2760 : buf->tag.forkNum,
2761 : buf->tag.blockNum,
2762 : bufToWrite,
2763 : false);
2764 :
2765 474838 : if (track_io_timing)
2766 : {
2767 0 : INSTR_TIME_SET_CURRENT(io_time);
2768 0 : INSTR_TIME_SUBTRACT(io_time, io_start);
2769 0 : pgstat_count_buffer_write_time(INSTR_TIME_GET_MICROSEC(io_time));
2770 0 : INSTR_TIME_ADD(pgBufferUsage.blk_write_time, io_time);
2771 : }
2772 :
2773 474838 : pgBufferUsage.shared_blks_written++;
2774 :
2775 : /*
2776 : * Mark the buffer as clean (unless BM_JUST_DIRTIED has become set) and
2777 : * end the io_in_progress state.
2778 : */
2779 474838 : TerminateBufferIO(buf, true, 0);
2780 :
2781 : TRACE_POSTGRESQL_BUFFER_FLUSH_DONE(buf->tag.forkNum,
2782 : buf->tag.blockNum,
2783 : reln->smgr_rnode.node.spcNode,
2784 : reln->smgr_rnode.node.dbNode,
2785 : reln->smgr_rnode.node.relNode);
2786 :
2787 : /* Pop the error context stack */
2788 474838 : error_context_stack = errcallback.previous;
2789 : }
2790 :
2791 : /*
2792 : * RelationGetNumberOfBlocksInFork
2793 : * Determines the current number of pages in the specified relation fork.
2794 : *
2795 : * Note that the accuracy of the result will depend on the details of the
2796 : * relation's storage. For builtin AMs it'll be accurate, but for external AMs
2797 : * it might not be.
2798 : */
2799 : BlockNumber
2800 2905388 : RelationGetNumberOfBlocksInFork(Relation relation, ForkNumber forkNum)
2801 : {
2802 2905388 : switch (relation->rd_rel->relkind)
2803 : {
2804 : case RELKIND_SEQUENCE:
2805 : case RELKIND_INDEX:
2806 : case RELKIND_PARTITIONED_INDEX:
2807 : /* Open it at the smgr level if not already done */
2808 676084 : RelationOpenSmgr(relation);
2809 :
2810 676084 : return smgrnblocks(relation->rd_smgr, forkNum);
2811 :
2812 : case RELKIND_RELATION:
2813 : case RELKIND_TOASTVALUE:
2814 : case RELKIND_MATVIEW:
2815 : {
2816 : /*
2817 : * Not every table AM uses BLCKSZ wide fixed size blocks.
2818 : * Therefore tableam returns the size in bytes - but for the
2819 : * purpose of this routine, we want the number of blocks.
2820 : * Therefore divide, rounding up.
2821 : */
2822 : uint64 szbytes;
2823 :
2824 2229304 : szbytes = table_relation_size(relation, forkNum);
2825 :
2826 2229304 : return (szbytes + (BLCKSZ - 1)) / BLCKSZ;
2827 : }
2828 : case RELKIND_VIEW:
2829 : case RELKIND_COMPOSITE_TYPE:
2830 : case RELKIND_FOREIGN_TABLE:
2831 : case RELKIND_PARTITIONED_TABLE:
2832 : default:
2833 : Assert(false);
2834 0 : break;
2835 : }
2836 :
2837 0 : return 0; /* keep compiler quiet */
2838 : }
2839 :
2840 : /*
2841 : * BufferIsPermanent
2842 : * Determines whether a buffer will potentially still be around after
2843 : * a crash. Caller must hold a buffer pin.
2844 : */
2845 : bool
2846 17350404 : BufferIsPermanent(Buffer buffer)
2847 : {
2848 : BufferDesc *bufHdr;
2849 :
2850 : /* Local buffers are used only for temp relations. */
2851 17350404 : if (BufferIsLocal(buffer))
2852 1522480 : return false;
2853 :
2854 : /* Make sure we've got a real buffer, and that we hold a pin on it. */
2855 : Assert(BufferIsValid(buffer));
2856 : Assert(BufferIsPinned(buffer));
2857 :
2858 : /*
2859 : * BM_PERMANENT can't be changed while we hold a pin on the buffer, so we
2860 : * need not bother with the buffer header spinlock. Even if someone else
2861 : * changes the buffer header state while we're doing this, the state is
2862 : * changed atomically, so we'll read the old value or the new value, but
2863 : * not random garbage.
2864 : */
2865 15827924 : bufHdr = GetBufferDescriptor(buffer - 1);
2866 15827924 : return (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT) != 0;
2867 : }
2868 :
2869 : /*
2870 : * BufferGetLSNAtomic
2871 : * Retrieves the LSN of the buffer atomically using a buffer header lock.
2872 : * This is necessary for some callers who may not have an exclusive lock
2873 : * on the buffer.
2874 : */
2875 : XLogRecPtr
2876 11497148 : BufferGetLSNAtomic(Buffer buffer)
2877 : {
2878 11497148 : BufferDesc *bufHdr = GetBufferDescriptor(buffer - 1);
2879 11497148 : char *page = BufferGetPage(buffer);
2880 : XLogRecPtr lsn;
2881 : uint32 buf_state;
2882 :
2883 : /*
2884 : * If we don't need locking for correctness, fastpath out.
2885 : */
2886 11497148 : if (!XLogHintBitIsNeeded() || BufferIsLocal(buffer))
2887 11202844 : return PageGetLSN(page);
2888 :
2889 : /* Make sure we've got a real buffer, and that we hold a pin on it. */
2890 : Assert(BufferIsValid(buffer));
2891 : Assert(BufferIsPinned(buffer));
2892 :
2893 294304 : buf_state = LockBufHdr(bufHdr);
2894 294304 : lsn = PageGetLSN(page);
2895 294304 : UnlockBufHdr(bufHdr, buf_state);
2896 :
2897 294304 : return lsn;
2898 : }
2899 :
2900 : /* ---------------------------------------------------------------------
2901 : * DropRelFileNodeBuffers
2902 : *
2903 : * This function removes from the buffer pool all the pages of the
2904 : * specified relation forks that have block numbers >= firstDelBlock.
2905 : * (In particular, with firstDelBlock = 0, all pages are removed.)
2906 : * Dirty pages are simply dropped, without bothering to write them
2907 : * out first. Therefore, this is NOT rollback-able, and so should be
2908 : * used only with extreme caution!
2909 : *
2910 : * Currently, this is called only from smgr.c when the underlying file
2911 : * is about to be deleted or truncated (firstDelBlock is needed for
2912 : * the truncation case). The data in the affected pages would therefore
2913 : * be deleted momentarily anyway, and there is no point in writing it.
2914 : * It is the responsibility of higher-level code to ensure that the
2915 : * deletion or truncation does not lose any data that could be needed
2916 : * later. It is also the responsibility of higher-level code to ensure
2917 : * that no other process could be trying to load more pages of the
2918 : * relation into buffers.
2919 : *
2920 : * XXX currently it sequentially searches the buffer pool, should be
2921 : * changed to more clever ways of searching. However, this routine
2922 : * is used only in code paths that aren't very performance-critical,
2923 : * and we shouldn't slow down the hot paths to make it faster ...
2924 : * --------------------------------------------------------------------
2925 : */
2926 : void
2927 440 : DropRelFileNodeBuffers(RelFileNodeBackend rnode, ForkNumber *forkNum,
2928 : int nforks, BlockNumber *firstDelBlock)
2929 : {
2930 : int i;
2931 : int j;
2932 :
2933 : /* If it's a local relation, it's localbuf.c's problem. */
2934 440 : if (RelFileNodeBackendIsTemp(rnode))
2935 : {
2936 204 : if (rnode.backend == MyBackendId)
2937 : {
2938 408 : for (j = 0; j < nforks; j++)
2939 204 : DropRelFileNodeLocalBuffers(rnode.node, forkNum[j],
2940 204 : firstDelBlock[j]);
2941 : }
2942 204 : return;
2943 : }
2944 :
2945 3736812 : for (i = 0; i < NBuffers; i++)
2946 : {
2947 3736576 : BufferDesc *bufHdr = GetBufferDescriptor(i);
2948 : uint32 buf_state;
2949 :
2950 : /*
2951 : * We can make this a tad faster by prechecking the buffer tag before
2952 : * we attempt to lock the buffer; this saves a lot of lock
2953 : * acquisitions in typical cases. It should be safe because the
2954 : * caller must have AccessExclusiveLock on the relation, or some other
2955 : * reason to be certain that no one is loading new pages of the rel
2956 : * into the buffer pool. (Otherwise we might well miss such pages
2957 : * entirely.) Therefore, while the tag might be changing while we
2958 : * look at it, it can't be changing *to* a value we care about, only
2959 : * *away* from such a value. So false negatives are impossible, and
2960 : * false positives are safe because we'll recheck after getting the
2961 : * buffer lock.
2962 : *
2963 : * We could check forkNum and blockNum as well as the rnode, but the
2964 : * incremental win from doing so seems small.
2965 : */
2966 3736576 : if (!RelFileNodeEquals(bufHdr->tag.rnode, rnode.node))
2967 3732812 : continue;
2968 :
2969 3764 : buf_state = LockBufHdr(bufHdr);
2970 :
2971 9218 : for (j = 0; j < nforks; j++)
2972 : {
2973 13224 : if (RelFileNodeEquals(bufHdr->tag.rnode, rnode.node) &&
2974 10296 : bufHdr->tag.forkNum == forkNum[j] &&
2975 3684 : bufHdr->tag.blockNum >= firstDelBlock[j])
2976 : {
2977 1158 : InvalidateBuffer(bufHdr); /* releases spinlock */
2978 1158 : break;
2979 : }
2980 : }
2981 3764 : if (j >= nforks)
2982 2606 : UnlockBufHdr(bufHdr, buf_state);
2983 : }
2984 : }
2985 :
2986 : /* ---------------------------------------------------------------------
2987 : * DropRelFileNodesAllBuffers
2988 : *
2989 : * This function removes from the buffer pool all the pages of all
2990 : * forks of the specified relations. It's equivalent to calling
2991 : * DropRelFileNodeBuffers once per fork per relation with
2992 : * firstDelBlock = 0.
2993 : * --------------------------------------------------------------------
2994 : */
2995 : void
2996 12300 : DropRelFileNodesAllBuffers(RelFileNodeBackend *rnodes, int nnodes)
2997 : {
2998 : int i,
2999 12300 : n = 0;
3000 : RelFileNode *nodes;
3001 : bool use_bsearch;
3002 :
3003 12300 : if (nnodes == 0)
3004 0 : return;
3005 :
3006 12300 : nodes = palloc(sizeof(RelFileNode) * nnodes); /* non-local relations */
3007 :
3008 : /* If it's a local relation, it's localbuf.c's problem. */
3009 54348 : for (i = 0; i < nnodes; i++)
3010 : {
3011 42048 : if (RelFileNodeBackendIsTemp(rnodes[i]))
3012 : {
3013 4780 : if (rnodes[i].backend == MyBackendId)
3014 4780 : DropRelFileNodeAllLocalBuffers(rnodes[i].node);
3015 : }
3016 : else
3017 37268 : nodes[n++] = rnodes[i].node;
3018 : }
3019 :
3020 : /*
3021 : * If there are no non-local relations, then we're done. Release the
3022 : * memory and return.
3023 : */
3024 12300 : if (n == 0)
3025 : {
3026 1336 : pfree(nodes);
3027 1336 : return;
3028 : }
3029 :
3030 : /*
3031 : * For low number of relations to drop just use a simple walk through, to
3032 : * save the bsearch overhead. The threshold to use is rather a guess than
3033 : * an exactly determined value, as it depends on many factors (CPU and RAM
3034 : * speeds, amount of shared buffers etc.).
3035 : */
3036 10964 : use_bsearch = n > DROP_RELS_BSEARCH_THRESHOLD;
3037 :
3038 : /* sort the list of rnodes if necessary */
3039 10964 : if (use_bsearch)
3040 192 : pg_qsort(nodes, n, sizeof(RelFileNode), rnode_comparator);
3041 :
3042 178962388 : for (i = 0; i < NBuffers; i++)
3043 : {
3044 178951424 : RelFileNode *rnode = NULL;
3045 178951424 : BufferDesc *bufHdr = GetBufferDescriptor(i);
3046 : uint32 buf_state;
3047 :
3048 : /*
3049 : * As in DropRelFileNodeBuffers, an unlocked precheck should be safe
3050 : * and saves some cycles.
3051 : */
3052 :
3053 178951424 : if (!use_bsearch)
3054 : {
3055 : int j;
3056 :
3057 696287792 : for (j = 0; j < n; j++)
3058 : {
3059 520592962 : if (RelFileNodeEquals(bufHdr->tag.rnode, nodes[j]))
3060 : {
3061 110866 : rnode = &nodes[j];
3062 110866 : break;
3063 : }
3064 : }
3065 : }
3066 : else
3067 : {
3068 3145728 : rnode = bsearch((const void *) &(bufHdr->tag.rnode),
3069 : nodes, n, sizeof(RelFileNode),
3070 : rnode_comparator);
3071 : }
3072 :
3073 : /* buffer doesn't belong to any of the given relfilenodes; skip it */
3074 178951424 : if (rnode == NULL)
3075 178838760 : continue;
3076 :
3077 112664 : buf_state = LockBufHdr(bufHdr);
3078 112664 : if (RelFileNodeEquals(bufHdr->tag.rnode, (*rnode)))
3079 112664 : InvalidateBuffer(bufHdr); /* releases spinlock */
3080 : else
3081 0 : UnlockBufHdr(bufHdr, buf_state);
3082 : }
3083 :
3084 10964 : pfree(nodes);
3085 : }
3086 :
3087 : /* ---------------------------------------------------------------------
3088 : * DropDatabaseBuffers
3089 : *
3090 : * This function removes all the buffers in the buffer cache for a
3091 : * particular database. Dirty pages are simply dropped, without
3092 : * bothering to write them out first. This is used when we destroy a
3093 : * database, to avoid trying to flush data to disk when the directory
3094 : * tree no longer exists. Implementation is pretty similar to
3095 : * DropRelFileNodeBuffers() which is for destroying just one relation.
3096 : * --------------------------------------------------------------------
3097 : */
3098 : void
3099 18 : DropDatabaseBuffers(Oid dbid)
3100 : {
3101 : int i;
3102 :
3103 : /*
3104 : * We needn't consider local buffers, since by assumption the target
3105 : * database isn't our own.
3106 : */
3107 :
3108 197394 : for (i = 0; i < NBuffers; i++)
3109 : {
3110 197376 : BufferDesc *bufHdr = GetBufferDescriptor(i);
3111 : uint32 buf_state;
3112 :
3113 : /*
3114 : * As in DropRelFileNodeBuffers, an unlocked precheck should be safe
3115 : * and saves some cycles.
3116 : */
3117 197376 : if (bufHdr->tag.rnode.dbNode != dbid)
3118 195506 : continue;
3119 :
3120 1870 : buf_state = LockBufHdr(bufHdr);
3121 1870 : if (bufHdr->tag.rnode.dbNode == dbid)
3122 1870 : InvalidateBuffer(bufHdr); /* releases spinlock */
3123 : else
3124 0 : UnlockBufHdr(bufHdr, buf_state);
3125 : }
3126 18 : }
3127 :
3128 : /* -----------------------------------------------------------------
3129 : * PrintBufferDescs
3130 : *
3131 : * this function prints all the buffer descriptors, for debugging
3132 : * use only.
3133 : * -----------------------------------------------------------------
3134 : */
3135 : #ifdef NOT_USED
3136 : void
3137 : PrintBufferDescs(void)
3138 : {
3139 : int i;
3140 :
3141 : for (i = 0; i < NBuffers; ++i)
3142 : {
3143 : BufferDesc *buf = GetBufferDescriptor(i);
3144 : Buffer b = BufferDescriptorGetBuffer(buf);
3145 :
3146 : /* theoretically we should lock the bufhdr here */
3147 : elog(LOG,
3148 : "[%02d] (freeNext=%d, rel=%s, "
3149 : "blockNum=%u, flags=0x%x, refcount=%u %d)",
3150 : i, buf->freeNext,
3151 : relpathbackend(buf->tag.rnode, InvalidBackendId, buf->tag.forkNum),
3152 : buf->tag.blockNum, buf->flags,
3153 : buf->refcount, GetPrivateRefCount(b));
3154 : }
3155 : }
3156 : #endif
3157 :
3158 : #ifdef NOT_USED
3159 : void
3160 : PrintPinnedBufs(void)
3161 : {
3162 : int i;
3163 :
3164 : for (i = 0; i < NBuffers; ++i)
3165 : {
3166 : BufferDesc *buf = GetBufferDescriptor(i);
3167 : Buffer b = BufferDescriptorGetBuffer(buf);
3168 :
3169 : if (GetPrivateRefCount(b) > 0)
3170 : {
3171 : /* theoretically we should lock the bufhdr here */
3172 : elog(LOG,
3173 : "[%02d] (freeNext=%d, rel=%s, "
3174 : "blockNum=%u, flags=0x%x, refcount=%u %d)",
3175 : i, buf->freeNext,
3176 : relpathperm(buf->tag.rnode, buf->tag.forkNum),
3177 : buf->tag.blockNum, buf->flags,
3178 : buf->refcount, GetPrivateRefCount(b));
3179 : }
3180 : }
3181 : }
3182 : #endif
3183 :
3184 : /* ---------------------------------------------------------------------
3185 : * FlushRelationBuffers
3186 : *
3187 : * This function writes all dirty pages of a relation out to disk
3188 : * (or more accurately, out to kernel disk buffers), ensuring that the
3189 : * kernel has an up-to-date view of the relation.
3190 : *
3191 : * Generally, the caller should be holding AccessExclusiveLock on the
3192 : * target relation to ensure that no other backend is busy dirtying
3193 : * more blocks of the relation; the effects can't be expected to last
3194 : * after the lock is released.
3195 : *
3196 : * XXX currently it sequentially searches the buffer pool, should be
3197 : * changed to more clever ways of searching. This routine is not
3198 : * used in any performance-critical code paths, so it's not worth
3199 : * adding additional overhead to normal paths to make it go faster;
3200 : * but see also DropRelFileNodeBuffers.
3201 : * --------------------------------------------------------------------
3202 : */
3203 : void
3204 526 : FlushRelationBuffers(Relation rel)
3205 : {
3206 : int i;
3207 : BufferDesc *bufHdr;
3208 :
3209 : /* Open rel at the smgr level if not already done */
3210 526 : RelationOpenSmgr(rel);
3211 :
3212 526 : if (RelationUsesLocalBuffers(rel))
3213 : {
3214 0 : for (i = 0; i < NLocBuffer; i++)
3215 : {
3216 : uint32 buf_state;
3217 :
3218 0 : bufHdr = GetLocalBufferDescriptor(i);
3219 0 : if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) &&
3220 0 : ((buf_state = pg_atomic_read_u32(&bufHdr->state)) &
3221 : (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
3222 : {
3223 : ErrorContextCallback errcallback;
3224 : Page localpage;
3225 :
3226 0 : localpage = (char *) LocalBufHdrGetBlock(bufHdr);
3227 :
3228 : /* Setup error traceback support for ereport() */
3229 0 : errcallback.callback = local_buffer_write_error_callback;
3230 0 : errcallback.arg = (void *) bufHdr;
3231 0 : errcallback.previous = error_context_stack;
3232 0 : error_context_stack = &errcallback;
3233 :
3234 0 : PageSetChecksumInplace(localpage, bufHdr->tag.blockNum);
3235 :
3236 0 : smgrwrite(rel->rd_smgr,
3237 : bufHdr->tag.forkNum,
3238 : bufHdr->tag.blockNum,
3239 : localpage,
3240 : false);
3241 :
3242 0 : buf_state &= ~(BM_DIRTY | BM_JUST_DIRTIED);
3243 0 : pg_atomic_unlocked_write_u32(&bufHdr->state, buf_state);
3244 :
3245 : /* Pop the error context stack */
3246 0 : error_context_stack = errcallback.previous;
3247 : }
3248 : }
3249 :
3250 0 : return;
3251 : }
3252 :
3253 : /* Make sure we can handle the pin inside the loop */
3254 526 : ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
3255 :
3256 8618510 : for (i = 0; i < NBuffers; i++)
3257 : {
3258 : uint32 buf_state;
3259 :
3260 8617984 : bufHdr = GetBufferDescriptor(i);
3261 :
3262 : /*
3263 : * As in DropRelFileNodeBuffers, an unlocked precheck should be safe
3264 : * and saves some cycles.
3265 : */
3266 8617984 : if (!RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node))
3267 8614130 : continue;
3268 :
3269 3854 : ReservePrivateRefCountEntry();
3270 :
3271 3854 : buf_state = LockBufHdr(bufHdr);
3272 7708 : if (RelFileNodeEquals(bufHdr->tag.rnode, rel->rd_node) &&
3273 3854 : (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
3274 : {
3275 3740 : PinBuffer_Locked(bufHdr);
3276 3740 : LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
3277 3740 : FlushBuffer(bufHdr, rel->rd_smgr);
3278 3740 : LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
3279 3740 : UnpinBuffer(bufHdr, true);
3280 : }
3281 : else
3282 114 : UnlockBufHdr(bufHdr, buf_state);
3283 : }
3284 : }
3285 :
3286 : /* ---------------------------------------------------------------------
3287 : * FlushDatabaseBuffers
3288 : *
3289 : * This function writes all dirty pages of a database out to disk
3290 : * (or more accurately, out to kernel disk buffers), ensuring that the
3291 : * kernel has an up-to-date view of the database.
3292 : *
3293 : * Generally, the caller should be holding an appropriate lock to ensure
3294 : * no other backend is active in the target database; otherwise more
3295 : * pages could get dirtied.
3296 : *
3297 : * Note we don't worry about flushing any pages of temporary relations.
3298 : * It's assumed these wouldn't be interesting.
3299 : * --------------------------------------------------------------------
3300 : */
3301 : void
3302 4 : FlushDatabaseBuffers(Oid dbid)
3303 : {
3304 : int i;
3305 : BufferDesc *bufHdr;
3306 :
3307 : /* Make sure we can handle the pin inside the loop */
3308 4 : ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
3309 :
3310 516 : for (i = 0; i < NBuffers; i++)
3311 : {
3312 : uint32 buf_state;
3313 :
3314 512 : bufHdr = GetBufferDescriptor(i);
3315 :
3316 : /*
3317 : * As in DropRelFileNodeBuffers, an unlocked precheck should be safe
3318 : * and saves some cycles.
3319 : */
3320 512 : if (bufHdr->tag.rnode.dbNode != dbid)
3321 512 : continue;
3322 :
3323 0 : ReservePrivateRefCountEntry();
3324 :
3325 0 : buf_state = LockBufHdr(bufHdr);
3326 0 : if (bufHdr->tag.rnode.dbNode == dbid &&
3327 0 : (buf_state & (BM_VALID | BM_DIRTY)) == (BM_VALID | BM_DIRTY))
3328 : {
3329 0 : PinBuffer_Locked(bufHdr);
3330 0 : LWLockAcquire(BufferDescriptorGetContentLock(bufHdr), LW_SHARED);
3331 0 : FlushBuffer(bufHdr, NULL);
3332 0 : LWLockRelease(BufferDescriptorGetContentLock(bufHdr));
3333 0 : UnpinBuffer(bufHdr, true);
3334 : }
3335 : else
3336 0 : UnlockBufHdr(bufHdr, buf_state);
3337 : }
3338 4 : }
3339 :
3340 : /*
3341 : * Flush a previously, shared or exclusively, locked and pinned buffer to the
3342 : * OS.
3343 : */
3344 : void
3345 0 : FlushOneBuffer(Buffer buffer)
3346 : {
3347 : BufferDesc *bufHdr;
3348 :
3349 : /* currently not needed, but no fundamental reason not to support */
3350 : Assert(!BufferIsLocal(buffer));
3351 :
3352 : Assert(BufferIsPinned(buffer));
3353 :
3354 0 : bufHdr = GetBufferDescriptor(buffer - 1);
3355 :
3356 : Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr)));
3357 :
3358 0 : FlushBuffer(bufHdr, NULL);
3359 0 : }
3360 :
3361 : /*
3362 : * ReleaseBuffer -- release the pin on a buffer
3363 : */
3364 : void
3365 73775940 : ReleaseBuffer(Buffer buffer)
3366 : {
3367 73775940 : if (!BufferIsValid(buffer))
3368 0 : elog(ERROR, "bad buffer ID: %d", buffer);
3369 :
3370 73775940 : if (BufferIsLocal(buffer))
3371 : {
3372 802090 : ResourceOwnerForgetBuffer(CurrentResourceOwner, buffer);
3373 :
3374 : Assert(LocalRefCount[-buffer - 1] > 0);
3375 802090 : LocalRefCount[-buffer - 1]--;
3376 802090 : return;
3377 : }
3378 :
3379 72973850 : UnpinBuffer(GetBufferDescriptor(buffer - 1), true);
3380 : }
3381 :
3382 : /*
3383 : * UnlockReleaseBuffer -- release the content lock and pin on a buffer
3384 : *
3385 : * This is just a shorthand for a common combination.
3386 : */
3387 : void
3388 34701102 : UnlockReleaseBuffer(Buffer buffer)
3389 : {
3390 34701102 : LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
3391 34701102 : ReleaseBuffer(buffer);
3392 34701102 : }
3393 :
3394 : /*
3395 : * IncrBufferRefCount
3396 : * Increment the pin count on a buffer that we have *already* pinned
3397 : * at least once.
3398 : *
3399 : * This function cannot be used on a buffer we do not have pinned,
3400 : * because it doesn't change the shared buffer state.
3401 : */
3402 : void
3403 13314004 : IncrBufferRefCount(Buffer buffer)
3404 : {
3405 : Assert(BufferIsPinned(buffer));
3406 13314004 : ResourceOwnerEnlargeBuffers(CurrentResourceOwner);
3407 13314004 : if (BufferIsLocal(buffer))
3408 446218 : LocalRefCount[-buffer - 1]++;
3409 : else
3410 : {
3411 : PrivateRefCountEntry *ref;
3412 :
3413 12867786 : ref = GetPrivateRefCountEntry(buffer, true);
3414 : Assert(ref != NULL);
3415 12867786 : ref->refcount++;
3416 : }
3417 13314004 : ResourceOwnerRememberBuffer(CurrentResourceOwner, buffer);
3418 13314004 : }
3419 :
3420 : /*
3421 : * MarkBufferDirtyHint
3422 : *
3423 : * Mark a buffer dirty for non-critical changes.
3424 : *
3425 : * This is essentially the same as MarkBufferDirty, except:
3426 : *
3427 : * 1. The caller does not write WAL; so if checksums are enabled, we may need
3428 : * to write an XLOG_FPI WAL record to protect against torn pages.
3429 : * 2. The caller might have only share-lock instead of exclusive-lock on the
3430 : * buffer's content lock.
3431 : * 3. This function does not guarantee that the buffer is always marked dirty
3432 : * (due to a race condition), so it cannot be used for important changes.
3433 : */
3434 : void
3435 17653132 : MarkBufferDirtyHint(Buffer buffer, bool buffer_std)
3436 : {
3437 : BufferDesc *bufHdr;
3438 17653132 : Page page = BufferGetPage(buffer);
3439 :
3440 17653132 : if (!BufferIsValid(buffer))
3441 0 : elog(ERROR, "bad buffer ID: %d", buffer);
3442 :
3443 17653132 : if (BufferIsLocal(buffer))
3444 : {
3445 1538744 : MarkLocalBufferDirty(buffer);
3446 1538744 : return;
3447 : }
3448 :
3449 16114388 : bufHdr = GetBufferDescriptor(buffer - 1);
3450 :
3451 : Assert(GetPrivateRefCount(buffer) > 0);
3452 : /* here, either share or exclusive lock is OK */
3453 : Assert(LWLockHeldByMe(BufferDescriptorGetContentLock(bufHdr)));
3454 :
3455 : /*
3456 : * This routine might get called many times on the same page, if we are
3457 : * making the first scan after commit of an xact that added/deleted many
3458 : * tuples. So, be as quick as we can if the buffer is already dirty. We
3459 : * do this by not acquiring spinlock if it looks like the status bits are
3460 : * already set. Since we make this test unlocked, there's a chance we
3461 : * might fail to notice that the flags have just been cleared, and failed
3462 : * to reset them, due to memory-ordering issues. But since this function
3463 : * is only intended to be used in cases where failing to write out the
3464 : * data would be harmless anyway, it doesn't really matter.
3465 : */
3466 16114388 : if ((pg_atomic_read_u32(&bufHdr->state) & (BM_DIRTY | BM_JUST_DIRTIED)) !=
3467 : (BM_DIRTY | BM_JUST_DIRTIED))
3468 : {
3469 170576 : XLogRecPtr lsn = InvalidXLogRecPtr;
3470 170576 : bool dirtied = false;
3471 170576 : bool delayChkpt = false;
3472 : uint32 buf_state;
3473 :
3474 : /*
3475 : * If we need to protect hint bit updates from torn writes, WAL-log a
3476 : * full page image of the page. This full page image is only necessary
3477 : * if the hint bit update is the first change to the page since the
3478 : * last checkpoint.
3479 : *
3480 : * We don't check full_page_writes here because that logic is included
3481 : * when we call XLogInsert() since the value changes dynamically.
3482 : */
3483 282636 : if (XLogHintBitIsNeeded() &&
3484 112060 : (pg_atomic_read_u32(&bufHdr->state) & BM_PERMANENT))
3485 : {
3486 : /*
3487 : * If we're in recovery we cannot dirty a page because of a hint.
3488 : * We can set the hint, just not dirty the page as a result so the
3489 : * hint is lost when we evict the page or shutdown.
3490 : *
3491 : * See src/backend/storage/page/README for longer discussion.
3492 : */
3493 112060 : if (RecoveryInProgress())
3494 108418 : return;
3495 :
3496 : /*
3497 : * If the block is already dirty because we either made a change
3498 : * or set a hint already, then we don't need to write a full page
3499 : * image. Note that aggressive cleaning of blocks dirtied by hint
3500 : * bit setting would increase the call rate. Bulk setting of hint
3501 : * bits would reduce the call rate...
3502 : *
3503 : * We must issue the WAL record before we mark the buffer dirty.
3504 : * Otherwise we might write the page before we write the WAL. That
3505 : * causes a race condition, since a checkpoint might occur between
3506 : * writing the WAL record and marking the buffer dirty. We solve
3507 : * that with a kluge, but one that is already in use during
3508 : * transaction commit to prevent race conditions. Basically, we
3509 : * simply prevent the checkpoint WAL record from being written
3510 : * until we have marked the buffer dirty. We don't start the
3511 : * checkpoint flush until we have marked dirty, so our checkpoint
3512 : * must flush the change to disk successfully or the checkpoint
3513 : * never gets written, so crash recovery will fix.
3514 : *
3515 : * It's possible we may enter here without an xid, so it is
3516 : * essential that CreateCheckpoint waits for virtual transactions
3517 : * rather than full transactionids.
3518 : */
3519 3642 : MyPgXact->delayChkpt = delayChkpt = true;
3520 3642 : lsn = XLogSaveBufferForHint(buffer, buffer_std);
3521 : }
3522 :
3523 62158 : buf_state = LockBufHdr(bufHdr);
3524 :
3525 : Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
3526 :
3527 62158 : if (!(buf_state & BM_DIRTY))
3528 : {
3529 62158 : dirtied = true; /* Means "will be dirtied by this action" */
3530 :
3531 : /*
3532 : * Set the page LSN if we wrote a backup block. We aren't supposed
3533 : * to set this when only holding a share lock but as long as we
3534 : * serialise it somehow we're OK. We choose to set LSN while
3535 : * holding the buffer header lock, which causes any reader of an
3536 : * LSN who holds only a share lock to also obtain a buffer header
3537 : * lock before using PageGetLSN(), which is enforced in
3538 : * BufferGetLSNAtomic().
3539 : *
3540 : * If checksums are enabled, you might think we should reset the
3541 : * checksum here. That will happen when the page is written
3542 : * sometime later in this checkpoint cycle.
3543 : */
3544 62158 : if (!XLogRecPtrIsInvalid(lsn))
3545 2226 : PageSetLSN(page, lsn);
3546 : }
3547 :
3548 62158 : buf_state |= BM_DIRTY | BM_JUST_DIRTIED;
3549 62158 : UnlockBufHdr(bufHdr, buf_state);
3550 :
3551 62158 : if (delayChkpt)
3552 3642 : MyPgXact->delayChkpt = false;
3553 :
3554 62158 : if (dirtied)
3555 : {
3556 62158 : VacuumPageDirty++;
3557 62158 : pgBufferUsage.shared_blks_dirtied++;
3558 62158 : if (VacuumCostActive)
3559 656 : VacuumCostBalance += VacuumCostPageDirty;
3560 : }
3561 : }
3562 : }
3563 :
3564 : /*
3565 : * Release buffer content locks for shared buffers.
3566 : *
3567 : * Used to clean up after errors.
3568 : *
3569 : * Currently, we can expect that lwlock.c's LWLockReleaseAll() took care
3570 : * of releasing buffer content locks per se; the only thing we need to deal
3571 : * with here is clearing any PIN_COUNT request that was in progress.
3572 : */
3573 : void
3574 33982 : UnlockBuffers(void)
3575 : {
3576 33982 : BufferDesc *buf = PinCountWaitBuf;
3577 :
3578 33982 : if (buf)
3579 : {
3580 : uint32 buf_state;
3581 :
3582 0 : buf_state = LockBufHdr(buf);
3583 :
3584 : /*
3585 : * Don't complain if flag bit not set; it could have been reset but we
3586 : * got a cancel/die interrupt before getting the signal.
3587 : */
3588 0 : if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
3589 0 : buf->wait_backend_pid == MyProcPid)
3590 0 : buf_state &= ~BM_PIN_COUNT_WAITER;
3591 :
3592 0 : UnlockBufHdr(buf, buf_state);
3593 :
3594 0 : PinCountWaitBuf = NULL;
3595 : }
3596 33982 : }
3597 :
3598 : /*
3599 : * Acquire or release the content_lock for the buffer.
3600 : */
3601 : void
3602 228309750 : LockBuffer(Buffer buffer, int mode)
3603 : {
3604 : BufferDesc *buf;
3605 :
3606 : Assert(BufferIsValid(buffer));
3607 228309750 : if (BufferIsLocal(buffer))
3608 2197698 : return; /* local buffers need no lock */
3609 :
3610 226112052 : buf = GetBufferDescriptor(buffer - 1);
3611 :
3612 226112052 : if (mode == BUFFER_LOCK_UNLOCK)
3613 113880034 : LWLockRelease(BufferDescriptorGetContentLock(buf));
3614 112232018 : else if (mode == BUFFER_LOCK_SHARE)
3615 77334700 : LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_SHARED);
3616 34897318 : else if (mode == BUFFER_LOCK_EXCLUSIVE)
3617 34897318 : LWLockAcquire(BufferDescriptorGetContentLock(buf), LW_EXCLUSIVE);
3618 : else
3619 0 : elog(ERROR, "unrecognized buffer lock mode: %d", mode);
3620 : }
3621 :
3622 : /*
3623 : * Acquire the content_lock for the buffer, but only if we don't have to wait.
3624 : *
3625 : * This assumes the caller wants BUFFER_LOCK_EXCLUSIVE mode.
3626 : */
3627 : bool
3628 1403674 : ConditionalLockBuffer(Buffer buffer)
3629 : {
3630 : BufferDesc *buf;
3631 :
3632 : Assert(BufferIsValid(buffer));
3633 1403674 : if (BufferIsLocal(buffer))
3634 32 : return true; /* act as though we got it */
3635 :
3636 1403642 : buf = GetBufferDescriptor(buffer - 1);
3637 :
3638 1403642 : return LWLockConditionalAcquire(BufferDescriptorGetContentLock(buf),
3639 : LW_EXCLUSIVE);
3640 : }
3641 :
3642 : /*
3643 : * LockBufferForCleanup - lock a buffer in preparation for deleting items
3644 : *
3645 : * Items may be deleted from a disk page only when the caller (a) holds an
3646 : * exclusive lock on the buffer and (b) has observed that no other backend
3647 : * holds a pin on the buffer. If there is a pin, then the other backend
3648 : * might have a pointer into the buffer (for example, a heapscan reference
3649 : * to an item --- see README for more details). It's OK if a pin is added
3650 : * after the cleanup starts, however; the newly-arrived backend will be
3651 : * unable to look at the page until we release the exclusive lock.
3652 : *
3653 : * To implement this protocol, a would-be deleter must pin the buffer and
3654 : * then call LockBufferForCleanup(). LockBufferForCleanup() is similar to
3655 : * LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE), except that it loops until
3656 : * it has successfully observed pin count = 1.
3657 : */
3658 : void
3659 113154 : LockBufferForCleanup(Buffer buffer)
3660 : {
3661 : BufferDesc *bufHdr;
3662 :
3663 : Assert(BufferIsValid(buffer));
3664 : Assert(PinCountWaitBuf == NULL);
3665 :
3666 113154 : if (BufferIsLocal(buffer))
3667 : {
3668 : /* There should be exactly one pin */
3669 0 : if (LocalRefCount[-buffer - 1] != 1)
3670 0 : elog(ERROR, "incorrect local pin count: %d",
3671 : LocalRefCount[-buffer - 1]);
3672 : /* Nobody else to wait for */
3673 0 : return;
3674 : }
3675 :
3676 : /* There should be exactly one local pin */
3677 113154 : if (GetPrivateRefCount(buffer) != 1)
3678 0 : elog(ERROR, "incorrect local pin count: %d",
3679 : GetPrivateRefCount(buffer));
3680 :
3681 113154 : bufHdr = GetBufferDescriptor(buffer - 1);
3682 :
3683 : for (;;)
3684 0 : {
3685 : uint32 buf_state;
3686 :
3687 : /* Try to acquire lock */
3688 113154 : LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE);
3689 113154 : buf_state = LockBufHdr(bufHdr);
3690 :
3691 : Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
3692 113154 : if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
3693 : {
3694 : /* Successfully acquired exclusive lock with pincount 1 */
3695 113154 : UnlockBufHdr(bufHdr, buf_state);
3696 113154 : return;
3697 : }
3698 : /* Failed, so mark myself as waiting for pincount 1 */
3699 0 : if (buf_state & BM_PIN_COUNT_WAITER)
3700 : {
3701 0 : UnlockBufHdr(bufHdr, buf_state);
3702 0 : LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
3703 0 : elog(ERROR, "multiple backends attempting to wait for pincount 1");
3704 : }
3705 0 : bufHdr->wait_backend_pid = MyProcPid;
3706 0 : PinCountWaitBuf = bufHdr;
3707 0 : buf_state |= BM_PIN_COUNT_WAITER;
3708 0 : UnlockBufHdr(bufHdr, buf_state);
3709 0 : LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
3710 :
3711 : /* Wait to be signaled by UnpinBuffer() */
3712 0 : if (InHotStandby)
3713 : {
3714 : /* Publish the bufid that Startup process waits on */
3715 0 : SetStartupBufferPinWaitBufId(buffer - 1);
3716 : /* Set alarm and then wait to be signaled by UnpinBuffer() */
3717 0 : ResolveRecoveryConflictWithBufferPin();
3718 : /* Reset the published bufid */
3719 0 : SetStartupBufferPinWaitBufId(-1);
3720 : }
3721 : else
3722 0 : ProcWaitForSignal(PG_WAIT_BUFFER_PIN);
3723 :
3724 : /*
3725 : * Remove flag marking us as waiter. Normally this will not be set
3726 : * anymore, but ProcWaitForSignal() can return for other signals as
3727 : * well. We take care to only reset the flag if we're the waiter, as
3728 : * theoretically another backend could have started waiting. That's
3729 : * impossible with the current usages due to table level locking, but
3730 : * better be safe.
3731 : */
3732 0 : buf_state = LockBufHdr(bufHdr);
3733 0 : if ((buf_state & BM_PIN_COUNT_WAITER) != 0 &&
3734 0 : bufHdr->wait_backend_pid == MyProcPid)
3735 0 : buf_state &= ~BM_PIN_COUNT_WAITER;
3736 0 : UnlockBufHdr(bufHdr, buf_state);
3737 :
3738 0 : PinCountWaitBuf = NULL;
3739 : /* Loop back and try again */
3740 : }
3741 : }
3742 :
3743 : /*
3744 : * Check called from RecoveryConflictInterrupt handler when Startup
3745 : * process requests cancellation of all pin holders that are blocking it.
3746 : */
3747 : bool
3748 0 : HoldingBufferPinThatDelaysRecovery(void)
3749 : {
3750 0 : int bufid = GetStartupBufferPinWaitBufId();
3751 :
3752 : /*
3753 : * If we get woken slowly then it's possible that the Startup process was
3754 : * already woken by other backends before we got here. Also possible that
3755 : * we get here by multiple interrupts or interrupts at inappropriate
3756 : * times, so make sure we do nothing if the bufid is not set.
3757 : */
3758 0 : if (bufid < 0)
3759 0 : return false;
3760 :
3761 0 : if (GetPrivateRefCount(bufid + 1) > 0)
3762 0 : return true;
3763 :
3764 0 : return false;
3765 : }
3766 :
3767 : /*
3768 : * ConditionalLockBufferForCleanup - as above, but don't wait to get the lock
3769 : *
3770 : * We won't loop, but just check once to see if the pin count is OK. If
3771 : * not, return false with no lock held.
3772 : */
3773 : bool
3774 251248 : ConditionalLockBufferForCleanup(Buffer buffer)
3775 : {
3776 : BufferDesc *bufHdr;
3777 : uint32 buf_state,
3778 : refcount;
3779 :
3780 : Assert(BufferIsValid(buffer));
3781 :
3782 251248 : if (BufferIsLocal(buffer))
3783 : {
3784 32 : refcount = LocalRefCount[-buffer - 1];
3785 : /* There should be exactly one pin */
3786 : Assert(refcount > 0);
3787 32 : if (refcount != 1)
3788 28 : return false;
3789 : /* Nobody else to wait for */
3790 4 : return true;
3791 : }
3792 :
3793 : /* There should be exactly one local pin */
3794 251216 : refcount = GetPrivateRefCount(buffer);
3795 : Assert(refcount);
3796 251216 : if (refcount != 1)
3797 60 : return false;
3798 :
3799 : /* Try to acquire lock */
3800 251156 : if (!ConditionalLockBuffer(buffer))
3801 8 : return false;
3802 :
3803 251148 : bufHdr = GetBufferDescriptor(buffer - 1);
3804 251148 : buf_state = LockBufHdr(bufHdr);
3805 251148 : refcount = BUF_STATE_GET_REFCOUNT(buf_state);
3806 :
3807 : Assert(refcount > 0);
3808 251148 : if (refcount == 1)
3809 : {
3810 : /* Successfully acquired exclusive lock with pincount 1 */
3811 251124 : UnlockBufHdr(bufHdr, buf_state);
3812 251124 : return true;
3813 : }
3814 :
3815 : /* Failed, so release the lock */
3816 24 : UnlockBufHdr(bufHdr, buf_state);
3817 24 : LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
3818 24 : return false;
3819 : }
3820 :
3821 : /*
3822 : * IsBufferCleanupOK - as above, but we already have the lock
3823 : *
3824 : * Check whether it's OK to perform cleanup on a buffer we've already
3825 : * locked. If we observe that the pin count is 1, our exclusive lock
3826 : * happens to be a cleanup lock, and we can proceed with anything that
3827 : * would have been allowable had we sought a cleanup lock originally.
3828 : */
3829 : bool
3830 1836 : IsBufferCleanupOK(Buffer buffer)
3831 : {
3832 : BufferDesc *bufHdr;
3833 : uint32 buf_state;
3834 :
3835 : Assert(BufferIsValid(buffer));
3836 :
3837 1836 : if (BufferIsLocal(buffer))
3838 : {
3839 : /* There should be exactly one pin */
3840 0 : if (LocalRefCount[-buffer - 1] != 1)
3841 0 : return false;
3842 : /* Nobody else to wait for */
3843 0 : return true;
3844 : }
3845 :
3846 : /* There should be exactly one local pin */
3847 1836 : if (GetPrivateRefCount(buffer) != 1)
3848 0 : return false;
3849 :
3850 1836 : bufHdr = GetBufferDescriptor(buffer - 1);
3851 :
3852 : /* caller must hold exclusive lock on buffer */
3853 : Assert(LWLockHeldByMeInMode(BufferDescriptorGetContentLock(bufHdr),
3854 : LW_EXCLUSIVE));
3855 :
3856 1836 : buf_state = LockBufHdr(bufHdr);
3857 :
3858 : Assert(BUF_STATE_GET_REFCOUNT(buf_state) > 0);
3859 1836 : if (BUF_STATE_GET_REFCOUNT(buf_state) == 1)
3860 : {
3861 : /* pincount is OK. */
3862 1836 : UnlockBufHdr(bufHdr, buf_state);
3863 1836 : return true;
3864 : }
3865 :
3866 0 : UnlockBufHdr(bufHdr, buf_state);
3867 0 : return false;
3868 : }
3869 :
3870 :
3871 : /*
3872 : * Functions for buffer I/O handling
3873 : *
3874 : * Note: We assume that nested buffer I/O never occurs.
3875 : * i.e at most one io_in_progress lock is held per proc.
3876 : *
3877 : * Also note that these are used only for shared buffers, not local ones.
3878 : */
3879 :
3880 : /*
3881 : * WaitIO -- Block until the IO_IN_PROGRESS flag on 'buf' is cleared.
3882 : */
3883 : static void
3884 0 : WaitIO(BufferDesc *buf)
3885 : {
3886 : /*
3887 : * Changed to wait until there's no IO - Inoue 01/13/2000
3888 : *
3889 : * Note this is *necessary* because an error abort in the process doing
3890 : * I/O could release the io_in_progress_lock prematurely. See
3891 : * AbortBufferIO.
3892 : */
3893 : for (;;)
3894 0 : {
3895 : uint32 buf_state;
3896 :
3897 : /*
3898 : * It may not be necessary to acquire the spinlock to check the flag
3899 : * here, but since this test is essential for correctness, we'd better
3900 : * play it safe.
3901 : */
3902 0 : buf_state = LockBufHdr(buf);
3903 0 : UnlockBufHdr(buf, buf_state);
3904 :
3905 0 : if (!(buf_state & BM_IO_IN_PROGRESS))
3906 0 : break;
3907 0 : LWLockAcquire(BufferDescriptorGetIOLock(buf), LW_SHARED);
3908 0 : LWLockRelease(BufferDescriptorGetIOLock(buf));
3909 : }
3910 0 : }
3911 :
3912 : /*
3913 : * StartBufferIO: begin I/O on this buffer
3914 : * (Assumptions)
3915 : * My process is executing no IO
3916 : * The buffer is Pinned
3917 : *
3918 : * In some scenarios there are race conditions in which multiple backends
3919 : * could attempt the same I/O operation concurrently. If someone else
3920 : * has already started I/O on this buffer then we will block on the
3921 : * io_in_progress lock until he's done.
3922 : *
3923 : * Input operations are only attempted on buffers that are not BM_VALID,
3924 : * and output operations only on buffers that are BM_VALID and BM_DIRTY,
3925 : * so we can always tell if the work is already done.
3926 : *
3927 : * Returns true if we successfully marked the buffer as I/O busy,
3928 : * false if someone else already did the work.
3929 : */
3930 : static bool
3931 1274618 : StartBufferIO(BufferDesc *buf, bool forInput)
3932 : {
3933 : uint32 buf_state;
3934 :
3935 : Assert(!InProgressBuf);
3936 :
3937 : for (;;)
3938 : {
3939 : /*
3940 : * Grab the io_in_progress lock so that other processes can wait for
3941 : * me to finish the I/O.
3942 : */
3943 1274618 : LWLockAcquire(BufferDescriptorGetIOLock(buf), LW_EXCLUSIVE);
3944 :
3945 1274618 : buf_state = LockBufHdr(buf);
3946 :
3947 1274618 : if (!(buf_state & BM_IO_IN_PROGRESS))
3948 1274618 : break;
3949 :
3950 : /*
3951 : * The only way BM_IO_IN_PROGRESS could be set when the io_in_progress
3952 : * lock isn't held is if the process doing the I/O is recovering from
3953 : * an error (see AbortBufferIO). If that's the case, we must wait for
3954 : * him to get unwedged.
3955 : */
3956 0 : UnlockBufHdr(buf, buf_state);
3957 0 : LWLockRelease(BufferDescriptorGetIOLock(buf));
3958 0 : WaitIO(buf);
3959 : }
3960 :
3961 : /* Once we get here, there is definitely no I/O active on this buffer */
3962 :
3963 1274618 : if (forInput ? (buf_state & BM_VALID) : !(buf_state & BM_DIRTY))
3964 : {
3965 : /* someone else already did the I/O */
3966 6 : UnlockBufHdr(buf, buf_state);
3967 6 : LWLockRelease(BufferDescriptorGetIOLock(buf));
3968 6 : return false;
3969 : }
3970 :
3971 1274612 : buf_state |= BM_IO_IN_PROGRESS;
3972 1274612 : UnlockBufHdr(buf, buf_state);
3973 :
3974 1274612 : InProgressBuf = buf;
3975 1274612 : IsForInput = forInput;
3976 :
3977 1274612 : return true;
3978 : }
3979 :
3980 : /*
3981 : * TerminateBufferIO: release a buffer we were doing I/O on
3982 : * (Assumptions)
3983 : * My process is executing IO for the buffer
3984 : * BM_IO_IN_PROGRESS bit is set for the buffer
3985 : * We hold the buffer's io_in_progress lock
3986 : * The buffer is Pinned
3987 : *
3988 : * If clear_dirty is true and BM_JUST_DIRTIED is not set, we clear the
3989 : * buffer's BM_DIRTY flag. This is appropriate when terminating a
3990 : * successful write. The check on BM_JUST_DIRTIED is necessary to avoid
3991 : * marking the buffer clean if it was re-dirtied while we were writing.
3992 : *
3993 : * set_flag_bits gets ORed into the buffer's flags. It must include
3994 : * BM_IO_ERROR in a failure case. For successful completion it could
3995 : * be 0, or BM_VALID if we just finished reading in the page.
3996 : */
3997 : static void
3998 1274612 : TerminateBufferIO(BufferDesc *buf, bool clear_dirty, uint32 set_flag_bits)
3999 : {
4000 : uint32 buf_state;
4001 :
4002 : Assert(buf == InProgressBuf);
4003 :
4004 1274612 : buf_state = LockBufHdr(buf);
4005 :
4006 : Assert(buf_state & BM_IO_IN_PROGRESS);
4007 :
4008 1274612 : buf_state &= ~(BM_IO_IN_PROGRESS | BM_IO_ERROR);
4009 1274612 : if (clear_dirty && !(buf_state & BM_JUST_DIRTIED))
4010 474838 : buf_state &= ~(BM_DIRTY | BM_CHECKPOINT_NEEDED);
4011 :
4012 1274612 : buf_state |= set_flag_bits;
4013 1274612 : UnlockBufHdr(buf, buf_state);
4014 :
4015 1274612 : InProgressBuf = NULL;
4016 :
4017 1274612 : LWLockRelease(BufferDescriptorGetIOLock(buf));
4018 1274612 : }
4019 :
4020 : /*
4021 : * AbortBufferIO: Clean up any active buffer I/O after an error.
4022 : *
4023 : * All LWLocks we might have held have been released,
4024 : * but we haven't yet released buffer pins, so the buffer is still pinned.
4025 : *
4026 : * If I/O was in progress, we always set BM_IO_ERROR, even though it's
4027 : * possible the error condition wasn't related to the I/O.
4028 : */
4029 : void
4030 33982 : AbortBufferIO(void)
4031 : {
4032 33982 : BufferDesc *buf = InProgressBuf;
4033 :
4034 33982 : if (buf)
4035 : {
4036 : uint32 buf_state;
4037 :
4038 : /*
4039 : * Since LWLockReleaseAll has already been called, we're not holding
4040 : * the buffer's io_in_progress_lock. We have to re-acquire it so that
4041 : * we can use TerminateBufferIO. Anyone who's executing WaitIO on the
4042 : * buffer will be in a busy spin until we succeed in doing this.
4043 : */
4044 0 : LWLockAcquire(BufferDescriptorGetIOLock(buf), LW_EXCLUSIVE);
4045 :
4046 0 : buf_state = LockBufHdr(buf);
4047 : Assert(buf_state & BM_IO_IN_PROGRESS);
4048 0 : if (IsForInput)
4049 : {
4050 : Assert(!(buf_state & BM_DIRTY));
4051 :
4052 : /* We'd better not think buffer is valid yet */
4053 : Assert(!(buf_state & BM_VALID));
4054 0 : UnlockBufHdr(buf, buf_state);
4055 : }
4056 : else
4057 : {
4058 : Assert(buf_state & BM_DIRTY);
4059 0 : UnlockBufHdr(buf, buf_state);
4060 : /* Issue notice if this is not the first failure... */
4061 0 : if (buf_state & BM_IO_ERROR)
4062 : {
4063 : /* Buffer is pinned, so we can read tag without spinlock */
4064 : char *path;
4065 :
4066 0 : path = relpathperm(buf->tag.rnode, buf->tag.forkNum);
4067 0 : ereport(WARNING,
4068 : (errcode(ERRCODE_IO_ERROR),
4069 : errmsg("could not write block %u of %s",
4070 : buf->tag.blockNum, path),
4071 : errdetail("Multiple failures --- write error might be permanent.")));
4072 0 : pfree(path);
4073 : }
4074 : }
4075 0 : TerminateBufferIO(buf, false, BM_IO_ERROR);
4076 : }
4077 33982 : }
4078 :
4079 : /*
4080 : * Error context callback for errors occurring during shared buffer writes.
4081 : */
4082 : static void
4083 0 : shared_buffer_write_error_callback(void *arg)
4084 : {
4085 0 : BufferDesc *bufHdr = (BufferDesc *) arg;
4086 :
4087 : /* Buffer is pinned, so we can read the tag without locking the spinlock */
4088 0 : if (bufHdr != NULL)
4089 : {
4090 0 : char *path = relpathperm(bufHdr->tag.rnode, bufHdr->tag.forkNum);
4091 :
4092 0 : errcontext("writing block %u of relation %s",
4093 : bufHdr->tag.blockNum, path);
4094 0 : pfree(path);
4095 : }
4096 0 : }
4097 :
4098 : /*
4099 : * Error context callback for errors occurring during local buffer writes.
4100 : */
4101 : static void
4102 0 : local_buffer_write_error_callback(void *arg)
4103 : {
4104 0 : BufferDesc *bufHdr = (BufferDesc *) arg;
4105 :
4106 0 : if (bufHdr != NULL)
4107 : {
4108 0 : char *path = relpathbackend(bufHdr->tag.rnode, MyBackendId,
4109 : bufHdr->tag.forkNum);
4110 :
4111 0 : errcontext("writing block %u of relation %s",
4112 : bufHdr->tag.blockNum, path);
4113 0 : pfree(path);
4114 : }
4115 0 : }
4116 :
4117 : /*
4118 : * RelFileNode qsort/bsearch comparator; see RelFileNodeEquals.
4119 : */
4120 : static int
4121 18505054 : rnode_comparator(const void *p1, const void *p2)
4122 : {
4123 18505054 : RelFileNode n1 = *(const RelFileNode *) p1;
4124 18505054 : RelFileNode n2 = *(const RelFileNode *) p2;
4125 :
4126 18505054 : if (n1.relNode < n2.relNode)
4127 16767066 : return -1;
4128 1737988 : else if (n1.relNode > n2.relNode)
4129 449210 : return 1;
4130 :
4131 1288778 : if (n1.dbNode < n2.dbNode)
4132 1606 : return -1;
4133 1287172 : else if (n1.dbNode > n2.dbNode)
4134 1886 : return 1;
4135 :
4136 1285286 : if (n1.spcNode < n2.spcNode)
4137 0 : return -1;
4138 1285286 : else if (n1.spcNode > n2.spcNode)
4139 0 : return 1;
4140 : else
4141 1285286 : return 0;
4142 : }
4143 :
4144 : /*
4145 : * Lock buffer header - set BM_LOCKED in buffer state.
4146 : */
4147 : uint32
4148 49386382 : LockBufHdr(BufferDesc *desc)
4149 : {
4150 : SpinDelayStatus delayStatus;
4151 : uint32 old_buf_state;
4152 :
4153 49386382 : init_local_spin_delay(&delayStatus);
4154 :
4155 : while (true)
4156 : {
4157 : /* set BM_LOCKED flag */
4158 49386390 : old_buf_state = pg_atomic_fetch_or_u32(&desc->state, BM_LOCKED);
4159 : /* if it wasn't set before we're OK */
4160 49386386 : if (!(old_buf_state & BM_LOCKED))
4161 49386382 : break;
4162 4 : perform_spin_delay(&delayStatus);
4163 : }
4164 49386382 : finish_spin_delay(&delayStatus);
4165 49386382 : return old_buf_state | BM_LOCKED;
4166 : }
4167 :
4168 : /*
4169 : * Wait until the BM_LOCKED flag isn't set anymore and return the buffer's
4170 : * state at that point.
4171 : *
4172 : * Obviously the buffer could be locked by the time the value is returned, so
4173 : * this is primarily useful in CAS style loops.
4174 : */
4175 : static uint32
4176 4 : WaitBufHdrUnlocked(BufferDesc *buf)
4177 : {
4178 : SpinDelayStatus delayStatus;
4179 : uint32 buf_state;
4180 :
4181 4 : init_local_spin_delay(&delayStatus);
4182 :
4183 4 : buf_state = pg_atomic_read_u32(&buf->state);
4184 :
4185 158 : while (buf_state & BM_LOCKED)
4186 : {
4187 150 : perform_spin_delay(&delayStatus);
4188 150 : buf_state = pg_atomic_read_u32(&buf->state);
4189 : }
4190 :
4191 4 : finish_spin_delay(&delayStatus);
4192 :
4193 4 : return buf_state;
4194 : }
4195 :
4196 : /*
4197 : * BufferTag comparator.
4198 : */
4199 : static int
4200 2109852 : buffertag_comparator(const void *a, const void *b)
4201 : {
4202 2109852 : const BufferTag *ba = (const BufferTag *) a;
4203 2109852 : const BufferTag *bb = (const BufferTag *) b;
4204 : int ret;
4205 :
4206 2109852 : ret = rnode_comparator(&ba->rnode, &bb->rnode);
4207 :
4208 2109852 : if (ret != 0)
4209 826364 : return ret;
4210 :
4211 1283488 : if (ba->forkNum < bb->forkNum)
4212 62958 : return -1;
4213 1220530 : if (ba->forkNum > bb->forkNum)
4214 71170 : return 1;
4215 :
4216 1149360 : if (ba->blockNum < bb->blockNum)
4217 688732 : return -1;
4218 460628 : if (ba->blockNum > bb->blockNum)
4219 460606 : return 1;
4220 :
4221 22 : return 0;
4222 : }
4223 :
4224 : /*
4225 : * Comparator determining the writeout order in a checkpoint.
4226 : *
4227 : * It is important that tablespaces are compared first, the logic balancing
4228 : * writes between tablespaces relies on it.
4229 : */
4230 : static int
4231 4699218 : ckpt_buforder_comparator(const void *pa, const void *pb)
4232 : {
4233 4699218 : const CkptSortItem *a = (const CkptSortItem *) pa;
4234 4699218 : const CkptSortItem *b = (const CkptSortItem *) pb;
4235 :
4236 : /* compare tablespace */
4237 4699218 : if (a->tsId < b->tsId)
4238 51354 : return -1;
4239 4647864 : else if (a->tsId > b->tsId)
4240 76556 : return 1;
4241 : /* compare relation */
4242 4571308 : if (a->relNode < b->relNode)
4243 1325360 : return -1;
4244 3245948 : else if (a->relNode > b->relNode)
4245 1250672 : return 1;
4246 : /* compare fork */
4247 1995276 : else if (a->forkNum < b->forkNum)
4248 85810 : return -1;
4249 1909466 : else if (a->forkNum > b->forkNum)
4250 105212 : return 1;
4251 : /* compare block number */
4252 1804254 : else if (a->blockNum < b->blockNum)
4253 912604 : return -1;
4254 891650 : else if (a->blockNum > b->blockNum)
4255 890468 : return 1;
4256 : /* equal page IDs are unlikely, but not impossible */
4257 1182 : return 0;
4258 : }
4259 :
4260 : /*
4261 : * Comparator for a Min-Heap over the per-tablespace checkpoint completion
4262 : * progress.
4263 : */
4264 : static int
4265 425004 : ts_ckpt_progress_comparator(Datum a, Datum b, void *arg)
4266 : {
4267 425004 : CkptTsStatus *sa = (CkptTsStatus *) a;
4268 425004 : CkptTsStatus *sb = (CkptTsStatus *) b;
4269 :
4270 : /* we want a min-heap, so return 1 for the a < b */
4271 425004 : if (sa->progress < sb->progress)
4272 370450 : return 1;
4273 54554 : else if (sa->progress == sb->progress)
4274 1266 : return 0;
4275 : else
4276 53288 : return -1;
4277 : }
4278 :
4279 : /*
4280 : * Initialize a writeback context, discarding potential previous state.
4281 : *
4282 : * *max_pending is a pointer instead of an immediate value, so the coalesce
4283 : * limits can easily changed by the GUC mechanism, and so calling code does
4284 : * not have to check the current configuration. A value is 0 means that no
4285 : * writeback control will be performed.
4286 : */
4287 : void
4288 4092 : WritebackContextInit(WritebackContext *context, int *max_pending)
4289 : {
4290 : Assert(*max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
4291 :
4292 4092 : context->max_pending = max_pending;
4293 4092 : context->nr_pending = 0;
4294 4092 : }
4295 :
4296 : /*
4297 : * Add buffer to list of pending writeback requests.
4298 : */
4299 : void
4300 471098 : ScheduleBufferTagForWriteback(WritebackContext *context, BufferTag *tag)
4301 : {
4302 : PendingWriteback *pending;
4303 :
4304 : /*
4305 : * Add buffer to the pending writeback array, unless writeback control is
4306 : * disabled.
4307 : */
4308 471098 : if (*context->max_pending > 0)
4309 : {
4310 : Assert(*context->max_pending <= WRITEBACK_MAX_PENDING_FLUSHES);
4311 :
4312 460792 : pending = &context->pending_writebacks[context->nr_pending++];
4313 :
4314 460792 : pending->tag = *tag;
4315 : }
4316 :
4317 : /*
4318 : * Perform pending flushes if the writeback limit is exceeded. This
4319 : * includes the case where previously an item has been added, but control
4320 : * is now disabled.
4321 : */
4322 471098 : if (context->nr_pending >= *context->max_pending)
4323 23976 : IssuePendingWritebacks(context);
4324 471098 : }
4325 :
4326 : /*
4327 : * Issue all pending writeback requests, previously scheduled with
4328 : * ScheduleBufferTagForWriteback, to the OS.
4329 : *
4330 : * Because this is only used to improve the OSs IO scheduling we try to never
4331 : * error out - it's just a hint.
4332 : */
4333 : void
4334 25810 : IssuePendingWritebacks(WritebackContext *context)
4335 : {
4336 : int i;
4337 :
4338 25810 : if (context->nr_pending == 0)
4339 10312 : return;
4340 :
4341 : /*
4342 : * Executing the writes in-order can make them a lot faster, and allows to
4343 : * merge writeback requests to consecutive blocks into larger writebacks.
4344 : */
4345 15498 : qsort(&context->pending_writebacks, context->nr_pending,
4346 : sizeof(PendingWriteback), buffertag_comparator);
4347 :
4348 : /*
4349 : * Coalesce neighbouring writes, but nothing else. For that we iterate
4350 : * through the, now sorted, array of pending flushes, and look forward to
4351 : * find all neighbouring (or identical) writes.
4352 : */
4353 167168 : for (i = 0; i < context->nr_pending; i++)
4354 : {
4355 : PendingWriteback *cur;
4356 : PendingWriteback *next;
4357 : SMgrRelation reln;
4358 : int ahead;
4359 : BufferTag tag;
4360 151670 : Size nblocks = 1;
4361 :
4362 151670 : cur = &context->pending_writebacks[i];
4363 151670 : tag = cur->tag;
4364 :
4365 : /*
4366 : * Peek ahead, into following writeback requests, to see if they can
4367 : * be combined with the current one.
4368 : */
4369 459582 : for (ahead = 0; i + ahead + 1 < context->nr_pending; ahead++)
4370 : {
4371 444084 : next = &context->pending_writebacks[i + ahead + 1];
4372 :
4373 : /* different file, stop */
4374 794520 : if (!RelFileNodeEquals(cur->tag.rnode, next->tag.rnode) ||
4375 350436 : cur->tag.forkNum != next->tag.forkNum)
4376 : break;
4377 :
4378 : /* ok, block queued twice, skip */
4379 318532 : if (cur->tag.blockNum == next->tag.blockNum)
4380 18 : continue;
4381 :
4382 : /* only merge consecutive writes */
4383 318514 : if (cur->tag.blockNum + 1 != next->tag.blockNum)
4384 10620 : break;
4385 :
4386 307894 : nblocks++;
4387 307894 : cur = next;
4388 : }
4389 :
4390 151670 : i += ahead;
4391 :
4392 : /* and finally tell the kernel to write the data to storage */
4393 151670 : reln = smgropen(tag.rnode, InvalidBackendId);
4394 151670 : smgrwriteback(reln, tag.forkNum, tag.blockNum, nblocks);
4395 : }
4396 :
4397 15498 : context->nr_pending = 0;
4398 : }
4399 :
4400 :
4401 : /*
4402 : * Implement slower/larger portions of TestForOldSnapshot
4403 : *
4404 : * Smaller/faster portions are put inline, but the entire set of logic is too
4405 : * big for that.
4406 : */
4407 : void
4408 6330 : TestForOldSnapshot_impl(Snapshot snapshot, Relation relation)
4409 : {
4410 6330 : if (RelationAllowsEarlyPruning(relation)
4411 6310 : && (snapshot)->whenTaken < GetOldSnapshotThresholdTimestamp())
4412 16 : ereport(ERROR,
4413 : (errcode(ERRCODE_SNAPSHOT_TOO_OLD),
4414 : errmsg("snapshot too old")));
4415 6314 : }
|
