Age Owner Branch data TLA Line data Source code
1 : : /*-------------------------------------------------------------------------
2 : : *
3 : : * shmem.c
4 : : * create shared memory and initialize shared memory data structures.
5 : : *
6 : : * Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
7 : : * Portions Copyright (c) 1994, Regents of the University of California
8 : : *
9 : : *
10 : : * IDENTIFICATION
11 : : * src/backend/storage/ipc/shmem.c
12 : : *
13 : : *-------------------------------------------------------------------------
14 : : */
15 : : /*
16 : : * POSTGRES processes share one or more regions of shared memory.
17 : : * The shared memory is created by a postmaster and is inherited
18 : : * by each backend via fork() (or, in some ports, via other OS-specific
19 : : * methods). The routines in this file are used for allocating and
20 : : * binding to shared memory data structures.
21 : : *
22 : : * This module provides facilities to allocate fixed-size structures in shared
23 : : * memory, for things like variables shared between all backend processes.
24 : : * Each such structure has a string name to identify it, specified when it is
25 : : * requested. shmem_hash.c provides a shared hash table implementation on top
26 : : * of that.
27 : : *
28 : : * Shared memory areas should usually not be allocated after postmaster
29 : : * startup, although we do allow small allocations later for the benefit of
30 : : * extension modules that are loaded after startup. Despite that allowance,
31 : : * extensions that need shared memory should be added in
32 : : * shared_preload_libraries, because the allowance is quite small and there is
33 : : * no guarantee that any memory is available after startup.
34 : : *
35 : : * Nowadays, there is also another way to allocate shared memory called
36 : : * Dynamic Shared Memory. See dsm.c for that facility. One big difference
37 : : * between traditional shared memory handled by shmem.c and dynamic shared
38 : : * memory is that traditional shared memory areas are mapped to the same
39 : : * address in all processes, so you can use normal pointers in shared memory
40 : : * structs. With Dynamic Shared Memory, you must use offsets or DSA pointers
41 : : * instead.
42 : : *
43 : : * Shared memory managed by shmem.c can never be freed, once allocated. Each
44 : : * hash table has its own free list, so hash buckets can be reused when an
45 : : * item is deleted.
46 : : *
47 : : * Usage
48 : : * -----
49 : : *
50 : : * To allocate shared memory, you need to register a set of callback functions
51 : : * which handle the lifecycle of the allocation. In the request_fn callback,
52 : : * call ShmemRequestStruct() with the desired name and size. When the area is
53 : : * later allocated or attached to, the global variable pointed to by the .ptr
54 : : * option is set to the shared memory location of the allocation. The init_fn
55 : : * callback can perform additional initialization.
56 : : *
57 : : * typedef struct MyShmemData {
58 : : * ...
59 : : * } MyShmemData;
60 : : *
61 : : * static MyShmemData *MyShmem;
62 : : *
63 : : * static void my_shmem_request(void *arg);
64 : : * static void my_shmem_init(void *arg);
65 : : *
66 : : * const ShmemCallbacks MyShmemCallbacks = {
67 : : * .request_fn = my_shmem_request,
68 : : * .init_fn = my_shmem_init,
69 : : * };
70 : : *
71 : : * static void
72 : : * my_shmem_request(void *arg)
73 : : * {
74 : : * ShmemRequestStruct(.name = "My shmem area",
75 : : * .size = sizeof(MyShmemData),
76 : : * .ptr = (void **) &MyShmem,
77 : : * );
78 : : * }
79 : : *
80 : : * In builtin PostgreSQL code, add the callbacks to the list in
81 : : * src/include/storage/subsystemlist.h. In an add-in module, you can register
82 : : * the callbacks by calling RegisterShmemCallbacks(&MyShmemCallbacks) in the
83 : : * extension's _PG_init() function.
84 : : *
85 : : * Lifecycle
86 : : * ---------
87 : : *
88 : : * Initializing shared memory happens in multiple phases. In the first phase,
89 : : * during postmaster startup, all the request_fn callbacks are called. Only
90 : : * after all the request_fn callbacks have been called and all the shmem areas
91 : : * have been requested by the ShmemRequestStruct() calls we know how much
92 : : * shared memory we need in total. After that, postmaster allocates global
93 : : * shared memory segment, and calls all the init_fn callbacks to initialize
94 : : * all the requested shmem areas.
95 : : *
96 : : * In standard Unix-ish environments, individual backends do not need to
97 : : * re-establish their local pointers into shared memory, because they inherit
98 : : * correct values of those variables via fork() from the postmaster. However,
99 : : * this does not work in the EXEC_BACKEND case. In ports using EXEC_BACKEND,
100 : : * backend startup also calls the shmem_request callbacks to re-establish the
101 : : * knowledge about each shared memory area, sets the pointer variables
102 : : * (*options->ptr), and calls the attach_fn callback, if any, for additional
103 : : * per-backend setup.
104 : : *
105 : : * Legacy ShmemInitStruct()/ShmemInitHash() functions
106 : : * --------------------------------------------------
107 : : *
108 : : * ShmemInitStruct()/ShmemInitHash() is another way of registering shmem
109 : : * areas. It pre-dates the ShmemRequestStruct()/ShmemRequestHash() functions,
110 : : * and should not be used in new code, but as of this writing it is still
111 : : * widely used in extensions.
112 : : *
113 : : * To allocate a shmem area with ShmemInitStruct(), you need to separately
114 : : * register the size needed for the area by calling RequestAddinShmemSpace()
115 : : * from the extension's shmem_request_hook, and allocate the area by calling
116 : : * ShmemInitStruct() from the extension's shmem_startup_hook. There are no
117 : : * init/attach callbacks. Instead, the caller of ShmemInitStruct() must check
118 : : * the return status of ShmemInitStruct() and initialize the struct if it was
119 : : * not previously initialized.
120 : : *
121 : : * Calling ShmemAlloc() directly
122 : : * -----------------------------
123 : : *
124 : : * There's a more low-level way of allocating shared memory too: you can call
125 : : * ShmemAlloc() directly. It's used to implement the higher level mechanisms,
126 : : * and should generally not be called directly.
127 : : */
128 : :
129 : : #include "postgres.h"
130 : :
131 : : #include <unistd.h>
132 : :
133 : : #include "access/slru.h"
134 : : #include "common/int.h"
135 : : #include "fmgr.h"
136 : : #include "funcapi.h"
137 : : #include "miscadmin.h"
138 : : #include "port/pg_bitutils.h"
139 : : #include "port/pg_numa.h"
140 : : #include "storage/lwlock.h"
141 : : #include "storage/pg_shmem.h"
142 : : #include "storage/shmem.h"
143 : : #include "storage/shmem_internal.h"
144 : : #include "storage/spin.h"
145 : : #include "utils/builtins.h"
146 : : #include "utils/tuplestore.h"
147 : :
148 : : typedef struct ShmemIndexEnt ShmemIndexEnt;
149 : :
150 : : /*
151 : : * Registered callbacks.
152 : : *
153 : : * During postmaster startup, we accumulate the callbacks from all subsystems
154 : : * in this list.
155 : : *
156 : : * This is in process private memory, although on Unix-like systems, we expect
157 : : * all the registrations to happen at postmaster startup time and be inherited
158 : : * by all the child processes via fork().
159 : : */
160 : : static List *registered_shmem_callbacks;
161 : :
162 : : /*
163 : : * In the shmem request phase, all the shmem areas requested with the
164 : : * ShmemRequest*() functions are accumulated in the 'pending_shmem_requests'
165 : : * list. The List, the ShmemRequest structs, and the 'options' are all
166 : : * allocated in TopMemoryContext.
167 : : */
168 : : typedef struct
169 : : {
170 : : ShmemStructOpts *options;
171 : : ShmemRequestKind kind;
172 : :
173 : : /* InitShmemIndexEntry() sets this pointer when the area is allocated */
174 : : ShmemIndexEnt *index_entry;
175 : : } ShmemRequest;
176 : :
177 : : static List *pending_shmem_requests; /* List of ShmemRequests */
178 : :
179 : : /*
180 : : * Per-process state machine, for sanity checking that we do things in the
181 : : * right order.
182 : : *
183 : : * Postmaster:
184 : : * INITIAL -> REQUESTING -> INITIALIZING -> DONE
185 : : *
186 : : * Backends in EXEC_BACKEND mode:
187 : : * INITIAL -> REQUESTING -> ATTACHING -> DONE
188 : : *
189 : : * Late request:
190 : : * DONE -> REQUESTING -> AFTER_STARTUP_ATTACH_OR_INIT -> DONE
191 : : */
192 : : enum shmem_request_state
193 : : {
194 : : /* Initial state */
195 : : SRS_INITIAL,
196 : :
197 : : /*
198 : : * When we start calling the shmem_request callbacks, we enter the
199 : : * SRS_REQUESTING phase. All ShmemRequestStruct calls happen in this
200 : : * state.
201 : : */
202 : : SRS_REQUESTING,
203 : :
204 : : /*
205 : : * Postmaster has finished all shmem requests, and is now initializing the
206 : : * shared memory segment. init_fn callbacks are called in this state.
207 : : */
208 : : SRS_INITIALIZING,
209 : :
210 : : /*
211 : : * A postmaster child process is starting up. attach_fn callbacks are
212 : : * called in this state.
213 : : */
214 : : SRS_ATTACHING,
215 : :
216 : : /* An after-startup allocation or attachment is in progress */
217 : : SRS_AFTER_STARTUP_ATTACH_OR_INIT,
218 : :
219 : : /* Normal state after shmem initialization / attachment */
220 : : SRS_DONE,
221 : : };
222 : : static enum shmem_request_state shmem_request_state = SRS_INITIAL;
223 : :
224 : : /*
225 : : * This is the first data structure stored in the shared memory segment, at
226 : : * the offset that PGShmemHeader->content_offset points to. Allocations by
227 : : * ShmemAlloc() are carved out of the space after this.
228 : : *
229 : : * For the base pointer and the total size of the shmem segment, we rely on
230 : : * the PGShmemHeader.
231 : : */
232 : : typedef struct ShmemAllocatorData
233 : : {
234 : : Size free_offset; /* offset to first free space from ShmemBase */
235 : :
236 : : /* protects 'free_offset' */
237 : : slock_t shmem_lock;
238 : :
239 : : HASHHDR *index; /* location of ShmemIndex */
240 : : size_t index_size; /* size of shmem region holding ShmemIndex */
241 : : LWLock index_lock; /* protects ShmemIndex */
242 : : } ShmemAllocatorData;
243 : :
244 : : #define ShmemIndexLock (&ShmemAllocator->index_lock)
245 : :
246 : : static void *ShmemAllocRaw(Size size, Size alignment, Size *allocated_size);
247 : :
248 : : /* shared memory global variables */
249 : :
250 : : static PGShmemHeader *ShmemSegHdr; /* shared mem segment header */
251 : : static void *ShmemBase; /* start address of shared memory */
252 : : static void *ShmemEnd; /* end+1 address of shared memory */
253 : :
254 : : static ShmemAllocatorData *ShmemAllocator;
255 : :
256 : : /*
257 : : * ShmemIndex is a global directory of shmem areas, itself also stored in the
258 : : * shared memory.
259 : : */
260 : : static HTAB *ShmemIndex;
261 : :
262 : : /* max size of data structure string name */
263 : : #define SHMEM_INDEX_KEYSIZE (48)
264 : :
265 : : /*
266 : : * # of additional entries to reserve in the shmem index table, for
267 : : * allocations after postmaster startup. (This is not a hard limit, the hash
268 : : * table can grow larger than that if there is shared memory available)
269 : : */
270 : : #define SHMEM_INDEX_ADDITIONAL_SIZE (128)
271 : :
272 : : /* this is a hash bucket in the shmem index table */
273 : : typedef struct ShmemIndexEnt
274 : : {
275 : : char key[SHMEM_INDEX_KEYSIZE]; /* string name */
276 : : void *location; /* location in shared mem */
277 : : Size size; /* # bytes requested for the structure */
278 : : Size allocated_size; /* # bytes actually allocated */
279 : : bool initialized; /* has the init callback been run? */
280 : : } ShmemIndexEnt;
281 : :
282 : : /* To get reliable results for NUMA inquiry we need to "touch pages" once */
283 : : static bool firstNumaTouch = true;
284 : :
285 : : static void CallShmemCallbacksAfterStartup(const ShmemCallbacks *callbacks);
286 : : static void ProcessShmemRequestsAfterStartup(const ShmemCallbacks *callbacks);
287 : : static void InitShmemIndexEntry(ShmemRequest *request);
288 : : static bool AttachShmemIndexEntry(ShmemRequest *request, bool missing_ok);
289 : :
290 : : Datum pg_numa_available(PG_FUNCTION_ARGS);
291 : :
292 : : /*
293 : : * ShmemRequestStruct() --- request a named shared memory area
294 : : *
295 : : * Subsystems call this to register their shared memory needs. This is
296 : : * usually done early in postmaster startup, before the shared memory segment
297 : : * has been created, so that the size can be included in the estimate for
298 : : * total amount of shared memory needed. We set aside a small amount of
299 : : * memory for allocations that happen later, for the benefit of non-preloaded
300 : : * extensions, but that should not be relied upon.
301 : : *
302 : : * This does not yet allocate the memory, but merely registers the need for
303 : : * it. The actual allocation happens later in the postmaster startup
304 : : * sequence.
305 : : *
306 : : * This must be called from a shmem_request callback function, registered with
307 : : * RegisterShmemCallbacks(). This enforces a coding pattern that works the
308 : : * same in normal Unix systems and with EXEC_BACKEND. On Unix systems, the
309 : : * shmem_request callbacks are called once, early in postmaster startup, and
310 : : * the child processes inherit the struct descriptors and any other
311 : : * per-process state from the postmaster. In EXEC_BACKEND mode, shmem_request
312 : : * callbacks are *also* called in each backend, at backend startup, to
313 : : * re-establish the struct descriptors. By calling the same function in both
314 : : * cases, we ensure that all the shmem areas are registered the same way in
315 : : * all processes.
316 : : *
317 : : * 'options' defines the name and size of the area, and any other optional
318 : : * features. Leave unused options as zeros. The options are copied to
319 : : * longer-lived memory, so it doesn't need to live after the
320 : : * ShmemRequestStruct() call and can point to a local variable in the calling
321 : : * function. The 'name' must point to a long-lived string though, only the
322 : : * pointer to it is copied.
323 : : */
324 : : void
167 heikki.linnakangas@i 325 :CBC 81732 : ShmemRequestStructWithOpts(const ShmemStructOpts *options)
326 : : {
327 : : ShmemStructOpts *options_copy;
328 : :
329 : 81732 : options_copy = MemoryContextAlloc(TopMemoryContext,
330 : : sizeof(ShmemStructOpts));
331 : 81732 : memcpy(options_copy, options, sizeof(ShmemStructOpts));
332 : :
333 : 81732 : ShmemRequestInternal(options_copy, SHMEM_KIND_STRUCT);
334 : 81732 : }
335 : :
336 : : /*
337 : : * Internal workhorse of ShmemRequestStruct() and ShmemRequestHash().
338 : : *
339 : : * Note: Unlike in the public ShmemRequestStruct() and ShmemRequestHash()
340 : : * functions, 'options' is *not* copied. It must be allocated in
341 : : * TopMemoryContext by the caller, and will be freed after the init/attach
342 : : * callbacks have been called. This allows ShmemRequestHash() to pass a
343 : : * pointer to the extended ShmemHashOpts struct instead.
344 : : */
345 : : void
346 : 100902 : ShmemRequestInternal(ShmemStructOpts *options, ShmemRequestKind kind)
347 : : {
348 : : MemoryContext oldcontext;
349 : : ShmemRequest *request;
350 : :
351 : : /* Check the options */
352 [ - + ]: 100902 : if (options->name == NULL)
167 heikki.linnakangas@i 353 [ # # ]:UBC 0 : elog(ERROR, "shared memory request is missing 'name' option");
354 : :
167 heikki.linnakangas@i 355 [ + + ]:CBC 100902 : if (IsUnderPostmaster)
356 : : {
357 [ + + - + ]: 8 : if (options->size <= 0 && options->size != SHMEM_ATTACH_UNKNOWN_SIZE)
167 heikki.linnakangas@i 358 [ # # ]:UBC 0 : elog(ERROR, "invalid size %zd for shared memory request for \"%s\"",
359 : : options->size, options->name);
360 : : }
361 : : else
362 : : {
167 heikki.linnakangas@i 363 [ - + ]:CBC 100894 : if (options->size == SHMEM_ATTACH_UNKNOWN_SIZE)
167 heikki.linnakangas@i 364 [ # # ]:UBC 0 : elog(ERROR, "SHMEM_ATTACH_UNKNOWN_SIZE cannot be used during startup");
167 heikki.linnakangas@i 365 [ - + ]:CBC 100894 : if (options->size <= 0)
167 heikki.linnakangas@i 366 [ # # ]:UBC 0 : elog(ERROR, "invalid size %zd for shared memory request for \"%s\"",
367 : : options->size, options->name);
368 : : }
369 : :
167 heikki.linnakangas@i 370 [ + + - + ]:CBC 100902 : if (options->alignment != 0 && pg_nextpower2_size_t(options->alignment) != options->alignment)
167 heikki.linnakangas@i 371 [ # # ]:UBC 0 : elog(ERROR, "invalid alignment %zu for shared memory request for \"%s\"",
372 : : options->alignment, options->name);
373 : :
374 : : /* Check that we're in the right state */
167 heikki.linnakangas@i 375 [ - + ]:CBC 100902 : if (shmem_request_state != SRS_REQUESTING)
167 heikki.linnakangas@i 376 [ # # ]:UBC 0 : elog(ERROR, "ShmemRequestStruct can only be called from a shmem_request callback");
377 : :
378 : : /* Check that it's not already registered in this process */
167 heikki.linnakangas@i 379 [ + + + + :CBC 4137078 : foreach_ptr(ShmemRequest, existing, pending_shmem_requests)
+ + ]
380 : : {
381 [ - + ]: 3935274 : if (strcmp(existing->options->name, options->name) == 0)
167 heikki.linnakangas@i 382 [ # # ]:UBC 0 : ereport(ERROR,
383 : : (errmsg("shared memory struct \"%s\" is already registered",
384 : : options->name)));
385 : : }
386 : :
387 : : /* Request looks valid, remember it */
23 heikki.linnakangas@i 388 :CBC 100902 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
34 michael@paquier.xyz 389 : 100902 : request = palloc_object(ShmemRequest);
167 heikki.linnakangas@i 390 : 100902 : request->options = options;
391 : 100902 : request->kind = kind;
23 392 : 100902 : request->index_entry = NULL;
167 393 : 100902 : pending_shmem_requests = lappend(pending_shmem_requests, request);
23 394 : 100902 : MemoryContextSwitchTo(oldcontext);
167 395 : 100902 : }
396 : :
397 : : /*
398 : : * ShmemGetRequestedSize() --- estimate the total size of all registered shared
399 : : * memory structures.
400 : : *
401 : : * This is called at postmaster startup, before the shared memory segment has
402 : : * been created.
403 : : */
404 : : size_t
405 : 2374 : ShmemGetRequestedSize(void)
406 : : {
407 : : size_t size;
408 : :
409 : : /* memory needed for the ShmemIndex */
410 : 2374 : size = hash_estimate_size(list_length(pending_shmem_requests) + SHMEM_INDEX_ADDITIONAL_SIZE,
411 : : sizeof(ShmemIndexEnt));
412 : 2374 : size = CACHELINEALIGN(size);
413 : :
414 : : /* memory needed for all the requested areas */
415 [ + - + + : 192316 : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
+ + ]
416 : : {
417 : 187568 : size_t alignment = request->options->alignment;
418 : :
419 : : /* pad the start address for alignment like ShmemAllocRaw() does */
420 [ + + ]: 187568 : if (alignment < PG_CACHE_LINE_SIZE)
421 : 180446 : alignment = PG_CACHE_LINE_SIZE;
422 : 187568 : size = TYPEALIGN(alignment, size);
423 : :
424 : 187568 : size = add_size(size, request->options->size);
425 : : }
426 : :
427 : 2374 : return size;
428 : : }
429 : :
430 : : /*
431 : : * ShmemInitRequested() --- allocate and initialize requested shared memory
432 : : * structures.
433 : : *
434 : : * This is called once at postmaster startup, after the shared memory segment
435 : : * has been created.
436 : : */
437 : : void
438 : 1274 : ShmemInitRequested(void)
439 : : {
440 : : /* should be called only by the postmaster or a standalone backend */
441 [ - + ]: 1274 : Assert(!IsUnderPostmaster);
442 [ - + ]: 1274 : Assert(shmem_request_state == SRS_INITIALIZING);
443 : :
444 : : /*
445 : : * Initialize the ShmemIndex entries and perform basic initialization of
446 : : * all the requested memory areas. There are no concurrent processes yet,
447 : : * so no need for locking.
448 : : */
449 [ + - + + : 103204 : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
+ + ]
450 : : {
451 : 100656 : InitShmemIndexEntry(request);
452 : : }
453 : :
454 : : /*
455 : : * Call the subsystem-specific init callbacks to finish initialization of
456 : : * all the areas.
457 : : */
458 [ + - + + : 58630 : foreach_ptr(const ShmemCallbacks, callbacks, registered_shmem_callbacks)
+ + ]
459 : : {
460 [ + + ]: 56082 : if (callbacks->init_fn)
461 : 50982 : callbacks->init_fn(callbacks->opaque_arg);
462 : : }
463 : :
464 : : /* Now we can mark all the areas as initialized and free the requests */
23 465 [ + - + + : 103204 : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
+ + ]
466 : : {
467 : 100656 : request->index_entry->initialized = true;
468 : 100656 : pfree(request->options);
469 : : }
470 : 1274 : list_free_deep(pending_shmem_requests);
471 : 1274 : pending_shmem_requests = NIL;
472 : :
167 473 : 1274 : shmem_request_state = SRS_DONE;
474 : 1274 : }
475 : :
476 : : /*
477 : : * Re-establish process private state related to shmem areas.
478 : : *
479 : : * This is called at backend startup in EXEC_BACKEND mode, in every backend.
480 : : */
481 : : #ifdef EXEC_BACKEND
482 : : void
483 : : ShmemAttachRequested(void)
484 : : {
485 : : ListCell *lc;
486 : :
487 : : /* Must be initializing a (non-standalone) backend */
488 : : Assert(IsUnderPostmaster);
489 : : Assert(ShmemAllocator->index != NULL);
490 : : Assert(shmem_request_state == SRS_REQUESTING);
491 : : shmem_request_state = SRS_ATTACHING;
492 : :
493 : : LWLockAcquire(ShmemIndexLock, LW_SHARED);
494 : :
495 : : /*
496 : : * Attach to all the requested memory areas.
497 : : */
498 : : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
499 : : {
500 : : AttachShmemIndexEntry(request, false);
501 : : pfree(request->options);
502 : : }
503 : : list_free_deep(pending_shmem_requests);
504 : : pending_shmem_requests = NIL;
505 : :
506 : : /* Call attach callbacks */
507 : : foreach(lc, registered_shmem_callbacks)
508 : : {
509 : : const ShmemCallbacks *callbacks = (const ShmemCallbacks *) lfirst(lc);
510 : :
511 : : if (callbacks->attach_fn)
512 : : callbacks->attach_fn(callbacks->opaque_arg);
513 : : }
514 : :
515 : : LWLockRelease(ShmemIndexLock);
516 : :
517 : : shmem_request_state = SRS_DONE;
518 : : }
519 : : #endif
520 : :
521 : : /*
522 : : * Insert requested shmem area into the shared memory index and initialize it.
523 : : *
524 : : * Note that this only does performs basic initialization depending on
525 : : * ShmemRequestKind, like setting the global pointer variable to the area for
526 : : * SHMEM_KIND_STRUCT or setting up the backend-private HTAB control struct.
527 : : * This does *not* call the subsystem-specific init callbacks. That's done
528 : : * later after all the shmem areas have been initialized or attached to.
529 : : */
530 : : static void
531 : 100662 : InitShmemIndexEntry(ShmemRequest *request)
532 : : {
533 : 100662 : const char *name = request->options->name;
534 : : ShmemIndexEnt *index_entry;
535 : : bool found;
536 : : size_t allocated_size;
537 : : void *structPtr;
538 : :
539 : : /* Size must be known at this point. */
2 540 [ - + ]: 100662 : Assert(request->options->size != SHMEM_ATTACH_UNKNOWN_SIZE);
541 : :
542 : : /* look it up in the shmem index */
543 : : index_entry = (ShmemIndexEnt *)
167 544 : 100662 : hash_search(ShmemIndex, name, HASH_ENTER_NULL, &found);
545 [ - + ]: 100662 : if (found)
167 heikki.linnakangas@i 546 [ # # ]:UBC 0 : elog(ERROR, "shared memory struct \"%s\" is already initialized", name);
167 heikki.linnakangas@i 547 [ - + ]:CBC 100662 : if (!index_entry)
548 : : {
549 : : /* tried to add it to the hash table, but there was no space */
167 heikki.linnakangas@i 550 [ # # ]:UBC 0 : ereport(ERROR,
551 : : (errcode(ERRCODE_OUT_OF_MEMORY),
552 : : errmsg("could not create ShmemIndex entry for data structure \"%s\"",
553 : : name)));
554 : : }
555 : :
556 : : /*
557 : : * We inserted the entry to the shared memory index. Allocate requested
558 : : * amount of shared memory for it, and initialize the index entry.
559 : : */
167 heikki.linnakangas@i 560 :CBC 100662 : structPtr = ShmemAllocRaw(request->options->size,
561 : 100662 : request->options->alignment,
562 : : &allocated_size);
563 [ + + ]: 100662 : if (structPtr == NULL)
564 : : {
565 : : /* out of memory; remove the failed ShmemIndex entry */
566 : 1 : hash_search(ShmemIndex, name, HASH_REMOVE, NULL);
567 [ + - ]: 1 : ereport(ERROR,
568 : : (errcode(ERRCODE_OUT_OF_MEMORY),
569 : : errmsg("not enough shared memory for data structure"
570 : : " \"%s\" (%zd bytes requested)",
571 : : name, request->options->size)));
572 : : }
573 : 100661 : index_entry->size = request->options->size;
574 : 100661 : index_entry->allocated_size = allocated_size;
575 : 100661 : index_entry->location = structPtr;
576 : :
577 : : /*
578 : : * The area is considered fully initialized only after the subsystem's
579 : : * init callback has been called. For now, perform only basic
580 : : * initialization based on the kind of shmem area it is.
581 : : */
23 582 : 100661 : index_entry->initialized = false;
167 583 [ + + + - ]: 100661 : switch (request->kind)
584 : : {
585 : 81536 : case SHMEM_KIND_STRUCT:
586 [ + - ]: 81536 : if (request->options->ptr)
587 : 81536 : *(request->options->ptr) = index_entry->location;
588 : 81536 : break;
589 : 10203 : case SHMEM_KIND_HASH:
590 : 10203 : shmem_hash_init(structPtr, request->options);
591 : 10203 : break;
592 : 8922 : case SHMEM_KIND_SLRU:
593 : 8922 : shmem_slru_init(structPtr, request->options);
594 : 8922 : break;
595 : : }
596 : :
597 : : /* return the pointer to the entry to the caller */
23 598 : 100661 : request->index_entry = index_entry;
167 599 : 100661 : }
600 : :
601 : : /*
602 : : * Look up a named shmem area in the shared memory index and attach to it.
603 : : *
604 : : * Note that this only performs the basic attachment actions depending on
605 : : * ShmemRequestKind, like setting the global pointer variable to the area for
606 : : * SHMEM_KIND_STRUCT or setting up the backend-private HTAB control struct.
607 : : * This does *not* call the subsystem-specific attach callbacks. That's done
608 : : * later after all the shmem areas have been initialized or attached to.
609 : : */
610 : : static bool
611 : 1 : AttachShmemIndexEntry(ShmemRequest *request, bool missing_ok)
612 : : {
613 : 1 : const char *name = request->options->name;
614 : : ShmemIndexEnt *index_entry;
615 : :
616 : : /* Look it up in the shmem index */
617 : : index_entry = (ShmemIndexEnt *)
618 : 1 : hash_search(ShmemIndex, name, HASH_FIND, NULL);
619 [ - + ]: 1 : if (!index_entry)
620 : : {
167 heikki.linnakangas@i 621 [ # # ]:UBC 0 : if (!missing_ok)
622 [ # # ]: 0 : ereport(ERROR,
623 : : (errmsg("could not find ShmemIndex entry for data structure \"%s\"",
624 : : request->options->name)));
625 : 0 : return false;
626 : : }
627 : :
628 : : /*
629 : : * If it was previously allocated but not fully initialized, error out.
630 : : * There is currently no way of retrying or cleaning up an uninitialized
631 : : * entry, it just lingers until the server is shut down. But this can
632 : : * only happen when allocating areas after postmaster startup, and it's
633 : : * unlikely that you could successfully retry anyway. The most likely
634 : : * reason for failed initialization is that you are out of shared memory
635 : : * and retrying won't help with that.
636 : : */
23 heikki.linnakangas@i 637 [ - + ]:CBC 1 : if (!index_entry->initialized)
23 heikki.linnakangas@i 638 [ # # ]:UBC 0 : ereport(ERROR,
639 : : (errmsg("cannot attach to shared memory struct \"%s\" because it was not fully initialized",
640 : : request->options->name)));
641 : :
642 : : /* Check that the size in the index matches the request */
167 heikki.linnakangas@i 643 [ - + ]:CBC 1 : if (index_entry->size != request->options->size &&
167 heikki.linnakangas@i 644 [ # # ]:UBC 0 : request->options->size != SHMEM_ATTACH_UNKNOWN_SIZE)
645 : : {
646 [ # # ]: 0 : ereport(ERROR,
647 : : (errmsg("shared memory struct \"%s\" was created with"
648 : : " different size: existing %zu, requested %zd",
649 : : name, index_entry->size, request->options->size)));
650 : : }
651 : :
652 : : /*
653 : : * Re-establish the caller's pointer variable, or do other actions to
654 : : * attach depending on the kind of shmem area it is.
655 : : */
167 heikki.linnakangas@i 656 [ + - - - ]:CBC 1 : switch (request->kind)
657 : : {
658 : 1 : case SHMEM_KIND_STRUCT:
659 [ + - ]: 1 : if (request->options->ptr)
660 : 1 : *(request->options->ptr) = index_entry->location;
661 : 1 : break;
167 heikki.linnakangas@i 662 :UBC 0 : case SHMEM_KIND_HASH:
663 : 0 : shmem_hash_attach(index_entry->location, request->options);
664 : 0 : break;
665 : 0 : case SHMEM_KIND_SLRU:
666 : 0 : shmem_slru_attach(index_entry->location, request->options);
667 : 0 : break;
668 : : }
669 : :
23 heikki.linnakangas@i 670 :CBC 1 : request->index_entry = index_entry;
671 : :
167 672 : 1 : return true;
673 : : }
674 : :
675 : : /*
676 : : * InitShmemAllocator() --- set up basic pointers to shared memory.
677 : : *
678 : : * Called at postmaster or stand-alone backend startup, to initialize the
679 : : * allocator's data structure in the shared memory segment. In EXEC_BACKEND,
680 : : * this is also called at backend startup, to set up pointers to the
681 : : * already-initialized data structure.
682 : : */
683 : : void
233 684 : 1274 : InitShmemAllocator(PGShmemHeader *seghdr)
685 : : {
686 : : Size offset;
687 : : int64 hash_nelems;
688 : : HASHCTL info;
689 : : int hash_flags;
690 : :
691 : : #ifndef EXEC_BACKEND
178 692 [ - + ]: 1274 : Assert(!IsUnderPostmaster);
693 : : #endif
233 694 [ - + ]: 1274 : Assert(seghdr != NULL);
695 : :
167 696 [ - + ]: 1274 : if (IsUnderPostmaster)
697 : : {
167 heikki.linnakangas@i 698 [ # # ]:UBC 0 : Assert(shmem_request_state == SRS_INITIAL);
699 : : }
700 : : else
701 : : {
167 heikki.linnakangas@i 702 [ - + ]:CBC 1274 : Assert(shmem_request_state == SRS_REQUESTING);
703 : 1274 : shmem_request_state = SRS_INITIALIZING;
704 : : }
705 : :
706 : : /*
707 : : * We assume the pointer and offset are MAXALIGN. Not a hard requirement,
708 : : * but it's true today and keeps the math below simpler.
709 : : */
233 710 [ - + ]: 1274 : Assert(seghdr == (void *) MAXALIGN(seghdr));
711 [ - + ]: 1274 : Assert(seghdr->content_offset == MAXALIGN(seghdr->content_offset));
712 : :
713 : : /*
714 : : * Allocations after this point should go through ShmemAlloc, which
715 : : * expects to allocate everything on cache line boundaries. Make sure the
716 : : * first allocation begins on a cache line boundary.
717 : : */
178 718 : 1274 : offset = CACHELINEALIGN(seghdr->content_offset + sizeof(ShmemAllocatorData));
719 [ - + ]: 1274 : if (offset > seghdr->totalsize)
178 heikki.linnakangas@i 720 [ # # ]:UBC 0 : ereport(ERROR,
721 : : (errcode(ERRCODE_OUT_OF_MEMORY),
722 : : errmsg("out of shared memory (%zu bytes requested)",
723 : : offset)));
724 : :
725 : : /*
726 : : * In postmaster or stand-alone backend, initialize the shared memory
727 : : * allocator so that we can allocate shared memory for ShmemIndex using
728 : : * ShmemAlloc(). In a regular backend just set up the pointers required
729 : : * by ShmemAlloc().
730 : : */
178 heikki.linnakangas@i 731 :CBC 1274 : ShmemAllocator = (ShmemAllocatorData *) ((char *) seghdr + seghdr->content_offset);
732 [ + - ]: 1274 : if (!IsUnderPostmaster)
733 : : {
734 : 1274 : SpinLockInit(&ShmemAllocator->shmem_lock);
735 : 1274 : ShmemAllocator->free_offset = offset;
736 : 1274 : LWLockInitialize(&ShmemAllocator->index_lock, LWTRANCHE_SHMEM_INDEX);
737 : : }
738 : :
663 peter@eisentraut.org 739 : 1274 : ShmemSegHdr = seghdr;
740 : 1274 : ShmemBase = seghdr;
741 : 1274 : ShmemEnd = (char *) ShmemBase + seghdr->totalsize;
742 : :
743 : : /*
744 : : * Create (or attach to) the shared memory index of shmem areas.
745 : : *
746 : : * This is the same initialization as ShmemInitHash() does, but we cannot
747 : : * use ShmemInitHash() here because it relies on ShmemIndex being already
748 : : * initialized.
749 : : */
167 heikki.linnakangas@i 750 : 1274 : hash_nelems = list_length(pending_shmem_requests) + SHMEM_INDEX_ADDITIONAL_SIZE;
751 : :
178 752 : 1274 : info.keysize = SHMEM_INDEX_KEYSIZE;
753 : 1274 : info.entrysize = sizeof(ShmemIndexEnt);
169 754 : 1274 : hash_flags = HASH_ELEM | HASH_STRINGS | HASH_FIXED_SIZE;
755 : :
178 756 [ + - ]: 1274 : if (!IsUnderPostmaster)
757 : : {
167 758 : 1274 : ShmemAllocator->index_size = hash_estimate_size(hash_nelems, info.entrysize);
169 759 : 1274 : ShmemAllocator->index = (HASHHDR *) ShmemAlloc(ShmemAllocator->index_size);
760 : : }
761 : 2548 : ShmemIndex = shmem_hash_create(ShmemAllocator->index,
762 : 1274 : ShmemAllocator->index_size,
763 : : IsUnderPostmaster,
764 : : "ShmemIndex", hash_nelems,
765 : : &info, hash_flags);
178 766 [ - + ]: 1274 : Assert(ShmemIndex != NULL);
767 : :
768 : : /*
769 : : * Add an entry for ShmemIndex itself into ShmemIndex, so that it's
770 : : * visible in the pg_shmem_allocations view
771 : : */
172 772 [ + - ]: 1274 : if (!IsUnderPostmaster)
773 : : {
774 : : bool found;
775 : : ShmemIndexEnt *result = (ShmemIndexEnt *)
776 : 1274 : hash_search(ShmemIndex, "ShmemIndex", HASH_ENTER, &found);
777 : :
778 [ - + ]: 1274 : Assert(!found);
169 779 : 1274 : result->size = ShmemAllocator->index_size;
780 : 1274 : result->allocated_size = ShmemAllocator->index_size;
172 781 : 1274 : result->location = ShmemAllocator->index;
23 782 : 1274 : result->initialized = true;
783 : : }
11030 scrappy@hub.org 784 : 1274 : }
785 : :
786 : : /*
787 : : * Reset state on postmaster crash restart.
788 : : */
789 : : void
167 heikki.linnakangas@i 790 : 5 : ResetShmemAllocator(void)
791 : : {
792 [ - + ]: 5 : Assert(!IsUnderPostmaster);
793 : 5 : shmem_request_state = SRS_INITIAL;
794 : :
795 : 5 : pending_shmem_requests = NIL;
796 : :
797 : : /*
798 : : * Note that we don't clear the registered callbacks. We will need to
799 : : * call them again as we restart
800 : : */
801 : 5 : }
802 : :
803 : : /*
804 : : * ShmemAlloc -- allocate max-aligned chunk from shared memory
805 : : *
806 : : * Throws error if request cannot be satisfied.
807 : : *
808 : : * Assumes ShmemSegHdr is initialized.
809 : : */
810 : : void *
7280 tgl@sss.pgh.pa.us 811 : 1274 : ShmemAlloc(Size size)
812 : : {
813 : : void *newSpace;
814 : : Size allocated_size;
815 : :
167 heikki.linnakangas@i 816 : 1274 : newSpace = ShmemAllocRaw(size, 0, &allocated_size);
3671 tgl@sss.pgh.pa.us 817 [ - + ]: 1274 : if (!newSpace)
3671 tgl@sss.pgh.pa.us 818 [ # # ]:UBC 0 : ereport(ERROR,
819 : : (errcode(ERRCODE_OUT_OF_MEMORY),
820 : : errmsg("out of shared memory (%zu bytes requested)",
821 : : size)));
3671 tgl@sss.pgh.pa.us 822 :CBC 1274 : return newSpace;
823 : : }
824 : :
825 : : /*
826 : : * ShmemAllocNoError -- allocate max-aligned chunk from shared memory
827 : : *
828 : : * As ShmemAlloc, but returns NULL if out of space, rather than erroring.
829 : : */
830 : : void *
3671 tgl@sss.pgh.pa.us 831 :UBC 0 : ShmemAllocNoError(Size size)
832 : : {
833 : : Size allocated_size;
834 : :
167 heikki.linnakangas@i 835 : 0 : return ShmemAllocRaw(size, 0, &allocated_size);
836 : : }
837 : :
838 : : /*
839 : : * ShmemAllocRaw -- allocate align chunk and return allocated size
840 : : *
841 : : * Also sets *allocated_size to the number of bytes allocated, which will
842 : : * be equal to the number requested plus any padding we choose to add.
843 : : *
844 : : * Returns NULL in case space can not be allocated.
845 : : */
846 : : static void *
167 heikki.linnakangas@i 847 :CBC 101936 : ShmemAllocRaw(Size size, Size alignment, Size *allocated_size)
848 : : {
849 : : Size rawStart;
850 : : Size newStart;
851 : : Size newFree;
852 : : void *newSpace;
853 : :
854 : : /*
855 : : * Ensure all space is adequately aligned. We used to only MAXALIGN this
856 : : * space but experience has proved that on modern systems that is not good
857 : : * enough. Many parts of the system are very sensitive to critical data
858 : : * structures getting split across cache line boundaries. To avoid that,
859 : : * attempt to align the beginning of the allocation to a cache line
860 : : * boundary. The calling code will still need to be careful about how it
861 : : * uses the allocated space - e.g. by padding each element in an array of
862 : : * structures out to a power-of-two size - but without this, even that
863 : : * won't be sufficient.
864 : : */
865 [ + + ]: 101936 : if (alignment < PG_CACHE_LINE_SIZE)
866 : 98114 : alignment = PG_CACHE_LINE_SIZE;
867 : :
3992 rhaas@postgresql.org 868 [ - + ]: 101936 : Assert(ShmemSegHdr != NULL);
869 : :
178 heikki.linnakangas@i 870 : 101936 : SpinLockAcquire(&ShmemAllocator->shmem_lock);
871 : :
167 872 : 101936 : rawStart = ShmemAllocator->free_offset;
873 : 101936 : newStart = TYPEALIGN(alignment, rawStart);
874 : :
875 : : /*
876 : : * newFree = newStart + size, which is the start of the remaining space
877 : : * after the allocation. If it exceeds the shmem segment size, we don't
878 : : * have enough space available.
879 : : */
2 880 [ + - ]: 101936 : if (!pg_add_size_overflow(newStart, size, &newFree) &&
881 [ + + ]: 101936 : newFree <= ShmemSegHdr->totalsize)
882 : : {
661 peter@eisentraut.org 883 : 101935 : newSpace = (char *) ShmemBase + newStart;
233 heikki.linnakangas@i 884 : 101935 : ShmemAllocator->free_offset = newFree;
885 : : }
886 : : else
10605 bruce@momjian.us 887 : 1 : newSpace = NULL;
888 : :
178 heikki.linnakangas@i 889 : 101936 : SpinLockRelease(&ShmemAllocator->shmem_lock);
890 : :
891 : : /* note this assert is okay with newSpace == NULL */
167 892 [ - + ]: 101936 : Assert(newSpace == (void *) TYPEALIGN(alignment, newSpace));
893 : :
894 : 101936 : *allocated_size = newFree - rawStart;
10246 bruce@momjian.us 895 : 101936 : return newSpace;
896 : : }
897 : :
898 : : /*
899 : : * ShmemAddrIsValid -- test if an address refers to shared memory
900 : : *
901 : : * Returns true if the pointer points within the shared memory segment.
902 : : */
903 : : bool
5704 heikki.linnakangas@i 904 : 1704 : ShmemAddrIsValid(const void *addr)
905 : : {
6531 tgl@sss.pgh.pa.us 906 [ + - + - ]: 1704 : return (addr >= ShmemBase) && (addr < ShmemEnd);
907 : : }
908 : :
909 : : /*
910 : : * Register callbacks that define a shared memory area (or multiple areas).
911 : : *
912 : : * The system will call the callbacks at different stages of postmaster or
913 : : * backend startup, to allocate and initialize the area.
914 : : *
915 : : * This is normally called early during postmaster startup, but if the
916 : : * SHMEM_CALLBACKS_ALLOW_AFTER_STARTUP is set, this can also be used after
917 : : * startup, although after startup there's no guarantee that there's enough
918 : : * shared memory available. When called after startup, this immediately calls
919 : : * the right callbacks depending on whether another backend had already
920 : : * initialized the area.
921 : : *
922 : : * Note: In EXEC_BACKEND mode, this needs to be called in every backend
923 : : * process. That's needed because we cannot pass down the callback function
924 : : * pointers from the postmaster process, because different processes may have
925 : : * loaded libraries to different addresses.
926 : : */
927 : : void
167 heikki.linnakangas@i 928 : 56573 : RegisterShmemCallbacks(const ShmemCallbacks *callbacks)
929 : : {
25 930 [ + + ]: 56573 : if (shmem_request_state == SRS_DONE)
931 : : {
932 : : /*
933 : : * After-startup initialization or attachment. Call the appropriate
934 : : * callbacks immediately.
935 : : *
936 : : * This is not allowed from the postmaster, because the postmaster
937 : : * cannot acquire locks.
938 : : */
167 939 [ - + ]: 9 : if ((callbacks->flags & SHMEM_CALLBACKS_ALLOW_AFTER_STARTUP) == 0)
167 heikki.linnakangas@i 940 [ # # ]:UBC 0 : elog(ERROR, "cannot request shared memory at this time");
25 heikki.linnakangas@i 941 [ + + - + ]:CBC 9 : Assert(IsUnderPostmaster || !IsPostmasterEnvironment);
942 : :
167 943 : 9 : CallShmemCallbacksAfterStartup(callbacks);
944 : : }
25 945 [ + - ]: 56564 : else if (shmem_request_state == SRS_INITIAL)
946 : : {
947 : : /* Remember the callbacks for later */
167 948 : 56564 : registered_shmem_callbacks = lappend(registered_shmem_callbacks,
949 : : (void *) callbacks);
950 : : }
951 : : else
25 heikki.linnakangas@i 952 [ # # ]:UBC 0 : elog(ERROR, "cannot request shared memory at this time");
167 heikki.linnakangas@i 953 :CBC 56569 : }
954 : :
955 : : /*
956 : : * Register a shmem area (or multiple areas) after startup.
957 : : */
958 : : static void
959 : 9 : CallShmemCallbacksAfterStartup(const ShmemCallbacks *callbacks)
960 : : {
961 [ - + ]: 9 : Assert(shmem_request_state == SRS_DONE);
962 [ - + ]: 9 : Assert(pending_shmem_requests == NIL);
963 : :
23 964 [ + + ]: 9 : PG_TRY();
965 : : {
966 : 9 : shmem_request_state = SRS_REQUESTING;
967 : :
968 : : /*
969 : : * Call the request callback first. The callback makes
970 : : * ShmemRequest*() calls for each shmem area, adding them to
971 : : * pending_shmem_requests.
972 : : */
973 [ + - ]: 9 : if (callbacks->request_fn)
974 : 9 : callbacks->request_fn(callbacks->opaque_arg);
975 : :
976 : : /* Process all the requests */
977 : 9 : shmem_request_state = SRS_AFTER_STARTUP_ATTACH_OR_INIT;
978 [ + - ]: 9 : if (pending_shmem_requests != NIL)
979 : 9 : ProcessShmemRequestsAfterStartup(callbacks);
980 : : }
981 : 4 : PG_FINALLY();
982 : : {
983 [ + - + + : 27 : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
+ + ]
984 : 9 : pfree(request->options);
985 : 9 : list_free_deep(pending_shmem_requests);
986 : 9 : pending_shmem_requests = NIL;
987 : :
167 988 : 9 : shmem_request_state = SRS_DONE;
989 : : }
23 990 [ + + ]: 9 : PG_END_TRY();
991 : 5 : }
992 : :
993 : : static void
994 : 9 : ProcessShmemRequestsAfterStartup(const ShmemCallbacks *callbacks)
995 : : {
996 : : bool found_any;
997 : : bool notfound_any;
998 : :
999 : : /* There should be some requests to process */
1000 [ - + ]: 9 : Assert(pending_shmem_requests != NIL);
1001 : :
1002 : : /* Caller manages the global state variable */
1003 [ - + ]: 9 : Assert(shmem_request_state == SRS_AFTER_STARTUP_ATTACH_OR_INIT);
1004 : :
1005 : : /*
1006 : : * Hold ShmemIndexLock while we allocate all the shmem entries and run all
1007 : : * the initializers.
1008 : : */
178 1009 : 9 : LWLockAcquire(ShmemIndexLock, LW_EXCLUSIVE);
1010 : :
1011 : : /*
1012 : : * Check if the requested shared memory areas have already been
1013 : : * initialized. We assume all the areas requested by the request callback
1014 : : * to form a coherent unit such that they're all already initialized or
1015 : : * none. Otherwise it would be ambiguous which callback, init or attach,
1016 : : * to callback afterwards.
1017 : : */
167 1018 : 9 : found_any = notfound_any = false;
1019 [ + - + + : 23 : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
+ + ]
1020 : : {
1021 : : ShmemIndexEnt *index_entry;
1022 : :
1023 : : index_entry = (ShmemIndexEnt *)
23 1024 : 9 : hash_search(ShmemIndex, request->options->name, HASH_FIND, NULL);
1025 [ + + ]: 9 : if (index_entry)
1026 : : {
1027 : : /*
1028 : : * Check for a half-initialized area. (See also similar check in
1029 : : * AttachShmemIndexEntry())
1030 : : */
1031 [ + + ]: 2 : if (!index_entry->initialized)
1032 [ + - ]: 1 : ereport(ERROR,
1033 : : (errmsg("cannot attach to shared memory struct \"%s\" because it was not fully initialized",
1034 : : request->options->name)));
167 1035 : 1 : found_any = true;
1036 : : }
1037 : : else
1038 : : {
2 1039 [ + + ]: 7 : if (request->options->size == SHMEM_ATTACH_UNKNOWN_SIZE)
1040 [ + - ]: 1 : ereport(ERROR,
1041 : : (errmsg("cannot attach to shared memory struct \"%s\" because it does not exist",
1042 : : request->options->name),
1043 : : errdetail("SHMEM_ATTACH_UNKNOWN_SIZE can only be used to attach to an existing shared memory structure.")));
167 1044 : 6 : notfound_any = true;
1045 : : }
1046 : : }
1047 [ + + - + ]: 7 : if (found_any && notfound_any)
75 heikki.linnakangas@i 1048 [ # # ]:UBC 0 : elog(ERROR, "some of the requested shmem areas have already been initialized");
1049 : :
1050 : : /*
1051 : : * Allocate or attach all the shmem areas requested by the request_fn
1052 : : * callback.
1053 : : */
167 heikki.linnakangas@i 1054 [ + - + + :CBC 19 : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
+ + ]
1055 : : {
1056 [ + + ]: 7 : if (found_any)
1057 : 1 : AttachShmemIndexEntry(request, false);
1058 : : else
1059 : 6 : InitShmemIndexEntry(request);
1060 : : }
1061 : :
1062 : : /* Finish by calling the appropriate subsystem-specific callback */
1063 [ + + ]: 6 : if (found_any)
1064 : : {
1065 [ + - ]: 1 : if (callbacks->attach_fn)
1066 : 1 : callbacks->attach_fn(callbacks->opaque_arg);
1067 : : }
1068 : : else
1069 : : {
1070 [ + - ]: 5 : if (callbacks->init_fn)
1071 : 5 : callbacks->init_fn(callbacks->opaque_arg);
1072 : : }
1073 : :
23 1074 [ + - + + : 15 : foreach_ptr(ShmemRequest, request, pending_shmem_requests)
+ + ]
1075 : : {
1076 : 5 : request->index_entry->initialized = true;
1077 : : }
1078 : :
7564 tgl@sss.pgh.pa.us 1079 : 5 : LWLockRelease(ShmemIndexLock);
167 heikki.linnakangas@i 1080 : 5 : }
1081 : :
1082 : : /*
1083 : : * Call all shmem request callbacks.
1084 : : */
1085 : : void
1086 : 1277 : ShmemCallRequestCallbacks(void)
1087 : : {
1088 : : ListCell *lc;
1089 : :
1090 [ - + ]: 1277 : Assert(shmem_request_state == SRS_INITIAL);
1091 : 1277 : shmem_request_state = SRS_REQUESTING;
1092 : :
1093 [ + - + + : 57491 : foreach(lc, registered_shmem_callbacks)
+ + ]
1094 : : {
1095 : 56214 : const ShmemCallbacks *callbacks = (const ShmemCallbacks *) lfirst(lc);
1096 : :
1097 [ + - ]: 56214 : if (callbacks->request_fn)
1098 : 56214 : callbacks->request_fn(callbacks->opaque_arg);
1099 : : }
11030 scrappy@hub.org 1100 : 1277 : }
1101 : :
1102 : : /*
1103 : : * ShmemInitStruct -- Create/attach to a structure in shared memory.
1104 : : *
1105 : : * This is called during initialization to find or allocate
1106 : : * a data structure in shared memory. If no other process
1107 : : * has created the structure, this routine allocates space
1108 : : * for it. If it exists already, a pointer to the existing
1109 : : * structure is returned.
1110 : : *
1111 : : * Returns: pointer to the object. *foundPtr is set true if the object was
1112 : : * already in the shmem index (hence, already initialized).
1113 : : *
1114 : : * Note: This is a legacy interface, kept for backwards compatibility with
1115 : : * extensions. Use ShmemRequestStruct() in new code!
1116 : : */
1117 : : void *
167 heikki.linnakangas@i 1118 :UBC 0 : ShmemInitStruct(const char *name, Size size, bool *foundPtr)
1119 : : {
1120 : 0 : void *ptr = NULL;
1121 : 0 : ShmemStructOpts options = {
1122 : : .name = name,
1123 : : .size = size,
1124 : : .ptr = &ptr,
1125 : : };
1126 : 0 : ShmemRequest request = {&options, SHMEM_KIND_STRUCT};
1127 : :
1128 [ # # # # : 0 : Assert(shmem_request_state == SRS_DONE ||
# # ]
1129 : : shmem_request_state == SRS_INITIALIZING ||
1130 : : shmem_request_state == SRS_REQUESTING);
1131 : :
1132 : 0 : LWLockAcquire(ShmemIndexLock, LW_EXCLUSIVE);
1133 : :
1134 : : /*
1135 : : * During postmaster startup, look up the existing entry if any.
1136 : : */
1137 : 0 : *foundPtr = false;
1138 [ # # ]: 0 : if (IsUnderPostmaster)
1139 : 0 : *foundPtr = AttachShmemIndexEntry(&request, true);
1140 : :
1141 : : /* Initialize it if not found */
1142 [ # # ]: 0 : if (!*foundPtr)
1143 : : {
1144 : 0 : InitShmemIndexEntry(&request);
1145 : : /* no additional initialization needed */
23 1146 : 0 : request.index_entry->initialized = true;
1147 : : }
1148 : :
167 1149 : 0 : LWLockRelease(ShmemIndexLock);
1150 : :
1151 [ # # ]: 0 : Assert(ptr != NULL);
1152 : 0 : return ptr;
1153 : : }
1154 : :
1155 : : /* SQL SRF showing allocated shared memory */
1156 : : Datum
2446 rhaas@postgresql.org 1157 :CBC 4 : pg_get_shmem_allocations(PG_FUNCTION_ARGS)
1158 : : {
1159 : : #define PG_GET_SHMEM_SIZES_COLS 4
1160 : 4 : ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
1161 : : HASH_SEQ_STATUS hstat;
1162 : : ShmemIndexEnt *ent;
2320 tgl@sss.pgh.pa.us 1163 : 4 : Size named_allocated = 0;
1164 : : Datum values[PG_GET_SHMEM_SIZES_COLS];
1165 : : bool nulls[PG_GET_SHMEM_SIZES_COLS];
1166 : :
1433 michael@paquier.xyz 1167 : 4 : InitMaterializedSRF(fcinfo, 0);
1168 : :
2446 rhaas@postgresql.org 1169 : 4 : LWLockAcquire(ShmemIndexLock, LW_SHARED);
1170 : :
1171 : 4 : hash_seq_init(&hstat, ShmemIndex);
1172 : :
1173 : : /* output all allocated entries */
1174 : 4 : memset(nulls, 0, sizeof(nulls));
1175 [ + + ]: 326 : while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL)
1176 : : {
1177 : 322 : values[0] = CStringGetTextDatum(ent->key);
1178 : 322 : values[1] = Int64GetDatum((char *) ent->location - (char *) ShmemSegHdr);
1179 : 322 : values[2] = Int64GetDatum(ent->size);
1180 : 322 : values[3] = Int64GetDatum(ent->allocated_size);
1181 : 322 : named_allocated += ent->allocated_size;
1182 : :
1658 michael@paquier.xyz 1183 : 322 : tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
1184 : : values, nulls);
1185 : : }
1186 : :
1187 : : /* output shared memory allocated but not counted via the shmem index */
2446 rhaas@postgresql.org 1188 : 4 : values[0] = CStringGetTextDatum("<anonymous>");
1189 : 4 : nulls[1] = true;
233 heikki.linnakangas@i 1190 : 4 : values[2] = Int64GetDatum(ShmemAllocator->free_offset - named_allocated);
2446 rhaas@postgresql.org 1191 : 4 : values[3] = values[2];
1658 michael@paquier.xyz 1192 : 4 : tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
1193 : :
1194 : : /* output as-of-yet unused shared memory */
2446 rhaas@postgresql.org 1195 : 4 : nulls[0] = true;
233 heikki.linnakangas@i 1196 : 4 : values[1] = Int64GetDatum(ShmemAllocator->free_offset);
2446 rhaas@postgresql.org 1197 : 4 : nulls[1] = false;
233 heikki.linnakangas@i 1198 : 4 : values[2] = Int64GetDatum(ShmemSegHdr->totalsize - ShmemAllocator->free_offset);
2446 rhaas@postgresql.org 1199 : 4 : values[3] = values[2];
1658 michael@paquier.xyz 1200 : 4 : tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls);
1201 : :
2446 rhaas@postgresql.org 1202 : 4 : LWLockRelease(ShmemIndexLock);
1203 : :
1204 : 4 : return (Datum) 0;
1205 : : }
1206 : :
1207 : : /*
1208 : : * SQL SRF showing NUMA memory nodes for allocated shared memory
1209 : : *
1210 : : * Compared to pg_get_shmem_allocations(), this function does not return
1211 : : * information about shared anonymous allocations and unused shared memory.
1212 : : */
1213 : : Datum
531 tomas.vondra@postgre 1214 : 4 : pg_get_shmem_allocations_numa(PG_FUNCTION_ARGS)
1215 : : {
1216 : : #define PG_GET_SHMEM_NUMA_SIZES_COLS 3
1217 : 4 : ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo;
1218 : : HASH_SEQ_STATUS hstat;
1219 : : ShmemIndexEnt *ent;
1220 : : Datum values[PG_GET_SHMEM_NUMA_SIZES_COLS];
1221 : : bool nulls[PG_GET_SHMEM_NUMA_SIZES_COLS];
1222 : : Size os_page_size;
1223 : : void **page_ptrs;
1224 : : int *pages_status;
1225 : : uint64 shm_total_page_count,
1226 : : shm_ent_page_count,
1227 : : max_nodes;
1228 : : Size *nodes;
1229 : :
1230 [ + - ]: 4 : if (pg_numa_init() == -1)
1231 [ + - ]: 4 : elog(ERROR, "libnuma initialization failed or NUMA is not supported on this platform");
1232 : :
531 tomas.vondra@postgre 1233 :UBC 0 : InitMaterializedSRF(fcinfo, 0);
1234 : :
1235 : 0 : max_nodes = pg_numa_get_max_node();
237 1236 : 0 : nodes = palloc_array(Size, max_nodes + 2);
1237 : :
1238 : : /*
1239 : : * Shared memory allocations can vary in size and may not align with OS
1240 : : * memory page boundaries, while NUMA queries work on pages.
1241 : : *
1242 : : * To correctly map each allocation to NUMA nodes, we need to: 1.
1243 : : * Determine the OS memory page size. 2. Align each allocation's start/end
1244 : : * addresses to page boundaries. 3. Query NUMA node information for all
1245 : : * pages spanning the allocation.
1246 : : */
529 1247 : 0 : os_page_size = pg_get_shmem_pagesize();
1248 : :
1249 : : /*
1250 : : * Allocate memory for page pointers and status based on total shared
1251 : : * memory size. This simplified approach allocates enough space for all
1252 : : * pages in shared memory rather than calculating the exact requirements
1253 : : * for each segment.
1254 : : *
1255 : : * Add 1, because we don't know how exactly the segments align to OS
1256 : : * pages, so the allocation might use one more memory page. In practice
1257 : : * this is not very likely, and moreover we have more entries, each of
1258 : : * them using only fraction of the total pages.
1259 : : */
531 1260 : 0 : shm_total_page_count = (ShmemSegHdr->totalsize / os_page_size) + 1;
284 michael@paquier.xyz 1261 : 0 : page_ptrs = palloc0_array(void *, shm_total_page_count);
1262 : 0 : pages_status = palloc_array(int, shm_total_page_count);
1263 : :
531 tomas.vondra@postgre 1264 [ # # ]: 0 : if (firstNumaTouch)
1265 [ # # ]: 0 : elog(DEBUG1, "NUMA: page-faulting shared memory segments for proper NUMA readouts");
1266 : :
1267 : 0 : LWLockAcquire(ShmemIndexLock, LW_SHARED);
1268 : :
1269 : 0 : hash_seq_init(&hstat, ShmemIndex);
1270 : :
1271 : : /* output all allocated entries */
1272 [ # # ]: 0 : while ((ent = (ShmemIndexEnt *) hash_seq_search(&hstat)) != NULL)
1273 : : {
1274 : : char *startptr,
1275 : : *endptr;
1276 : : Size total_len;
1277 : :
1278 : : /*
1279 : : * Calculate the range of OS pages used by this segment. The segment
1280 : : * may start / end half-way through a page, we want to count these
1281 : : * pages too. So we align the start/end pointers down/up, and then
1282 : : * calculate the number of pages from that.
1283 : : */
1284 : 0 : startptr = (char *) TYPEALIGN_DOWN(os_page_size, ent->location);
1285 : 0 : endptr = (char *) TYPEALIGN(os_page_size,
1286 : : (char *) ent->location + ent->allocated_size);
1287 : 0 : total_len = (endptr - startptr);
1288 : :
1289 : 0 : shm_ent_page_count = total_len / os_page_size;
1290 : :
1291 : : /*
1292 : : * If we ever get 0xff (-1) back from kernel inquiry, then we probably
1293 : : * have a bug in mapping buffers to OS pages.
1294 : : */
1295 : 0 : memset(pages_status, 0xff, sizeof(int) * shm_ent_page_count);
1296 : :
1297 : : /*
1298 : : * Setup page_ptrs[] with pointers to all OS pages for this segment,
1299 : : * and get the NUMA status using pg_numa_query_pages.
1300 : : *
1301 : : * In order to get reliable results we also need to touch memory
1302 : : * pages, so that inquiry about NUMA memory node doesn't return -2
1303 : : * (ENOENT, which indicates unmapped/unallocated pages).
1304 : : */
71 peter@eisentraut.org 1305 [ # # ]:UNC 0 : for (uint64 i = 0; i < shm_ent_page_count; i++)
1306 : : {
531 tomas.vondra@postgre 1307 :UBC 0 : page_ptrs[i] = startptr + (i * os_page_size);
1308 : :
1309 : 0 : if (firstNumaTouch)
1310 : : pg_numa_touch_mem_if_required(page_ptrs[i]);
1311 : :
1312 [ # # ]: 0 : CHECK_FOR_INTERRUPTS();
1313 : : }
1314 : :
1315 [ # # ]: 0 : if (pg_numa_query_pages(0, shm_ent_page_count, page_ptrs, pages_status) == -1)
1316 [ # # ]: 0 : elog(ERROR, "failed NUMA pages inquiry status: %m");
1317 : :
1318 : : /* Count number of NUMA nodes used for this shared memory entry */
237 1319 : 0 : memset(nodes, 0, sizeof(Size) * (max_nodes + 2));
1320 : :
71 peter@eisentraut.org 1321 [ # # ]:UNC 0 : for (uint64 i = 0; i < shm_ent_page_count; i++)
1322 : : {
531 tomas.vondra@postgre 1323 :UBC 0 : int s = pages_status[i];
1324 : :
1325 : : /* Ensure we are adding only valid index to the array */
237 1326 [ # # # # ]: 0 : if (s >= 0 && s <= max_nodes)
1327 : : {
1328 : : /* valid NUMA node */
1329 : 0 : nodes[s]++;
1330 : 0 : continue;
1331 : : }
1332 [ # # ]: 0 : else if (s == -2)
1333 : : {
1334 : : /* -2 means ENOENT (e.g. page was moved to swap) */
1335 : 0 : nodes[max_nodes + 1]++;
1336 : 0 : continue;
1337 : : }
1338 : :
1339 [ # # ]: 0 : elog(ERROR, "invalid NUMA node id outside of allowed range "
1340 : : "[0, " UINT64_FORMAT "]: %d", max_nodes, s);
1341 : : }
1342 : :
1343 : : /* no NULLs for regular nodes */
1344 : 0 : memset(nulls, 0, sizeof(nulls));
1345 : :
1346 : : /*
1347 : : * Add one entry for each NUMA node, including those without allocated
1348 : : * memory for this segment.
1349 : : */
71 peter@eisentraut.org 1350 [ # # ]:UNC 0 : for (uint64 i = 0; i <= max_nodes; i++)
1351 : : {
531 tomas.vondra@postgre 1352 :UBC 0 : values[0] = CStringGetTextDatum(ent->key);
408 peter@eisentraut.org 1353 : 0 : values[1] = Int32GetDatum(i);
531 tomas.vondra@postgre 1354 : 0 : values[2] = Int64GetDatum(nodes[i] * os_page_size);
1355 : :
1356 : 0 : tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
1357 : : values, nulls);
1358 : : }
1359 : :
1360 : : /* The last entry is used for pages without a NUMA node. */
237 1361 : 0 : nulls[1] = true;
1362 : 0 : values[0] = CStringGetTextDatum(ent->key);
1363 : 0 : values[2] = Int64GetDatum(nodes[max_nodes + 1] * os_page_size);
1364 : :
1365 : 0 : tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc,
1366 : : values, nulls);
1367 : : }
1368 : :
531 1369 : 0 : LWLockRelease(ShmemIndexLock);
1370 : 0 : firstNumaTouch = false;
1371 : :
1372 : 0 : return (Datum) 0;
1373 : : }
1374 : :
1375 : : /*
1376 : : * Determine the memory page size used for the shared memory segment.
1377 : : *
1378 : : * If the shared segment was allocated using huge pages, returns the size of
1379 : : * a huge page. Otherwise returns the size of regular memory page.
1380 : : *
1381 : : * This should be used only after the server is started.
1382 : : */
1383 : : Size
529 tomas.vondra@postgre 1384 :CBC 2 : pg_get_shmem_pagesize(void)
1385 : : {
1386 : : Size os_page_size;
1387 : : #ifdef WIN32
1388 : : SYSTEM_INFO sysinfo;
1389 : :
1390 : : GetSystemInfo(&sysinfo);
1391 : : os_page_size = sysinfo.dwPageSize;
1392 : : #else
1393 : 2 : os_page_size = sysconf(_SC_PAGESIZE);
1394 : : #endif
1395 : :
1396 [ - + ]: 2 : Assert(IsUnderPostmaster);
1397 [ - + ]: 2 : Assert(huge_pages_status != HUGE_PAGES_UNKNOWN);
1398 : :
1399 [ - + ]: 2 : if (huge_pages_status == HUGE_PAGES_ON)
529 tomas.vondra@postgre 1400 :UBC 0 : GetHugePageSize(&os_page_size, NULL);
1401 : :
529 tomas.vondra@postgre 1402 :CBC 2 : return os_page_size;
1403 : : }
1404 : :
1405 : : Datum
1406 : 5 : pg_numa_available(PG_FUNCTION_ARGS)
1407 : : {
1408 : 5 : PG_RETURN_BOOL(pg_numa_init() != -1);
1409 : : }
|
