Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit.c
4 : * Core part of the LLVM JIT provider.
5 : *
6 : * Copyright (c) 2016-2023, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/jit/llvm/llvmjit.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 :
14 : #include "postgres.h"
15 :
16 : #include <llvm-c/Analysis.h>
17 : #include <llvm-c/BitReader.h>
18 : #include <llvm-c/BitWriter.h>
19 : #include <llvm-c/Core.h>
20 : #include <llvm-c/ExecutionEngine.h>
21 : #if LLVM_VERSION_MAJOR > 16
22 : #include <llvm-c/Transforms/PassBuilder.h>
23 : #endif
24 : #if LLVM_VERSION_MAJOR > 11
25 : #include <llvm-c/Orc.h>
26 : #include <llvm-c/OrcEE.h>
27 : #include <llvm-c/LLJIT.h>
28 : #else
29 : #include <llvm-c/OrcBindings.h>
30 : #endif
31 : #include <llvm-c/Support.h>
32 : #include <llvm-c/Target.h>
33 : #if LLVM_VERSION_MAJOR < 17
34 : #include <llvm-c/Transforms/IPO.h>
35 : #include <llvm-c/Transforms/PassManagerBuilder.h>
36 : #include <llvm-c/Transforms/Scalar.h>
37 : #if LLVM_VERSION_MAJOR > 6
38 : #include <llvm-c/Transforms/Utils.h>
39 : #endif
40 : #endif
41 :
42 : #include "jit/llvmjit.h"
43 : #include "jit/llvmjit_emit.h"
44 : #include "miscadmin.h"
45 : #include "portability/instr_time.h"
46 : #include "storage/ipc.h"
47 : #include "utils/memutils.h"
48 : #include "utils/resowner.h"
49 :
50 : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
51 :
52 : /* Handle of a module emitted via ORC JIT */
53 : typedef struct LLVMJitHandle
54 : {
55 : #if LLVM_VERSION_MAJOR > 11
56 : LLVMOrcLLJITRef lljit;
57 : LLVMOrcResourceTrackerRef resource_tracker;
58 : #else
59 : LLVMOrcJITStackRef stack;
60 : LLVMOrcModuleHandle orc_handle;
61 : #endif
62 : } LLVMJitHandle;
63 :
64 :
65 : /* types & functions commonly needed for JITing */
66 : LLVMTypeRef TypeSizeT;
67 : LLVMTypeRef TypeParamBool;
68 : LLVMTypeRef TypeStorageBool;
69 : LLVMTypeRef TypePGFunction;
70 : LLVMTypeRef StructNullableDatum;
71 : LLVMTypeRef StructHeapTupleData;
72 : LLVMTypeRef StructMinimalTupleData;
73 : LLVMTypeRef StructTupleDescData;
74 : LLVMTypeRef StructTupleTableSlot;
75 : LLVMTypeRef StructHeapTupleHeaderData;
76 : LLVMTypeRef StructHeapTupleTableSlot;
77 : LLVMTypeRef StructMinimalTupleTableSlot;
78 : LLVMTypeRef StructMemoryContextData;
79 : LLVMTypeRef StructFunctionCallInfoData;
80 : LLVMTypeRef StructExprContext;
81 : LLVMTypeRef StructExprEvalStep;
82 : LLVMTypeRef StructExprState;
83 : LLVMTypeRef StructAggState;
84 : LLVMTypeRef StructAggStatePerGroupData;
85 : LLVMTypeRef StructAggStatePerTransData;
86 : LLVMTypeRef StructPlanState;
87 :
88 : LLVMValueRef AttributeTemplate;
89 : LLVMValueRef ExecEvalSubroutineTemplate;
90 : LLVMValueRef ExecEvalBoolSubroutineTemplate;
91 :
92 : static LLVMModuleRef llvm_types_module = NULL;
93 :
94 : static bool llvm_session_initialized = false;
95 : static size_t llvm_generation = 0;
96 :
97 : /* number of LLVMJitContexts that currently are in use */
98 : static size_t llvm_jit_context_in_use_count = 0;
99 :
100 : /* how many times has the current LLVMContextRef been used */
101 : static size_t llvm_llvm_context_reuse_count = 0;
102 : static const char *llvm_triple = NULL;
103 : static const char *llvm_layout = NULL;
104 : static LLVMContextRef llvm_context;
105 :
106 :
107 : static LLVMTargetRef llvm_targetref;
108 : #if LLVM_VERSION_MAJOR > 11
109 : static LLVMOrcThreadSafeContextRef llvm_ts_context;
110 : static LLVMOrcLLJITRef llvm_opt0_orc;
111 : static LLVMOrcLLJITRef llvm_opt3_orc;
112 : #else /* LLVM_VERSION_MAJOR > 11 */
113 : static LLVMOrcJITStackRef llvm_opt0_orc;
114 : static LLVMOrcJITStackRef llvm_opt3_orc;
115 : #endif /* LLVM_VERSION_MAJOR > 11 */
116 :
117 :
118 : static void llvm_release_context(JitContext *context);
119 : static void llvm_session_initialize(void);
120 : static void llvm_shutdown(int code, Datum arg);
121 : static void llvm_compile_module(LLVMJitContext *context);
122 : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
123 :
124 : static void llvm_create_types(void);
125 : static void llvm_set_target(void);
126 : static void llvm_recreate_llvm_context(void);
127 : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
128 :
129 : #if LLVM_VERSION_MAJOR > 11
130 : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
131 : static char *llvm_error_message(LLVMErrorRef error);
132 : #endif /* LLVM_VERSION_MAJOR > 11 */
133 :
134 : /* ResourceOwner callbacks to hold JitContexts */
135 : static void ResOwnerReleaseJitContext(Datum res);
136 :
137 : static const ResourceOwnerDesc jit_resowner_desc =
138 : {
139 : .name = "LLVM JIT context",
140 : .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
141 : .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
142 : .ReleaseResource = ResOwnerReleaseJitContext,
143 : .DebugPrint = NULL /* the default message is fine */
144 : };
145 :
146 : /* Convenience wrappers over ResourceOwnerRemember/Forget */
147 : static inline void
148 1672 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
149 : {
150 1672 : ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
151 1672 : }
152 : static inline void
153 1656 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
154 : {
155 1656 : ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
156 1656 : }
157 :
158 1960 : PG_MODULE_MAGIC;
159 :
160 :
161 : /*
162 : * Initialize LLVM JIT provider.
163 : */
164 : void
165 770 : _PG_jit_provider_init(JitProviderCallbacks *cb)
166 : {
167 770 : cb->reset_after_error = llvm_reset_after_error;
168 770 : cb->release_context = llvm_release_context;
169 770 : cb->compile_expr = llvm_compile_expr;
170 770 : }
171 :
172 :
173 : /*
174 : * Every now and then create a new LLVMContextRef. Unfortunately, during every
175 : * round of inlining, types may "leak" (they can still be found/used via the
176 : * context, but new types will be created the next time in inlining is
177 : * performed). To prevent that from slowly accumulating problematic amounts of
178 : * memory, recreate the LLVMContextRef we use. We don't want to do so too
179 : * often, as that implies some overhead (particularly re-loading the module
180 : * summaries / modules is fairly expensive). A future TODO would be to make
181 : * this more finegrained and only drop/recreate the LLVMContextRef when we know
182 : * there has been inlining. If we can get the size of the context from LLVM
183 : * then that might be a better way to determine when to drop/recreate rather
184 : * then the usagecount heuristic currently employed.
185 : */
186 : static void
187 1672 : llvm_recreate_llvm_context(void)
188 : {
189 1672 : if (!llvm_context)
190 0 : elog(ERROR, "Trying to recreate a non-existing context");
191 :
192 : /*
193 : * We can only safely recreate the LLVM context if no other code is being
194 : * JITed, otherwise we'd release the types in use for that.
195 : */
196 1672 : if (llvm_jit_context_in_use_count > 0)
197 : {
198 0 : llvm_llvm_context_reuse_count++;
199 0 : return;
200 : }
201 :
202 1672 : if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
203 : {
204 1672 : llvm_llvm_context_reuse_count++;
205 1672 : return;
206 : }
207 :
208 : /*
209 : * Need to reset the modules that the inlining code caches before
210 : * disposing of the context. LLVM modules exist within a specific LLVM
211 : * context, therefore disposing of the context before resetting the cache
212 : * would lead to dangling pointers to modules.
213 : */
214 0 : llvm_inline_reset_caches();
215 :
216 0 : LLVMContextDispose(llvm_context);
217 0 : llvm_context = LLVMContextCreate();
218 0 : llvm_llvm_context_reuse_count = 0;
219 :
220 : /*
221 : * Re-build cached type information, so code generation code can rely on
222 : * that information to be present (also prevents the variables to be
223 : * dangling references).
224 : */
225 0 : llvm_create_types();
226 : }
227 :
228 :
229 : /*
230 : * Create a context for JITing work.
231 : *
232 : * The context, including subsidiary resources, will be cleaned up either when
233 : * the context is explicitly released, or when the lifetime of
234 : * CurrentResourceOwner ends (usually the end of the current [sub]xact).
235 : */
236 : LLVMJitContext *
237 1672 : llvm_create_context(int jitFlags)
238 : {
239 : LLVMJitContext *context;
240 :
241 1672 : llvm_assert_in_fatal_section();
242 :
243 1672 : llvm_session_initialize();
244 :
245 1672 : llvm_recreate_llvm_context();
246 :
247 1672 : ResourceOwnerEnlarge(CurrentResourceOwner);
248 :
249 1672 : context = MemoryContextAllocZero(TopMemoryContext,
250 : sizeof(LLVMJitContext));
251 1672 : context->base.flags = jitFlags;
252 :
253 : /* ensure cleanup */
254 1672 : context->base.resowner = CurrentResourceOwner;
255 1672 : ResourceOwnerRememberJIT(CurrentResourceOwner, context);
256 :
257 1672 : llvm_jit_context_in_use_count++;
258 :
259 1672 : return context;
260 : }
261 :
262 : /*
263 : * Release resources required by one llvm context.
264 : */
265 : static void
266 1672 : llvm_release_context(JitContext *context)
267 : {
268 1672 : LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
269 : ListCell *lc;
270 :
271 : /*
272 : * Consider as cleaned up even if we skip doing so below, that way we can
273 : * verify the tracking is correct (see llvm_shutdown()).
274 : */
275 1672 : llvm_jit_context_in_use_count--;
276 :
277 : /*
278 : * When this backend is exiting, don't clean up LLVM. As an error might
279 : * have occurred from within LLVM, we do not want to risk reentering. All
280 : * resource cleanup is going to happen through process exit.
281 : */
282 1672 : if (proc_exit_inprogress)
283 0 : return;
284 :
285 1672 : llvm_enter_fatal_on_oom();
286 :
287 1672 : if (llvm_jit_context->module)
288 : {
289 486 : LLVMDisposeModule(llvm_jit_context->module);
290 486 : llvm_jit_context->module = NULL;
291 : }
292 :
293 2980 : foreach(lc, llvm_jit_context->handles)
294 : {
295 1308 : LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
296 :
297 : #if LLVM_VERSION_MAJOR > 11
298 : {
299 : LLVMOrcExecutionSessionRef ee;
300 : LLVMOrcSymbolStringPoolRef sp;
301 :
302 : LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
303 : LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
304 :
305 : /*
306 : * Without triggering cleanup of the string pool, we'd leak
307 : * memory. It'd be sufficient to do this far less often, but in
308 : * experiments the required time was small enough to just always
309 : * do it.
310 : */
311 : ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
312 : sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
313 : LLVMOrcSymbolStringPoolClearDeadEntries(sp);
314 : }
315 : #else /* LLVM_VERSION_MAJOR > 11 */
316 : {
317 1308 : LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
318 : }
319 : #endif /* LLVM_VERSION_MAJOR > 11 */
320 :
321 1308 : pfree(jit_handle);
322 : }
323 1672 : list_free(llvm_jit_context->handles);
324 1672 : llvm_jit_context->handles = NIL;
325 :
326 1672 : llvm_leave_fatal_on_oom();
327 :
328 1672 : if (context->resowner)
329 1656 : ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
330 : }
331 :
332 : /*
333 : * Return module which may be modified, e.g. by creating new functions.
334 : */
335 : LLVMModuleRef
336 16194 : llvm_mutable_module(LLVMJitContext *context)
337 : {
338 16194 : llvm_assert_in_fatal_section();
339 :
340 : /*
341 : * If there's no in-progress module, create a new one.
342 : */
343 16194 : if (!context->module)
344 : {
345 1794 : context->compiled = false;
346 1794 : context->module_generation = llvm_generation++;
347 1794 : context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
348 1794 : LLVMSetTarget(context->module, llvm_triple);
349 1794 : LLVMSetDataLayout(context->module, llvm_layout);
350 : }
351 :
352 16194 : return context->module;
353 : }
354 :
355 : /*
356 : * Expand function name to be non-conflicting. This should be used by code
357 : * generating code, when adding new externally visible function definitions to
358 : * a Module.
359 : */
360 : char *
361 16194 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
362 : {
363 : Assert(context->module != NULL);
364 :
365 16194 : context->base.instr.created_functions++;
366 :
367 : /*
368 : * Previously we used dots to separate, but turns out some tools, e.g.
369 : * GDB, don't like that and truncate name.
370 : */
371 32388 : return psprintf("%s_%zu_%d",
372 : basename,
373 : context->module_generation,
374 16194 : context->counter++);
375 : }
376 :
377 : /*
378 : * Return pointer to function funcname, which has to exist. If there's pending
379 : * code to be optimized and emitted, do so first.
380 : */
381 : void *
382 6002 : llvm_get_function(LLVMJitContext *context, const char *funcname)
383 : {
384 : #if LLVM_VERSION_MAJOR > 11 || \
385 : defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
386 : ListCell *lc;
387 : #endif
388 :
389 6002 : llvm_assert_in_fatal_section();
390 :
391 : /*
392 : * If there is a pending / not emitted module, compile and emit now.
393 : * Otherwise we might not find the [correct] function.
394 : */
395 6002 : if (!context->compiled)
396 : {
397 1308 : llvm_compile_module(context);
398 : }
399 :
400 : /*
401 : * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
402 : * to mangle here.
403 : */
404 :
405 : #if LLVM_VERSION_MAJOR > 11
406 : foreach(lc, context->handles)
407 : {
408 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
409 : instr_time starttime;
410 : instr_time endtime;
411 : LLVMErrorRef error;
412 : LLVMOrcJITTargetAddress addr;
413 :
414 : INSTR_TIME_SET_CURRENT(starttime);
415 :
416 : addr = 0;
417 : error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
418 : if (error)
419 : elog(ERROR, "failed to look up symbol \"%s\": %s",
420 : funcname, llvm_error_message(error));
421 :
422 : /*
423 : * LLJIT only actually emits code the first time a symbol is
424 : * referenced. Thus add lookup time to emission time. That's counting
425 : * a bit more than with older LLVM versions, but unlikely to ever
426 : * matter.
427 : */
428 : INSTR_TIME_SET_CURRENT(endtime);
429 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
430 : endtime, starttime);
431 :
432 : if (addr)
433 : return (void *) (uintptr_t) addr;
434 : }
435 : #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
436 6136 : foreach(lc, context->handles)
437 : {
438 : LLVMOrcTargetAddress addr;
439 6136 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
440 :
441 6136 : addr = 0;
442 6136 : if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
443 0 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
444 6136 : if (addr)
445 6002 : return (void *) (uintptr_t) addr;
446 : }
447 : #elif LLVM_VERSION_MAJOR < 5
448 : {
449 : LLVMOrcTargetAddress addr;
450 :
451 : if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
452 : return (void *) (uintptr_t) addr;
453 : if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
454 : return (void *) (uintptr_t) addr;
455 : }
456 : #else
457 : {
458 : LLVMOrcTargetAddress addr;
459 :
460 : if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
461 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
462 : if (addr)
463 : return (void *) (uintptr_t) addr;
464 : if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
465 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
466 : if (addr)
467 : return (void *) (uintptr_t) addr;
468 : }
469 : #endif
470 :
471 0 : elog(ERROR, "failed to JIT: %s", funcname);
472 :
473 : return NULL;
474 : }
475 :
476 : /*
477 : * Return type of a variable in llvmjit_types.c. This is useful to keep types
478 : * in sync between plain C and JIT related code.
479 : */
480 : LLVMTypeRef
481 15518 : llvm_pg_var_type(const char *varname)
482 : {
483 : LLVMValueRef v_srcvar;
484 : LLVMTypeRef typ;
485 :
486 : /* this'll return a *pointer* to the global */
487 15518 : v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
488 15518 : if (!v_srcvar)
489 0 : elog(ERROR, "variable %s not in llvmjit_types.c", varname);
490 :
491 15518 : typ = LLVMGlobalGetValueType(v_srcvar);
492 :
493 15518 : return typ;
494 : }
495 :
496 : /*
497 : * Return function type of a variable in llvmjit_types.c. This is useful to
498 : * keep function types in sync between C and JITed code.
499 : */
500 : LLVMTypeRef
501 14638 : llvm_pg_var_func_type(const char *varname)
502 : {
503 : LLVMValueRef v_srcvar;
504 : LLVMTypeRef typ;
505 :
506 14638 : v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
507 14638 : if (!v_srcvar)
508 0 : elog(ERROR, "function %s not in llvmjit_types.c", varname);
509 :
510 14638 : typ = LLVMGetFunctionType(v_srcvar);
511 :
512 14638 : return typ;
513 : }
514 :
515 : /*
516 : * Return declaration for a function referenced in llvmjit_types.c, adding it
517 : * to the module if necessary.
518 : *
519 : * This is used to make functions discovered via llvm_create_types() known to
520 : * the module that's currently being worked on.
521 : */
522 : LLVMValueRef
523 13244 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
524 : {
525 : LLVMValueRef v_srcfn;
526 : LLVMValueRef v_fn;
527 :
528 : /* don't repeatedly add function */
529 13244 : v_fn = LLVMGetNamedFunction(mod, funcname);
530 13244 : if (v_fn)
531 9500 : return v_fn;
532 :
533 3744 : v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
534 :
535 3744 : if (!v_srcfn)
536 0 : elog(ERROR, "function %s not in llvmjit_types.c", funcname);
537 :
538 3744 : v_fn = LLVMAddFunction(mod,
539 : funcname,
540 : LLVMGetFunctionType(v_srcfn));
541 3744 : llvm_copy_attributes(v_srcfn, v_fn);
542 :
543 3744 : return v_fn;
544 : }
545 :
546 : /*
547 : * Copy attributes from one function to another, for a specific index (an
548 : * index can reference return value, function and parameter attributes).
549 : */
550 : static void
551 65360 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
552 : {
553 : int num_attributes;
554 : LLVMAttributeRef *attrs;
555 :
556 65360 : num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
557 :
558 : /*
559 : * Not just for efficiency: LLVM <= 3.9 crashes when
560 : * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
561 : */
562 65360 : if (num_attributes == 0)
563 29166 : return;
564 :
565 36194 : attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
566 36194 : LLVMGetAttributesAtIndex(v_from, index, attrs);
567 :
568 425002 : for (int attno = 0; attno < num_attributes; attno++)
569 388808 : LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
570 :
571 36194 : pfree(attrs);
572 : }
573 :
574 : /*
575 : * Copy all attributes from one function to another. I.e. function, return and
576 : * parameters will be copied.
577 : */
578 : void
579 19938 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
580 : {
581 : uint32 param_count;
582 :
583 : /* copy function attributes */
584 19938 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
585 :
586 : /* and the return value attributes */
587 19938 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
588 :
589 : /* and each function parameter's attribute */
590 19938 : param_count = LLVMCountParams(v_from);
591 :
592 45422 : for (int paramidx = 1; paramidx <= param_count; paramidx++)
593 25484 : llvm_copy_attributes_at_index(v_from, v_to, paramidx);
594 19938 : }
595 :
596 : /*
597 : * Return a callable LLVMValueRef for fcinfo.
598 : */
599 : LLVMValueRef
600 5476 : llvm_function_reference(LLVMJitContext *context,
601 : LLVMBuilderRef builder,
602 : LLVMModuleRef mod,
603 : FunctionCallInfo fcinfo)
604 : {
605 : char *modname;
606 : char *basename;
607 : char *funcname;
608 :
609 : LLVMValueRef v_fn;
610 :
611 5476 : fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
612 :
613 5476 : if (modname != NULL && basename != NULL)
614 : {
615 : /* external function in loadable library */
616 38 : funcname = psprintf("pgextern.%s.%s", modname, basename);
617 : }
618 5438 : else if (basename != NULL)
619 : {
620 : /* internal function */
621 5434 : funcname = pstrdup(basename);
622 : }
623 : else
624 : {
625 : /*
626 : * Function we don't know to handle, return pointer. We do so by
627 : * creating a global constant containing a pointer to the function.
628 : * Makes IR more readable.
629 : */
630 : LLVMValueRef v_fn_addr;
631 :
632 4 : funcname = psprintf("pgoidextern.%u",
633 4 : fcinfo->flinfo->fn_oid);
634 4 : v_fn = LLVMGetNamedGlobal(mod, funcname);
635 4 : if (v_fn != 0)
636 0 : return l_load(builder, TypePGFunction, v_fn, "");
637 :
638 4 : v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
639 :
640 4 : v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
641 4 : LLVMSetInitializer(v_fn, v_fn_addr);
642 4 : LLVMSetGlobalConstant(v_fn, true);
643 4 : LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
644 4 : LLVMSetUnnamedAddr(v_fn, true);
645 :
646 4 : return l_load(builder, TypePGFunction, v_fn, "");
647 : }
648 :
649 : /* check if function already has been added */
650 5472 : v_fn = LLVMGetNamedFunction(mod, funcname);
651 5472 : if (v_fn != 0)
652 2318 : return v_fn;
653 :
654 3154 : v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
655 :
656 3154 : return v_fn;
657 : }
658 :
659 : /*
660 : * Optimize code in module using the flags set in context.
661 : */
662 : static void
663 1308 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
664 : {
665 : #if LLVM_VERSION_MAJOR < 17
666 : LLVMPassManagerBuilderRef llvm_pmb;
667 : LLVMPassManagerRef llvm_mpm;
668 : LLVMPassManagerRef llvm_fpm;
669 : LLVMValueRef func;
670 : int compile_optlevel;
671 :
672 1308 : if (context->base.flags & PGJIT_OPT3)
673 978 : compile_optlevel = 3;
674 : else
675 330 : compile_optlevel = 0;
676 :
677 : /*
678 : * Have to create a new pass manager builder every pass through, as the
679 : * inliner has some per-builder state. Otherwise one ends up only inlining
680 : * a function the first time though.
681 : */
682 1308 : llvm_pmb = LLVMPassManagerBuilderCreate();
683 1308 : LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
684 1308 : llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
685 :
686 1308 : if (context->base.flags & PGJIT_OPT3)
687 : {
688 : /* TODO: Unscientifically determined threshold */
689 978 : LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
690 : }
691 : else
692 : {
693 : /* we rely on mem2reg heavily, so emit even in the O0 case */
694 330 : LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
695 : }
696 :
697 1308 : LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
698 :
699 : /*
700 : * Do function level optimization. This could be moved to the point where
701 : * functions are emitted, to reduce memory usage a bit.
702 : */
703 1308 : LLVMInitializeFunctionPassManager(llvm_fpm);
704 31494 : for (func = LLVMGetFirstFunction(context->module);
705 : func != NULL;
706 30186 : func = LLVMGetNextFunction(func))
707 30186 : LLVMRunFunctionPassManager(llvm_fpm, func);
708 1308 : LLVMFinalizeFunctionPassManager(llvm_fpm);
709 1308 : LLVMDisposePassManager(llvm_fpm);
710 :
711 : /*
712 : * Perform module level optimization. We do so even in the non-optimized
713 : * case, so always-inline functions etc get inlined. It's cheap enough.
714 : */
715 1308 : llvm_mpm = LLVMCreatePassManager();
716 1308 : LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
717 : llvm_mpm);
718 : /* always use always-inliner pass */
719 1308 : if (!(context->base.flags & PGJIT_OPT3))
720 330 : LLVMAddAlwaysInlinerPass(llvm_mpm);
721 : /* if doing inlining, but no expensive optimization, add inlining pass */
722 1308 : if (context->base.flags & PGJIT_INLINE
723 978 : && !(context->base.flags & PGJIT_OPT3))
724 0 : LLVMAddFunctionInliningPass(llvm_mpm);
725 1308 : LLVMRunPassManager(llvm_mpm, context->module);
726 1308 : LLVMDisposePassManager(llvm_mpm);
727 :
728 1308 : LLVMPassManagerBuilderDispose(llvm_pmb);
729 : #else
730 : LLVMPassBuilderOptionsRef options;
731 : LLVMErrorRef err;
732 : const char *passes;
733 :
734 : if (context->base.flags & PGJIT_OPT3)
735 : passes = "default<O3>";
736 : else
737 : passes = "default<O0>,mem2reg";
738 :
739 : options = LLVMCreatePassBuilderOptions();
740 :
741 : #ifdef LLVM_PASS_DEBUG
742 : LLVMPassBuilderOptionsSetDebugLogging(options, 1);
743 : #endif
744 :
745 : LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
746 :
747 : err = LLVMRunPasses(module, passes, NULL, options);
748 :
749 : if (err)
750 : elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
751 :
752 : LLVMDisposePassBuilderOptions(options);
753 : #endif
754 1308 : }
755 :
756 : /*
757 : * Emit code for the currently pending module.
758 : */
759 : static void
760 1308 : llvm_compile_module(LLVMJitContext *context)
761 : {
762 : LLVMJitHandle *handle;
763 : MemoryContext oldcontext;
764 : instr_time starttime;
765 : instr_time endtime;
766 : #if LLVM_VERSION_MAJOR > 11
767 : LLVMOrcLLJITRef compile_orc;
768 : #else
769 : LLVMOrcJITStackRef compile_orc;
770 : #endif
771 :
772 1308 : if (context->base.flags & PGJIT_OPT3)
773 978 : compile_orc = llvm_opt3_orc;
774 : else
775 330 : compile_orc = llvm_opt0_orc;
776 :
777 : /* perform inlining */
778 1308 : if (context->base.flags & PGJIT_INLINE)
779 : {
780 978 : INSTR_TIME_SET_CURRENT(starttime);
781 978 : llvm_inline(context->module);
782 978 : INSTR_TIME_SET_CURRENT(endtime);
783 978 : INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
784 : endtime, starttime);
785 : }
786 :
787 1308 : if (jit_dump_bitcode)
788 : {
789 : char *filename;
790 :
791 0 : filename = psprintf("%d.%zu.bc",
792 : MyProcPid,
793 : context->module_generation);
794 0 : LLVMWriteBitcodeToFile(context->module, filename);
795 0 : pfree(filename);
796 : }
797 :
798 :
799 : /* optimize according to the chosen optimization settings */
800 1308 : INSTR_TIME_SET_CURRENT(starttime);
801 1308 : llvm_optimize_module(context, context->module);
802 1308 : INSTR_TIME_SET_CURRENT(endtime);
803 1308 : INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
804 : endtime, starttime);
805 :
806 1308 : if (jit_dump_bitcode)
807 : {
808 : char *filename;
809 :
810 0 : filename = psprintf("%d.%zu.optimized.bc",
811 : MyProcPid,
812 : context->module_generation);
813 0 : LLVMWriteBitcodeToFile(context->module, filename);
814 0 : pfree(filename);
815 : }
816 :
817 : handle = (LLVMJitHandle *)
818 1308 : MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
819 :
820 : /*
821 : * Emit the code. Note that this can, depending on the optimization
822 : * settings, take noticeable resources as code emission executes low-level
823 : * instruction combining/selection passes etc. Without optimization a
824 : * faster instruction selection mechanism is used.
825 : */
826 1308 : INSTR_TIME_SET_CURRENT(starttime);
827 : #if LLVM_VERSION_MAJOR > 11
828 : {
829 : LLVMOrcThreadSafeModuleRef ts_module;
830 : LLVMErrorRef error;
831 : LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
832 :
833 : ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
834 :
835 : handle->lljit = compile_orc;
836 : handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
837 :
838 : /*
839 : * NB: This doesn't actually emit code. That happens lazily the first
840 : * time a symbol defined in the module is requested. Due to that
841 : * llvm_get_function() also accounts for emission time.
842 : */
843 :
844 : context->module = NULL; /* will be owned by LLJIT */
845 : error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
846 : handle->resource_tracker,
847 : ts_module);
848 :
849 : if (error)
850 : elog(ERROR, "failed to JIT module: %s",
851 : llvm_error_message(error));
852 :
853 : /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
854 : }
855 : #elif LLVM_VERSION_MAJOR > 6
856 : {
857 1308 : handle->stack = compile_orc;
858 1308 : if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
859 : llvm_resolve_symbol, NULL))
860 0 : elog(ERROR, "failed to JIT module");
861 :
862 : /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
863 : }
864 : #elif LLVM_VERSION_MAJOR > 4
865 : {
866 : LLVMSharedModuleRef smod;
867 :
868 : smod = LLVMOrcMakeSharedModule(context->module);
869 : handle->stack = compile_orc;
870 : if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
871 : llvm_resolve_symbol, NULL))
872 : elog(ERROR, "failed to JIT module");
873 :
874 : LLVMOrcDisposeSharedModuleRef(smod);
875 : }
876 : #else /* LLVM 4.0 and 3.9 */
877 : {
878 : handle->stack = compile_orc;
879 : handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
880 : llvm_resolve_symbol, NULL);
881 :
882 : LLVMDisposeModule(context->module);
883 : }
884 : #endif
885 :
886 1308 : INSTR_TIME_SET_CURRENT(endtime);
887 1308 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
888 : endtime, starttime);
889 :
890 1308 : context->module = NULL;
891 1308 : context->compiled = true;
892 :
893 : /* remember emitted code for cleanup and lookups */
894 1308 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
895 1308 : context->handles = lappend(context->handles, handle);
896 1308 : MemoryContextSwitchTo(oldcontext);
897 :
898 1308 : ereport(DEBUG1,
899 : (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
900 : INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
901 : INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
902 : INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
903 : errhidestmt(true),
904 : errhidecontext(true)));
905 1308 : }
906 :
907 : /*
908 : * Per session initialization.
909 : */
910 : static void
911 1672 : llvm_session_initialize(void)
912 : {
913 : MemoryContext oldcontext;
914 1672 : char *error = NULL;
915 1672 : char *cpu = NULL;
916 1672 : char *features = NULL;
917 : LLVMTargetMachineRef opt0_tm;
918 : LLVMTargetMachineRef opt3_tm;
919 :
920 1672 : if (llvm_session_initialized)
921 902 : return;
922 :
923 770 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
924 :
925 770 : LLVMInitializeNativeTarget();
926 770 : LLVMInitializeNativeAsmPrinter();
927 770 : LLVMInitializeNativeAsmParser();
928 :
929 770 : if (llvm_context == NULL)
930 : {
931 770 : llvm_context = LLVMContextCreate();
932 :
933 770 : llvm_jit_context_in_use_count = 0;
934 770 : llvm_llvm_context_reuse_count = 0;
935 : }
936 :
937 : /*
938 : * When targeting LLVM 15, turn off opaque pointers for the context we
939 : * build our code in. We don't need to do so for other contexts (e.g.
940 : * llvm_ts_context). Once the IR is generated, it carries the necessary
941 : * information.
942 : *
943 : * For 16 and above, opaque pointers must be used, and we have special
944 : * code for that.
945 : */
946 : #if LLVM_VERSION_MAJOR == 15
947 : LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
948 : #endif
949 :
950 : /*
951 : * Synchronize types early, as that also includes inferring the target
952 : * triple.
953 : */
954 770 : llvm_create_types();
955 :
956 : /*
957 : * Extract target information from loaded module.
958 : */
959 770 : llvm_set_target();
960 :
961 770 : if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
962 : {
963 0 : elog(FATAL, "failed to query triple %s", error);
964 : }
965 :
966 : /*
967 : * We want the generated code to use all available features. Therefore
968 : * grab the host CPU string and detect features of the current CPU. The
969 : * latter is needed because some CPU architectures default to enabling
970 : * features not all CPUs have (weird, huh).
971 : */
972 770 : cpu = LLVMGetHostCPUName();
973 770 : features = LLVMGetHostCPUFeatures();
974 770 : elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
975 : cpu, features);
976 :
977 : opt0_tm =
978 770 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
979 : LLVMCodeGenLevelNone,
980 : LLVMRelocDefault,
981 : LLVMCodeModelJITDefault);
982 : opt3_tm =
983 770 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
984 : LLVMCodeGenLevelAggressive,
985 : LLVMRelocDefault,
986 : LLVMCodeModelJITDefault);
987 :
988 770 : LLVMDisposeMessage(cpu);
989 770 : cpu = NULL;
990 770 : LLVMDisposeMessage(features);
991 770 : features = NULL;
992 :
993 : /* force symbols in main binary to be loaded */
994 770 : LLVMLoadLibraryPermanently(NULL);
995 :
996 : #if LLVM_VERSION_MAJOR > 11
997 : {
998 : llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
999 :
1000 : llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
1001 : opt0_tm = 0;
1002 :
1003 : llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
1004 : opt3_tm = 0;
1005 : }
1006 : #else /* LLVM_VERSION_MAJOR > 11 */
1007 : {
1008 770 : llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
1009 770 : llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
1010 :
1011 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1012 770 : if (jit_debugging_support)
1013 : {
1014 0 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1015 :
1016 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
1017 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
1018 : }
1019 : #endif
1020 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1021 770 : if (jit_profiling_support)
1022 : {
1023 0 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1024 :
1025 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
1026 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
1027 : }
1028 : #endif
1029 : }
1030 : #endif /* LLVM_VERSION_MAJOR > 11 */
1031 :
1032 770 : on_proc_exit(llvm_shutdown, 0);
1033 :
1034 770 : llvm_session_initialized = true;
1035 :
1036 770 : MemoryContextSwitchTo(oldcontext);
1037 : }
1038 :
1039 : static void
1040 770 : llvm_shutdown(int code, Datum arg)
1041 : {
1042 : /*
1043 : * If llvm_shutdown() is reached while in a fatal-on-oom section an error
1044 : * has occurred in the middle of LLVM code. It is not safe to call back
1045 : * into LLVM (which is why a FATAL error was thrown).
1046 : *
1047 : * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
1048 : * profiling data won't be written out.
1049 : */
1050 770 : if (llvm_in_fatal_on_oom())
1051 : {
1052 : Assert(proc_exit_inprogress);
1053 0 : return;
1054 : }
1055 :
1056 770 : if (llvm_jit_context_in_use_count != 0)
1057 0 : elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
1058 : llvm_jit_context_in_use_count);
1059 :
1060 : #if LLVM_VERSION_MAJOR > 11
1061 : {
1062 : if (llvm_opt3_orc)
1063 : {
1064 : LLVMOrcDisposeLLJIT(llvm_opt3_orc);
1065 : llvm_opt3_orc = NULL;
1066 : }
1067 : if (llvm_opt0_orc)
1068 : {
1069 : LLVMOrcDisposeLLJIT(llvm_opt0_orc);
1070 : llvm_opt0_orc = NULL;
1071 : }
1072 : if (llvm_ts_context)
1073 : {
1074 : LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
1075 : llvm_ts_context = NULL;
1076 : }
1077 : }
1078 : #else /* LLVM_VERSION_MAJOR > 11 */
1079 : {
1080 : /* unregister profiling support, needs to be flushed to be useful */
1081 :
1082 770 : if (llvm_opt3_orc)
1083 : {
1084 : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1085 : if (jit_profiling_support)
1086 : LLVMOrcUnregisterPerf(llvm_opt3_orc);
1087 : #endif
1088 770 : LLVMOrcDisposeInstance(llvm_opt3_orc);
1089 770 : llvm_opt3_orc = NULL;
1090 : }
1091 :
1092 770 : if (llvm_opt0_orc)
1093 : {
1094 : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1095 : if (jit_profiling_support)
1096 : LLVMOrcUnregisterPerf(llvm_opt0_orc);
1097 : #endif
1098 770 : LLVMOrcDisposeInstance(llvm_opt0_orc);
1099 770 : llvm_opt0_orc = NULL;
1100 : }
1101 : }
1102 : #endif /* LLVM_VERSION_MAJOR > 11 */
1103 : }
1104 :
1105 : /* helper for llvm_create_types, returning a function's return type */
1106 : static LLVMTypeRef
1107 770 : load_return_type(LLVMModuleRef mod, const char *name)
1108 : {
1109 : LLVMValueRef value;
1110 : LLVMTypeRef typ;
1111 :
1112 : /* this'll return a *pointer* to the function */
1113 770 : value = LLVMGetNamedFunction(mod, name);
1114 770 : if (!value)
1115 0 : elog(ERROR, "function %s is unknown", name);
1116 :
1117 770 : typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
1118 :
1119 770 : return typ;
1120 : }
1121 :
1122 : /*
1123 : * Load triple & layout from clang emitted file so we're guaranteed to be
1124 : * compatible.
1125 : */
1126 : static void
1127 770 : llvm_set_target(void)
1128 : {
1129 770 : if (!llvm_types_module)
1130 0 : elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
1131 :
1132 770 : if (llvm_triple == NULL)
1133 770 : llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1134 :
1135 770 : if (llvm_layout == NULL)
1136 770 : llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1137 770 : }
1138 :
1139 : /*
1140 : * Load required information, types, function signatures from llvmjit_types.c
1141 : * and make them available in global variables.
1142 : *
1143 : * Those global variables are then used while emitting code.
1144 : */
1145 : static void
1146 770 : llvm_create_types(void)
1147 : {
1148 : char path[MAXPGPATH];
1149 : LLVMMemoryBufferRef buf;
1150 : char *msg;
1151 :
1152 770 : snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
1153 :
1154 : /* open file */
1155 770 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1156 : {
1157 0 : elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1158 : path, msg);
1159 : }
1160 :
1161 : /* eagerly load contents, going to need it all */
1162 770 : if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1163 : {
1164 0 : elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1165 : }
1166 770 : LLVMDisposeMemoryBuffer(buf);
1167 :
1168 770 : TypeSizeT = llvm_pg_var_type("TypeSizeT");
1169 770 : TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1170 770 : TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1171 770 : TypePGFunction = llvm_pg_var_type("TypePGFunction");
1172 770 : StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1173 770 : StructExprContext = llvm_pg_var_type("StructExprContext");
1174 770 : StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1175 770 : StructExprState = llvm_pg_var_type("StructExprState");
1176 770 : StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1177 770 : StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1178 770 : StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1179 770 : StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1180 770 : StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1181 770 : StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1182 770 : StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1183 770 : StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1184 770 : StructAggState = llvm_pg_var_type("StructAggState");
1185 770 : StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1186 770 : StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1187 770 : StructPlanState = llvm_pg_var_type("StructPlanState");
1188 770 : StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1189 :
1190 770 : AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1191 770 : ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1192 770 : ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1193 770 : }
1194 :
1195 : /*
1196 : * Split a symbol into module / function parts. If the function is in the
1197 : * main binary (or an external library) *modname will be NULL.
1198 : */
1199 : void
1200 33254 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1201 : {
1202 33254 : *modname = NULL;
1203 33254 : *funcname = NULL;
1204 :
1205 : /*
1206 : * Module function names are pgextern.$module.$funcname
1207 : */
1208 33254 : if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1209 : {
1210 : /*
1211 : * Symbol names cannot contain a ., therefore we can split based on
1212 : * first and last occurrence of one.
1213 : */
1214 110 : *funcname = rindex(name, '.');
1215 110 : (*funcname)++; /* jump over . */
1216 :
1217 220 : *modname = pnstrdup(name + strlen("pgextern."),
1218 110 : *funcname - name - strlen("pgextern.") - 1);
1219 : Assert(funcname);
1220 :
1221 110 : *funcname = pstrdup(*funcname);
1222 : }
1223 : else
1224 : {
1225 33144 : *modname = NULL;
1226 33144 : *funcname = pstrdup(name);
1227 : }
1228 33254 : }
1229 :
1230 : /*
1231 : * Attempt to resolve symbol, so LLVM can emit a reference to it.
1232 : */
1233 : static uint64_t
1234 5690 : llvm_resolve_symbol(const char *symname, void *ctx)
1235 : {
1236 : uintptr_t addr;
1237 : char *funcname;
1238 : char *modname;
1239 :
1240 : /*
1241 : * macOS prefixes all object level symbols with an underscore. But neither
1242 : * dlsym() nor PG's inliner expect that. So undo.
1243 : */
1244 : #if defined(__darwin__)
1245 : if (symname[0] != '_')
1246 : elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1247 : symname++;
1248 : #endif
1249 :
1250 5690 : llvm_split_symbol_name(symname, &modname, &funcname);
1251 :
1252 : /* functions that aren't resolved to names shouldn't ever get here */
1253 : Assert(funcname);
1254 :
1255 5690 : if (modname)
1256 38 : addr = (uintptr_t) load_external_function(modname, funcname,
1257 : true, NULL);
1258 : else
1259 5652 : addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1260 :
1261 5690 : pfree(funcname);
1262 5690 : if (modname)
1263 38 : pfree(modname);
1264 :
1265 : /* let LLVM will error out - should never happen */
1266 5690 : if (!addr)
1267 0 : elog(WARNING, "failed to resolve name %s", symname);
1268 :
1269 5690 : return (uint64_t) addr;
1270 : }
1271 :
1272 : #if LLVM_VERSION_MAJOR > 11
1273 :
1274 : static LLVMErrorRef
1275 : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1276 : LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1277 : LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1278 : LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1279 : {
1280 : #if LLVM_VERSION_MAJOR > 14
1281 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1282 : #else
1283 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1284 : #endif
1285 : LLVMErrorRef error;
1286 : LLVMOrcMaterializationUnitRef mu;
1287 :
1288 : for (int i = 0; i < LookupSetSize; i++)
1289 : {
1290 : const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1291 :
1292 : #if LLVM_VERSION_MAJOR > 12
1293 : LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1294 : #endif
1295 : symbols[i].Name = LookupSet[i].Name;
1296 : symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1297 : symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1298 : }
1299 :
1300 : mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1301 : error = LLVMOrcJITDylibDefine(JD, mu);
1302 : if (error != LLVMErrorSuccess)
1303 : LLVMOrcDisposeMaterializationUnit(mu);
1304 :
1305 : pfree(symbols);
1306 :
1307 : return error;
1308 : }
1309 :
1310 : /*
1311 : * We cannot throw errors through LLVM (without causing a FATAL at least), so
1312 : * just use WARNING here. That's OK anyway, as the error is also reported at
1313 : * the top level action (with less detail) and there might be multiple
1314 : * invocations of errors with details.
1315 : *
1316 : * This doesn't really happen during normal operation, but in cases like
1317 : * symbol resolution breakage. So just using elog(WARNING) is fine.
1318 : */
1319 : static void
1320 : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1321 : {
1322 : elog(WARNING, "error during JITing: %s",
1323 : llvm_error_message(error));
1324 : }
1325 :
1326 : /*
1327 : * Create our own object layer, so we can add event listeners.
1328 : */
1329 : static LLVMOrcObjectLayerRef
1330 : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1331 : {
1332 : LLVMOrcObjectLayerRef objlayer =
1333 : LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1334 :
1335 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1336 : if (jit_debugging_support)
1337 : {
1338 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1339 :
1340 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1341 : }
1342 : #endif
1343 :
1344 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1345 : if (jit_profiling_support)
1346 : {
1347 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1348 :
1349 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1350 : }
1351 : #endif
1352 :
1353 : return objlayer;
1354 : }
1355 :
1356 : /*
1357 : * Create LLJIT instance, using the passed in target machine. Note that the
1358 : * target machine afterwards is owned by the LLJIT instance.
1359 : */
1360 : static LLVMOrcLLJITRef
1361 : llvm_create_jit_instance(LLVMTargetMachineRef tm)
1362 : {
1363 : LLVMOrcLLJITRef lljit;
1364 : LLVMOrcJITTargetMachineBuilderRef tm_builder;
1365 : LLVMOrcLLJITBuilderRef lljit_builder;
1366 : LLVMErrorRef error;
1367 : LLVMOrcDefinitionGeneratorRef main_gen;
1368 : LLVMOrcDefinitionGeneratorRef ref_gen;
1369 :
1370 : lljit_builder = LLVMOrcCreateLLJITBuilder();
1371 : tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1372 : LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1373 :
1374 : LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1375 : llvm_create_object_layer,
1376 : NULL);
1377 :
1378 : error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1379 : if (error)
1380 : elog(ERROR, "failed to create lljit instance: %s",
1381 : llvm_error_message(error));
1382 :
1383 : LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1384 : llvm_log_jit_error, NULL);
1385 :
1386 : /*
1387 : * Symbol resolution support for symbols in the postgres binary /
1388 : * libraries already loaded.
1389 : */
1390 : error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1391 : LLVMOrcLLJITGetGlobalPrefix(lljit),
1392 : 0, NULL);
1393 : if (error)
1394 : elog(ERROR, "failed to create generator: %s",
1395 : llvm_error_message(error));
1396 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1397 :
1398 : /*
1399 : * Symbol resolution support for "special" functions, e.g. a call into an
1400 : * SQL callable function.
1401 : */
1402 : #if LLVM_VERSION_MAJOR > 14
1403 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1404 : #else
1405 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1406 : #endif
1407 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1408 :
1409 : return lljit;
1410 : }
1411 :
1412 : static char *
1413 : llvm_error_message(LLVMErrorRef error)
1414 : {
1415 : char *orig = LLVMGetErrorMessage(error);
1416 : char *msg = pstrdup(orig);
1417 :
1418 : LLVMDisposeErrorMessage(orig);
1419 :
1420 : return msg;
1421 : }
1422 :
1423 : #endif /* LLVM_VERSION_MAJOR > 11 */
1424 :
1425 : /*
1426 : * ResourceOwner callbacks
1427 : */
1428 : static void
1429 16 : ResOwnerReleaseJitContext(Datum res)
1430 : {
1431 16 : JitContext *context = (JitContext *) DatumGetPointer(res);
1432 :
1433 16 : context->resowner = NULL;
1434 16 : jit_release_context(context);
1435 16 : }
|
