Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit.c
4 : * Core part of the LLVM JIT provider.
5 : *
6 : * Copyright (c) 2016-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/jit/llvm/llvmjit.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 :
14 : #include "postgres.h"
15 :
16 : #include <llvm-c/Analysis.h>
17 : #include <llvm-c/BitReader.h>
18 : #include <llvm-c/BitWriter.h>
19 : #include <llvm-c/Core.h>
20 : #include <llvm-c/ExecutionEngine.h>
21 : #if LLVM_VERSION_MAJOR > 16
22 : #include <llvm-c/Transforms/PassBuilder.h>
23 : #endif
24 : #if LLVM_VERSION_MAJOR > 11
25 : #include <llvm-c/Orc.h>
26 : #include <llvm-c/OrcEE.h>
27 : #include <llvm-c/LLJIT.h>
28 : #else
29 : #include <llvm-c/OrcBindings.h>
30 : #endif
31 : #include <llvm-c/Support.h>
32 : #include <llvm-c/Target.h>
33 : #if LLVM_VERSION_MAJOR < 17
34 : #include <llvm-c/Transforms/IPO.h>
35 : #include <llvm-c/Transforms/PassManagerBuilder.h>
36 : #include <llvm-c/Transforms/Scalar.h>
37 : #include <llvm-c/Transforms/Utils.h>
38 : #endif
39 :
40 : #include "jit/llvmjit.h"
41 : #include "jit/llvmjit_emit.h"
42 : #include "miscadmin.h"
43 : #include "portability/instr_time.h"
44 : #include "storage/ipc.h"
45 : #include "utils/memutils.h"
46 : #include "utils/resowner.h"
47 :
48 : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
49 :
50 : /* Handle of a module emitted via ORC JIT */
51 : typedef struct LLVMJitHandle
52 : {
53 : #if LLVM_VERSION_MAJOR > 11
54 : LLVMOrcLLJITRef lljit;
55 : LLVMOrcResourceTrackerRef resource_tracker;
56 : #else
57 : LLVMOrcJITStackRef stack;
58 : LLVMOrcModuleHandle orc_handle;
59 : #endif
60 : } LLVMJitHandle;
61 :
62 :
63 : /* types & functions commonly needed for JITing */
64 : LLVMTypeRef TypeSizeT;
65 : LLVMTypeRef TypeParamBool;
66 : LLVMTypeRef TypeStorageBool;
67 : LLVMTypeRef TypePGFunction;
68 : LLVMTypeRef StructNullableDatum;
69 : LLVMTypeRef StructHeapTupleData;
70 : LLVMTypeRef StructMinimalTupleData;
71 : LLVMTypeRef StructTupleDescData;
72 : LLVMTypeRef StructTupleTableSlot;
73 : LLVMTypeRef StructHeapTupleHeaderData;
74 : LLVMTypeRef StructHeapTupleTableSlot;
75 : LLVMTypeRef StructMinimalTupleTableSlot;
76 : LLVMTypeRef StructMemoryContextData;
77 : LLVMTypeRef StructFunctionCallInfoData;
78 : LLVMTypeRef StructExprContext;
79 : LLVMTypeRef StructExprEvalStep;
80 : LLVMTypeRef StructExprState;
81 : LLVMTypeRef StructAggState;
82 : LLVMTypeRef StructAggStatePerGroupData;
83 : LLVMTypeRef StructAggStatePerTransData;
84 : LLVMTypeRef StructPlanState;
85 :
86 : LLVMValueRef AttributeTemplate;
87 : LLVMValueRef ExecEvalSubroutineTemplate;
88 : LLVMValueRef ExecEvalBoolSubroutineTemplate;
89 :
90 : static LLVMModuleRef llvm_types_module = NULL;
91 :
92 : static bool llvm_session_initialized = false;
93 : static size_t llvm_generation = 0;
94 :
95 : /* number of LLVMJitContexts that currently are in use */
96 : static size_t llvm_jit_context_in_use_count = 0;
97 :
98 : /* how many times has the current LLVMContextRef been used */
99 : static size_t llvm_llvm_context_reuse_count = 0;
100 : static const char *llvm_triple = NULL;
101 : static const char *llvm_layout = NULL;
102 : static LLVMContextRef llvm_context;
103 :
104 :
105 : static LLVMTargetRef llvm_targetref;
106 : #if LLVM_VERSION_MAJOR > 11
107 : static LLVMOrcThreadSafeContextRef llvm_ts_context;
108 : static LLVMOrcLLJITRef llvm_opt0_orc;
109 : static LLVMOrcLLJITRef llvm_opt3_orc;
110 : #else /* LLVM_VERSION_MAJOR > 11 */
111 : static LLVMOrcJITStackRef llvm_opt0_orc;
112 : static LLVMOrcJITStackRef llvm_opt3_orc;
113 : #endif /* LLVM_VERSION_MAJOR > 11 */
114 :
115 :
116 : static void llvm_release_context(JitContext *context);
117 : static void llvm_session_initialize(void);
118 : static void llvm_shutdown(int code, Datum arg);
119 : static void llvm_compile_module(LLVMJitContext *context);
120 : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
121 :
122 : static void llvm_create_types(void);
123 : static void llvm_set_target(void);
124 : static void llvm_recreate_llvm_context(void);
125 : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
126 :
127 : #if LLVM_VERSION_MAJOR > 11
128 : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
129 : static char *llvm_error_message(LLVMErrorRef error);
130 : #endif /* LLVM_VERSION_MAJOR > 11 */
131 :
132 : /* ResourceOwner callbacks to hold JitContexts */
133 : static void ResOwnerReleaseJitContext(Datum res);
134 :
135 : static const ResourceOwnerDesc jit_resowner_desc =
136 : {
137 : .name = "LLVM JIT context",
138 : .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
139 : .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
140 : .ReleaseResource = ResOwnerReleaseJitContext,
141 : .DebugPrint = NULL /* the default message is fine */
142 : };
143 :
144 : /* Convenience wrappers over ResourceOwnerRemember/Forget */
145 : static inline void
146 1800 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
147 : {
148 1800 : ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
149 1800 : }
150 : static inline void
151 1786 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
152 : {
153 1786 : ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
154 1786 : }
155 :
156 2016 : PG_MODULE_MAGIC;
157 :
158 :
159 : /*
160 : * Initialize LLVM JIT provider.
161 : */
162 : void
163 804 : _PG_jit_provider_init(JitProviderCallbacks *cb)
164 : {
165 804 : cb->reset_after_error = llvm_reset_after_error;
166 804 : cb->release_context = llvm_release_context;
167 804 : cb->compile_expr = llvm_compile_expr;
168 804 : }
169 :
170 :
171 : /*
172 : * Every now and then create a new LLVMContextRef. Unfortunately, during every
173 : * round of inlining, types may "leak" (they can still be found/used via the
174 : * context, but new types will be created the next time in inlining is
175 : * performed). To prevent that from slowly accumulating problematic amounts of
176 : * memory, recreate the LLVMContextRef we use. We don't want to do so too
177 : * often, as that implies some overhead (particularly re-loading the module
178 : * summaries / modules is fairly expensive). A future TODO would be to make
179 : * this more finegrained and only drop/recreate the LLVMContextRef when we know
180 : * there has been inlining. If we can get the size of the context from LLVM
181 : * then that might be a better way to determine when to drop/recreate rather
182 : * then the usagecount heuristic currently employed.
183 : */
184 : static void
185 1800 : llvm_recreate_llvm_context(void)
186 : {
187 1800 : if (!llvm_context)
188 0 : elog(ERROR, "Trying to recreate a non-existing context");
189 :
190 : /*
191 : * We can only safely recreate the LLVM context if no other code is being
192 : * JITed, otherwise we'd release the types in use for that.
193 : */
194 1800 : if (llvm_jit_context_in_use_count > 0)
195 : {
196 0 : llvm_llvm_context_reuse_count++;
197 0 : return;
198 : }
199 :
200 1800 : if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
201 : {
202 1800 : llvm_llvm_context_reuse_count++;
203 1800 : return;
204 : }
205 :
206 : /*
207 : * Need to reset the modules that the inlining code caches before
208 : * disposing of the context. LLVM modules exist within a specific LLVM
209 : * context, therefore disposing of the context before resetting the cache
210 : * would lead to dangling pointers to modules.
211 : */
212 0 : llvm_inline_reset_caches();
213 :
214 0 : LLVMContextDispose(llvm_context);
215 0 : llvm_context = LLVMContextCreate();
216 0 : llvm_llvm_context_reuse_count = 0;
217 :
218 : /*
219 : * Re-build cached type information, so code generation code can rely on
220 : * that information to be present (also prevents the variables to be
221 : * dangling references).
222 : */
223 0 : llvm_create_types();
224 : }
225 :
226 :
227 : /*
228 : * Create a context for JITing work.
229 : *
230 : * The context, including subsidiary resources, will be cleaned up either when
231 : * the context is explicitly released, or when the lifetime of
232 : * CurrentResourceOwner ends (usually the end of the current [sub]xact).
233 : */
234 : LLVMJitContext *
235 1800 : llvm_create_context(int jitFlags)
236 : {
237 : LLVMJitContext *context;
238 :
239 1800 : llvm_assert_in_fatal_section();
240 :
241 1800 : llvm_session_initialize();
242 :
243 1800 : llvm_recreate_llvm_context();
244 :
245 1800 : ResourceOwnerEnlarge(CurrentResourceOwner);
246 :
247 1800 : context = MemoryContextAllocZero(TopMemoryContext,
248 : sizeof(LLVMJitContext));
249 1800 : context->base.flags = jitFlags;
250 :
251 : /* ensure cleanup */
252 1800 : context->resowner = CurrentResourceOwner;
253 1800 : ResourceOwnerRememberJIT(CurrentResourceOwner, context);
254 :
255 1800 : llvm_jit_context_in_use_count++;
256 :
257 1800 : return context;
258 : }
259 :
260 : /*
261 : * Release resources required by one llvm context.
262 : */
263 : static void
264 1800 : llvm_release_context(JitContext *context)
265 : {
266 1800 : LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
267 : ListCell *lc;
268 :
269 : /*
270 : * Consider as cleaned up even if we skip doing so below, that way we can
271 : * verify the tracking is correct (see llvm_shutdown()).
272 : */
273 1800 : llvm_jit_context_in_use_count--;
274 :
275 : /*
276 : * When this backend is exiting, don't clean up LLVM. As an error might
277 : * have occurred from within LLVM, we do not want to risk reentering. All
278 : * resource cleanup is going to happen through process exit.
279 : */
280 1800 : if (proc_exit_inprogress)
281 0 : return;
282 :
283 1800 : llvm_enter_fatal_on_oom();
284 :
285 1800 : if (llvm_jit_context->module)
286 : {
287 604 : LLVMDisposeModule(llvm_jit_context->module);
288 604 : llvm_jit_context->module = NULL;
289 : }
290 :
291 3118 : foreach(lc, llvm_jit_context->handles)
292 : {
293 1318 : LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
294 :
295 : #if LLVM_VERSION_MAJOR > 11
296 : {
297 : LLVMOrcExecutionSessionRef ee;
298 : LLVMOrcSymbolStringPoolRef sp;
299 :
300 : LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
301 : LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
302 :
303 : /*
304 : * Without triggering cleanup of the string pool, we'd leak
305 : * memory. It'd be sufficient to do this far less often, but in
306 : * experiments the required time was small enough to just always
307 : * do it.
308 : */
309 : ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
310 : sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
311 : LLVMOrcSymbolStringPoolClearDeadEntries(sp);
312 : }
313 : #else /* LLVM_VERSION_MAJOR > 11 */
314 : {
315 1318 : LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
316 : }
317 : #endif /* LLVM_VERSION_MAJOR > 11 */
318 :
319 1318 : pfree(jit_handle);
320 : }
321 1800 : list_free(llvm_jit_context->handles);
322 1800 : llvm_jit_context->handles = NIL;
323 :
324 1800 : llvm_leave_fatal_on_oom();
325 :
326 1800 : if (llvm_jit_context->resowner)
327 1786 : ResourceOwnerForgetJIT(llvm_jit_context->resowner, llvm_jit_context);
328 : }
329 :
330 : /*
331 : * Return module which may be modified, e.g. by creating new functions.
332 : */
333 : LLVMModuleRef
334 18088 : llvm_mutable_module(LLVMJitContext *context)
335 : {
336 18088 : llvm_assert_in_fatal_section();
337 :
338 : /*
339 : * If there's no in-progress module, create a new one.
340 : */
341 18088 : if (!context->module)
342 : {
343 1922 : context->compiled = false;
344 1922 : context->module_generation = llvm_generation++;
345 1922 : context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
346 1922 : LLVMSetTarget(context->module, llvm_triple);
347 1922 : LLVMSetDataLayout(context->module, llvm_layout);
348 : }
349 :
350 18088 : return context->module;
351 : }
352 :
353 : /*
354 : * Expand function name to be non-conflicting. This should be used by code
355 : * generating code, when adding new externally visible function definitions to
356 : * a Module.
357 : */
358 : char *
359 18088 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
360 : {
361 : Assert(context->module != NULL);
362 :
363 18088 : context->base.instr.created_functions++;
364 :
365 : /*
366 : * Previously we used dots to separate, but turns out some tools, e.g.
367 : * GDB, don't like that and truncate name.
368 : */
369 36176 : return psprintf("%s_%zu_%d",
370 : basename,
371 : context->module_generation,
372 18088 : context->counter++);
373 : }
374 :
375 : /*
376 : * Return pointer to function funcname, which has to exist. If there's pending
377 : * code to be optimized and emitted, do so first.
378 : */
379 : void *
380 6272 : llvm_get_function(LLVMJitContext *context, const char *funcname)
381 : {
382 : ListCell *lc;
383 :
384 6272 : llvm_assert_in_fatal_section();
385 :
386 : /*
387 : * If there is a pending / not emitted module, compile and emit now.
388 : * Otherwise we might not find the [correct] function.
389 : */
390 6272 : if (!context->compiled)
391 : {
392 1318 : llvm_compile_module(context);
393 : }
394 :
395 : /*
396 : * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
397 : * to mangle here.
398 : */
399 :
400 : #if LLVM_VERSION_MAJOR > 11
401 : foreach(lc, context->handles)
402 : {
403 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
404 : instr_time starttime;
405 : instr_time endtime;
406 : LLVMErrorRef error;
407 : LLVMOrcJITTargetAddress addr;
408 :
409 : INSTR_TIME_SET_CURRENT(starttime);
410 :
411 : addr = 0;
412 : error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
413 : if (error)
414 : elog(ERROR, "failed to look up symbol \"%s\": %s",
415 : funcname, llvm_error_message(error));
416 :
417 : /*
418 : * LLJIT only actually emits code the first time a symbol is
419 : * referenced. Thus add lookup time to emission time. That's counting
420 : * a bit more than with older LLVM versions, but unlikely to ever
421 : * matter.
422 : */
423 : INSTR_TIME_SET_CURRENT(endtime);
424 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
425 : endtime, starttime);
426 :
427 : if (addr)
428 : return (void *) (uintptr_t) addr;
429 : }
430 : #else
431 6406 : foreach(lc, context->handles)
432 : {
433 : LLVMOrcTargetAddress addr;
434 6406 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
435 :
436 6406 : addr = 0;
437 6406 : if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
438 0 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
439 6406 : if (addr)
440 6272 : return (void *) (uintptr_t) addr;
441 : }
442 : #endif
443 :
444 0 : elog(ERROR, "failed to JIT: %s", funcname);
445 :
446 : return NULL;
447 : }
448 :
449 : /*
450 : * Return type of a variable in llvmjit_types.c. This is useful to keep types
451 : * in sync between plain C and JIT related code.
452 : */
453 : LLVMTypeRef
454 16198 : llvm_pg_var_type(const char *varname)
455 : {
456 : LLVMValueRef v_srcvar;
457 : LLVMTypeRef typ;
458 :
459 : /* this'll return a *pointer* to the global */
460 16198 : v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
461 16198 : if (!v_srcvar)
462 0 : elog(ERROR, "variable %s not in llvmjit_types.c", varname);
463 :
464 16198 : typ = LLVMGlobalGetValueType(v_srcvar);
465 :
466 16198 : return typ;
467 : }
468 :
469 : /*
470 : * Return function type of a variable in llvmjit_types.c. This is useful to
471 : * keep function types in sync between C and JITed code.
472 : */
473 : LLVMTypeRef
474 15780 : llvm_pg_var_func_type(const char *varname)
475 : {
476 : LLVMValueRef v_srcvar;
477 : LLVMTypeRef typ;
478 :
479 15780 : v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
480 15780 : if (!v_srcvar)
481 0 : elog(ERROR, "function %s not in llvmjit_types.c", varname);
482 :
483 15780 : typ = LLVMGetFunctionType(v_srcvar);
484 :
485 15780 : return typ;
486 : }
487 :
488 : /*
489 : * Return declaration for a function referenced in llvmjit_types.c, adding it
490 : * to the module if necessary.
491 : *
492 : * This is used to make functions discovered via llvm_create_types() known to
493 : * the module that's currently being worked on.
494 : */
495 : LLVMValueRef
496 14368 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
497 : {
498 : LLVMValueRef v_srcfn;
499 : LLVMValueRef v_fn;
500 :
501 : /* don't repeatedly add function */
502 14368 : v_fn = LLVMGetNamedFunction(mod, funcname);
503 14368 : if (v_fn)
504 10390 : return v_fn;
505 :
506 3978 : v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
507 :
508 3978 : if (!v_srcfn)
509 0 : elog(ERROR, "function %s not in llvmjit_types.c", funcname);
510 :
511 3978 : v_fn = LLVMAddFunction(mod,
512 : funcname,
513 : LLVMGetFunctionType(v_srcfn));
514 3978 : llvm_copy_attributes(v_srcfn, v_fn);
515 :
516 3978 : return v_fn;
517 : }
518 :
519 : /*
520 : * Copy attributes from one function to another, for a specific index (an
521 : * index can reference return value, function and parameter attributes).
522 : */
523 : static void
524 62118 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
525 : {
526 : int num_attributes;
527 : LLVMAttributeRef *attrs;
528 :
529 62118 : num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
530 :
531 62118 : if (num_attributes == 0)
532 21878 : return;
533 :
534 40240 : attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
535 40240 : LLVMGetAttributesAtIndex(v_from, index, attrs);
536 :
537 471888 : for (int attno = 0; attno < num_attributes; attno++)
538 431648 : LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
539 :
540 40240 : pfree(attrs);
541 : }
542 :
543 : /*
544 : * Copy all attributes from one function to another. I.e. function, return and
545 : * parameters will be copied.
546 : */
547 : void
548 22066 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
549 : {
550 : uint32 param_count;
551 :
552 : /* copy function attributes */
553 22066 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
554 :
555 22066 : if (LLVMGetTypeKind(LLVMGetFunctionReturnType(v_to)) != LLVMVoidTypeKind)
556 : {
557 : /* and the return value attributes */
558 12014 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
559 : }
560 :
561 : /* and each function parameter's attribute */
562 22066 : param_count = LLVMCountParams(v_from);
563 :
564 50104 : for (int paramidx = 1; paramidx <= param_count; paramidx++)
565 28038 : llvm_copy_attributes_at_index(v_from, v_to, paramidx);
566 22066 : }
567 :
568 : /*
569 : * Return a callable LLVMValueRef for fcinfo.
570 : */
571 : LLVMValueRef
572 5860 : llvm_function_reference(LLVMJitContext *context,
573 : LLVMBuilderRef builder,
574 : LLVMModuleRef mod,
575 : FunctionCallInfo fcinfo)
576 : {
577 : char *modname;
578 : char *basename;
579 : char *funcname;
580 :
581 : LLVMValueRef v_fn;
582 :
583 5860 : fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
584 :
585 5860 : if (modname != NULL && basename != NULL)
586 : {
587 : /* external function in loadable library */
588 20 : funcname = psprintf("pgextern.%s.%s", modname, basename);
589 : }
590 5840 : else if (basename != NULL)
591 : {
592 : /* internal function */
593 5828 : funcname = pstrdup(basename);
594 : }
595 : else
596 : {
597 : /*
598 : * Function we don't know to handle, return pointer. We do so by
599 : * creating a global constant containing a pointer to the function.
600 : * Makes IR more readable.
601 : */
602 : LLVMValueRef v_fn_addr;
603 :
604 12 : funcname = psprintf("pgoidextern.%u",
605 12 : fcinfo->flinfo->fn_oid);
606 12 : v_fn = LLVMGetNamedGlobal(mod, funcname);
607 12 : if (v_fn != 0)
608 0 : return l_load(builder, TypePGFunction, v_fn, "");
609 :
610 12 : v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
611 :
612 12 : v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
613 12 : LLVMSetInitializer(v_fn, v_fn_addr);
614 12 : LLVMSetGlobalConstant(v_fn, true);
615 12 : LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
616 12 : LLVMSetUnnamedAddr(v_fn, true);
617 :
618 12 : return l_load(builder, TypePGFunction, v_fn, "");
619 : }
620 :
621 : /* check if function already has been added */
622 5848 : v_fn = LLVMGetNamedFunction(mod, funcname);
623 5848 : if (v_fn != 0)
624 2516 : return v_fn;
625 :
626 3332 : v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
627 :
628 3332 : return v_fn;
629 : }
630 :
631 : /*
632 : * Optimize code in module using the flags set in context.
633 : */
634 : static void
635 1318 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
636 : {
637 : #if LLVM_VERSION_MAJOR < 17
638 : LLVMPassManagerBuilderRef llvm_pmb;
639 : LLVMPassManagerRef llvm_mpm;
640 : LLVMPassManagerRef llvm_fpm;
641 : LLVMValueRef func;
642 : int compile_optlevel;
643 :
644 1318 : if (context->base.flags & PGJIT_OPT3)
645 984 : compile_optlevel = 3;
646 : else
647 334 : compile_optlevel = 0;
648 :
649 : /*
650 : * Have to create a new pass manager builder every pass through, as the
651 : * inliner has some per-builder state. Otherwise one ends up only inlining
652 : * a function the first time though.
653 : */
654 1318 : llvm_pmb = LLVMPassManagerBuilderCreate();
655 1318 : LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
656 1318 : llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
657 :
658 1318 : if (context->base.flags & PGJIT_OPT3)
659 : {
660 : /* TODO: Unscientifically determined threshold */
661 984 : LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
662 : }
663 : else
664 : {
665 : /* we rely on mem2reg heavily, so emit even in the O0 case */
666 334 : LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
667 : }
668 :
669 1318 : LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
670 :
671 : /*
672 : * Do function level optimization. This could be moved to the point where
673 : * functions are emitted, to reduce memory usage a bit.
674 : */
675 1318 : LLVMInitializeFunctionPassManager(llvm_fpm);
676 32326 : for (func = LLVMGetFirstFunction(context->module);
677 : func != NULL;
678 31008 : func = LLVMGetNextFunction(func))
679 31008 : LLVMRunFunctionPassManager(llvm_fpm, func);
680 1318 : LLVMFinalizeFunctionPassManager(llvm_fpm);
681 1318 : LLVMDisposePassManager(llvm_fpm);
682 :
683 : /*
684 : * Perform module level optimization. We do so even in the non-optimized
685 : * case, so always-inline functions etc get inlined. It's cheap enough.
686 : */
687 1318 : llvm_mpm = LLVMCreatePassManager();
688 1318 : LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
689 : llvm_mpm);
690 : /* always use always-inliner pass */
691 1318 : if (!(context->base.flags & PGJIT_OPT3))
692 334 : LLVMAddAlwaysInlinerPass(llvm_mpm);
693 : /* if doing inlining, but no expensive optimization, add inlining pass */
694 1318 : if (context->base.flags & PGJIT_INLINE
695 984 : && !(context->base.flags & PGJIT_OPT3))
696 0 : LLVMAddFunctionInliningPass(llvm_mpm);
697 1318 : LLVMRunPassManager(llvm_mpm, context->module);
698 1318 : LLVMDisposePassManager(llvm_mpm);
699 :
700 1318 : LLVMPassManagerBuilderDispose(llvm_pmb);
701 : #else
702 : LLVMPassBuilderOptionsRef options;
703 : LLVMErrorRef err;
704 : const char *passes;
705 :
706 : if (context->base.flags & PGJIT_OPT3)
707 : passes = "default<O3>";
708 : else
709 : passes = "default<O0>,mem2reg";
710 :
711 : options = LLVMCreatePassBuilderOptions();
712 :
713 : #ifdef LLVM_PASS_DEBUG
714 : LLVMPassBuilderOptionsSetDebugLogging(options, 1);
715 : #endif
716 :
717 : /* In assertion builds, run the LLVM verify pass. */
718 : #ifdef USE_ASSERT_CHECKING
719 : LLVMPassBuilderOptionsSetVerifyEach(options, true);
720 : #endif
721 :
722 : LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
723 :
724 : err = LLVMRunPasses(module, passes, NULL, options);
725 :
726 : if (err)
727 : elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
728 :
729 : LLVMDisposePassBuilderOptions(options);
730 : #endif
731 1318 : }
732 :
733 : /*
734 : * Emit code for the currently pending module.
735 : */
736 : static void
737 1318 : llvm_compile_module(LLVMJitContext *context)
738 : {
739 : LLVMJitHandle *handle;
740 : MemoryContext oldcontext;
741 : instr_time starttime;
742 : instr_time endtime;
743 : #if LLVM_VERSION_MAJOR > 11
744 : LLVMOrcLLJITRef compile_orc;
745 : #else
746 : LLVMOrcJITStackRef compile_orc;
747 : #endif
748 :
749 1318 : if (context->base.flags & PGJIT_OPT3)
750 984 : compile_orc = llvm_opt3_orc;
751 : else
752 334 : compile_orc = llvm_opt0_orc;
753 :
754 : /* perform inlining */
755 1318 : if (context->base.flags & PGJIT_INLINE)
756 : {
757 984 : INSTR_TIME_SET_CURRENT(starttime);
758 984 : llvm_inline(context->module);
759 984 : INSTR_TIME_SET_CURRENT(endtime);
760 984 : INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
761 : endtime, starttime);
762 : }
763 :
764 1318 : if (jit_dump_bitcode)
765 : {
766 : char *filename;
767 :
768 0 : filename = psprintf("%d.%zu.bc",
769 : MyProcPid,
770 : context->module_generation);
771 0 : LLVMWriteBitcodeToFile(context->module, filename);
772 0 : pfree(filename);
773 : }
774 :
775 :
776 : /* optimize according to the chosen optimization settings */
777 1318 : INSTR_TIME_SET_CURRENT(starttime);
778 1318 : llvm_optimize_module(context, context->module);
779 1318 : INSTR_TIME_SET_CURRENT(endtime);
780 1318 : INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
781 : endtime, starttime);
782 :
783 1318 : if (jit_dump_bitcode)
784 : {
785 : char *filename;
786 :
787 0 : filename = psprintf("%d.%zu.optimized.bc",
788 : MyProcPid,
789 : context->module_generation);
790 0 : LLVMWriteBitcodeToFile(context->module, filename);
791 0 : pfree(filename);
792 : }
793 :
794 : handle = (LLVMJitHandle *)
795 1318 : MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
796 :
797 : /*
798 : * Emit the code. Note that this can, depending on the optimization
799 : * settings, take noticeable resources as code emission executes low-level
800 : * instruction combining/selection passes etc. Without optimization a
801 : * faster instruction selection mechanism is used.
802 : */
803 1318 : INSTR_TIME_SET_CURRENT(starttime);
804 : #if LLVM_VERSION_MAJOR > 11
805 : {
806 : LLVMOrcThreadSafeModuleRef ts_module;
807 : LLVMErrorRef error;
808 : LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
809 :
810 : ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
811 :
812 : handle->lljit = compile_orc;
813 : handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
814 :
815 : /*
816 : * NB: This doesn't actually emit code. That happens lazily the first
817 : * time a symbol defined in the module is requested. Due to that
818 : * llvm_get_function() also accounts for emission time.
819 : */
820 :
821 : context->module = NULL; /* will be owned by LLJIT */
822 : error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
823 : handle->resource_tracker,
824 : ts_module);
825 :
826 : if (error)
827 : elog(ERROR, "failed to JIT module: %s",
828 : llvm_error_message(error));
829 :
830 : /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
831 : }
832 : #else
833 : {
834 1318 : handle->stack = compile_orc;
835 1318 : if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
836 : llvm_resolve_symbol, NULL))
837 0 : elog(ERROR, "failed to JIT module");
838 :
839 : /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
840 : }
841 : #endif
842 :
843 1318 : INSTR_TIME_SET_CURRENT(endtime);
844 1318 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
845 : endtime, starttime);
846 :
847 1318 : context->module = NULL;
848 1318 : context->compiled = true;
849 :
850 : /* remember emitted code for cleanup and lookups */
851 1318 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
852 1318 : context->handles = lappend(context->handles, handle);
853 1318 : MemoryContextSwitchTo(oldcontext);
854 :
855 1318 : ereport(DEBUG1,
856 : (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
857 : INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
858 : INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
859 : INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
860 : errhidestmt(true),
861 : errhidecontext(true)));
862 1318 : }
863 :
864 : /*
865 : * Per session initialization.
866 : */
867 : static void
868 1800 : llvm_session_initialize(void)
869 : {
870 : MemoryContext oldcontext;
871 1800 : char *error = NULL;
872 1800 : char *cpu = NULL;
873 1800 : char *features = NULL;
874 : LLVMTargetMachineRef opt0_tm;
875 : LLVMTargetMachineRef opt3_tm;
876 :
877 1800 : if (llvm_session_initialized)
878 996 : return;
879 :
880 804 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
881 :
882 804 : LLVMInitializeNativeTarget();
883 804 : LLVMInitializeNativeAsmPrinter();
884 804 : LLVMInitializeNativeAsmParser();
885 :
886 804 : if (llvm_context == NULL)
887 : {
888 804 : llvm_context = LLVMContextCreate();
889 :
890 804 : llvm_jit_context_in_use_count = 0;
891 804 : llvm_llvm_context_reuse_count = 0;
892 : }
893 :
894 : /*
895 : * When targeting LLVM 15, turn off opaque pointers for the context we
896 : * build our code in. We don't need to do so for other contexts (e.g.
897 : * llvm_ts_context). Once the IR is generated, it carries the necessary
898 : * information.
899 : *
900 : * For 16 and above, opaque pointers must be used, and we have special
901 : * code for that.
902 : */
903 : #if LLVM_VERSION_MAJOR == 15
904 : LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
905 : #endif
906 :
907 : /*
908 : * Synchronize types early, as that also includes inferring the target
909 : * triple.
910 : */
911 804 : llvm_create_types();
912 :
913 : /*
914 : * Extract target information from loaded module.
915 : */
916 804 : llvm_set_target();
917 :
918 804 : if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
919 : {
920 0 : elog(FATAL, "failed to query triple %s", error);
921 : }
922 :
923 : /*
924 : * We want the generated code to use all available features. Therefore
925 : * grab the host CPU string and detect features of the current CPU. The
926 : * latter is needed because some CPU architectures default to enabling
927 : * features not all CPUs have (weird, huh).
928 : */
929 804 : cpu = LLVMGetHostCPUName();
930 804 : features = LLVMGetHostCPUFeatures();
931 804 : elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
932 : cpu, features);
933 :
934 : opt0_tm =
935 804 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
936 : LLVMCodeGenLevelNone,
937 : LLVMRelocDefault,
938 : LLVMCodeModelJITDefault);
939 : opt3_tm =
940 804 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
941 : LLVMCodeGenLevelAggressive,
942 : LLVMRelocDefault,
943 : LLVMCodeModelJITDefault);
944 :
945 804 : LLVMDisposeMessage(cpu);
946 804 : cpu = NULL;
947 804 : LLVMDisposeMessage(features);
948 804 : features = NULL;
949 :
950 : /* force symbols in main binary to be loaded */
951 804 : LLVMLoadLibraryPermanently(NULL);
952 :
953 : #if LLVM_VERSION_MAJOR > 11
954 : {
955 : llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
956 :
957 : llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
958 : opt0_tm = 0;
959 :
960 : llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
961 : opt3_tm = 0;
962 : }
963 : #else /* LLVM_VERSION_MAJOR > 11 */
964 : {
965 804 : llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
966 804 : llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
967 :
968 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
969 804 : if (jit_debugging_support)
970 : {
971 0 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
972 :
973 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
974 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
975 : }
976 : #endif
977 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
978 804 : if (jit_profiling_support)
979 : {
980 0 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
981 :
982 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
983 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
984 : }
985 : #endif
986 : }
987 : #endif /* LLVM_VERSION_MAJOR > 11 */
988 :
989 804 : on_proc_exit(llvm_shutdown, 0);
990 :
991 804 : llvm_session_initialized = true;
992 :
993 804 : MemoryContextSwitchTo(oldcontext);
994 : }
995 :
996 : static void
997 804 : llvm_shutdown(int code, Datum arg)
998 : {
999 : /*
1000 : * If llvm_shutdown() is reached while in a fatal-on-oom section an error
1001 : * has occurred in the middle of LLVM code. It is not safe to call back
1002 : * into LLVM (which is why a FATAL error was thrown).
1003 : *
1004 : * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
1005 : * profiling data won't be written out.
1006 : */
1007 804 : if (llvm_in_fatal_on_oom())
1008 : {
1009 : Assert(proc_exit_inprogress);
1010 0 : return;
1011 : }
1012 :
1013 804 : if (llvm_jit_context_in_use_count != 0)
1014 0 : elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
1015 : llvm_jit_context_in_use_count);
1016 :
1017 : #if LLVM_VERSION_MAJOR > 11
1018 : {
1019 : if (llvm_opt3_orc)
1020 : {
1021 : LLVMOrcDisposeLLJIT(llvm_opt3_orc);
1022 : llvm_opt3_orc = NULL;
1023 : }
1024 : if (llvm_opt0_orc)
1025 : {
1026 : LLVMOrcDisposeLLJIT(llvm_opt0_orc);
1027 : llvm_opt0_orc = NULL;
1028 : }
1029 : if (llvm_ts_context)
1030 : {
1031 : LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
1032 : llvm_ts_context = NULL;
1033 : }
1034 : }
1035 : #else /* LLVM_VERSION_MAJOR > 11 */
1036 : {
1037 : /* unregister profiling support, needs to be flushed to be useful */
1038 :
1039 804 : if (llvm_opt3_orc)
1040 : {
1041 804 : LLVMOrcDisposeInstance(llvm_opt3_orc);
1042 804 : llvm_opt3_orc = NULL;
1043 : }
1044 :
1045 804 : if (llvm_opt0_orc)
1046 : {
1047 804 : LLVMOrcDisposeInstance(llvm_opt0_orc);
1048 804 : llvm_opt0_orc = NULL;
1049 : }
1050 : }
1051 : #endif /* LLVM_VERSION_MAJOR > 11 */
1052 : }
1053 :
1054 : /* helper for llvm_create_types, returning a function's return type */
1055 : static LLVMTypeRef
1056 804 : load_return_type(LLVMModuleRef mod, const char *name)
1057 : {
1058 : LLVMValueRef value;
1059 : LLVMTypeRef typ;
1060 :
1061 : /* this'll return a *pointer* to the function */
1062 804 : value = LLVMGetNamedFunction(mod, name);
1063 804 : if (!value)
1064 0 : elog(ERROR, "function %s is unknown", name);
1065 :
1066 804 : typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
1067 :
1068 804 : return typ;
1069 : }
1070 :
1071 : /*
1072 : * Load triple & layout from clang emitted file so we're guaranteed to be
1073 : * compatible.
1074 : */
1075 : static void
1076 804 : llvm_set_target(void)
1077 : {
1078 804 : if (!llvm_types_module)
1079 0 : elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
1080 :
1081 804 : if (llvm_triple == NULL)
1082 804 : llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1083 :
1084 804 : if (llvm_layout == NULL)
1085 804 : llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1086 804 : }
1087 :
1088 : /*
1089 : * Load required information, types, function signatures from llvmjit_types.c
1090 : * and make them available in global variables.
1091 : *
1092 : * Those global variables are then used while emitting code.
1093 : */
1094 : static void
1095 804 : llvm_create_types(void)
1096 : {
1097 : char path[MAXPGPATH];
1098 : LLVMMemoryBufferRef buf;
1099 : char *msg;
1100 :
1101 804 : snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
1102 :
1103 : /* open file */
1104 804 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1105 : {
1106 0 : elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1107 : path, msg);
1108 : }
1109 :
1110 : /* eagerly load contents, going to need it all */
1111 804 : if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1112 : {
1113 0 : elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1114 : }
1115 804 : LLVMDisposeMemoryBuffer(buf);
1116 :
1117 804 : TypeSizeT = llvm_pg_var_type("TypeSizeT");
1118 804 : TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1119 804 : TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1120 804 : TypePGFunction = llvm_pg_var_type("TypePGFunction");
1121 804 : StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1122 804 : StructExprContext = llvm_pg_var_type("StructExprContext");
1123 804 : StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1124 804 : StructExprState = llvm_pg_var_type("StructExprState");
1125 804 : StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1126 804 : StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1127 804 : StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1128 804 : StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1129 804 : StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1130 804 : StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1131 804 : StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1132 804 : StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1133 804 : StructAggState = llvm_pg_var_type("StructAggState");
1134 804 : StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1135 804 : StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1136 804 : StructPlanState = llvm_pg_var_type("StructPlanState");
1137 804 : StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1138 :
1139 804 : AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1140 804 : ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1141 804 : ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1142 804 : }
1143 :
1144 : /*
1145 : * Split a symbol into module / function parts. If the function is in the
1146 : * main binary (or an external library) *modname will be NULL.
1147 : */
1148 : void
1149 34338 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1150 : {
1151 34338 : *modname = NULL;
1152 34338 : *funcname = NULL;
1153 :
1154 : /*
1155 : * Module function names are pgextern.$module.$funcname
1156 : */
1157 34338 : if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1158 : {
1159 : /*
1160 : * Symbol names cannot contain a ., therefore we can split based on
1161 : * first and last occurrence of one.
1162 : */
1163 58 : *funcname = rindex(name, '.');
1164 58 : (*funcname)++; /* jump over . */
1165 :
1166 116 : *modname = pnstrdup(name + strlen("pgextern."),
1167 58 : *funcname - name - strlen("pgextern.") - 1);
1168 : Assert(funcname);
1169 :
1170 58 : *funcname = pstrdup(*funcname);
1171 : }
1172 : else
1173 : {
1174 34280 : *modname = NULL;
1175 34280 : *funcname = pstrdup(name);
1176 : }
1177 34338 : }
1178 :
1179 : /*
1180 : * Attempt to resolve symbol, so LLVM can emit a reference to it.
1181 : */
1182 : static uint64_t
1183 5200 : llvm_resolve_symbol(const char *symname, void *ctx)
1184 : {
1185 : uintptr_t addr;
1186 : char *funcname;
1187 : char *modname;
1188 :
1189 : /*
1190 : * macOS prefixes all object level symbols with an underscore. But neither
1191 : * dlsym() nor PG's inliner expect that. So undo.
1192 : */
1193 : #if defined(__darwin__)
1194 : if (symname[0] != '_')
1195 : elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1196 : symname++;
1197 : #endif
1198 :
1199 5200 : llvm_split_symbol_name(symname, &modname, &funcname);
1200 :
1201 : /* functions that aren't resolved to names shouldn't ever get here */
1202 : Assert(funcname);
1203 :
1204 5200 : if (modname)
1205 20 : addr = (uintptr_t) load_external_function(modname, funcname,
1206 : true, NULL);
1207 : else
1208 5180 : addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1209 :
1210 5200 : pfree(funcname);
1211 5200 : if (modname)
1212 20 : pfree(modname);
1213 :
1214 : /* let LLVM will error out - should never happen */
1215 5200 : if (!addr)
1216 0 : elog(WARNING, "failed to resolve name %s", symname);
1217 :
1218 5200 : return (uint64_t) addr;
1219 : }
1220 :
1221 : #if LLVM_VERSION_MAJOR > 11
1222 :
1223 : static LLVMErrorRef
1224 : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1225 : LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1226 : LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1227 : LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1228 : {
1229 : #if LLVM_VERSION_MAJOR > 14
1230 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1231 : #else
1232 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1233 : #endif
1234 : LLVMErrorRef error;
1235 : LLVMOrcMaterializationUnitRef mu;
1236 :
1237 : for (int i = 0; i < LookupSetSize; i++)
1238 : {
1239 : const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1240 :
1241 : #if LLVM_VERSION_MAJOR > 12
1242 : LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1243 : #endif
1244 : symbols[i].Name = LookupSet[i].Name;
1245 : symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1246 : symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1247 : }
1248 :
1249 : mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1250 : error = LLVMOrcJITDylibDefine(JD, mu);
1251 : if (error != LLVMErrorSuccess)
1252 : LLVMOrcDisposeMaterializationUnit(mu);
1253 :
1254 : pfree(symbols);
1255 :
1256 : return error;
1257 : }
1258 :
1259 : /*
1260 : * We cannot throw errors through LLVM (without causing a FATAL at least), so
1261 : * just use WARNING here. That's OK anyway, as the error is also reported at
1262 : * the top level action (with less detail) and there might be multiple
1263 : * invocations of errors with details.
1264 : *
1265 : * This doesn't really happen during normal operation, but in cases like
1266 : * symbol resolution breakage. So just using elog(WARNING) is fine.
1267 : */
1268 : static void
1269 : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1270 : {
1271 : elog(WARNING, "error during JITing: %s",
1272 : llvm_error_message(error));
1273 : }
1274 :
1275 : /*
1276 : * Create our own object layer, so we can add event listeners.
1277 : */
1278 : static LLVMOrcObjectLayerRef
1279 : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1280 : {
1281 : LLVMOrcObjectLayerRef objlayer =
1282 : LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1283 :
1284 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1285 : if (jit_debugging_support)
1286 : {
1287 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1288 :
1289 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1290 : }
1291 : #endif
1292 :
1293 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1294 : if (jit_profiling_support)
1295 : {
1296 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1297 :
1298 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1299 : }
1300 : #endif
1301 :
1302 : return objlayer;
1303 : }
1304 :
1305 : /*
1306 : * Create LLJIT instance, using the passed in target machine. Note that the
1307 : * target machine afterwards is owned by the LLJIT instance.
1308 : */
1309 : static LLVMOrcLLJITRef
1310 : llvm_create_jit_instance(LLVMTargetMachineRef tm)
1311 : {
1312 : LLVMOrcLLJITRef lljit;
1313 : LLVMOrcJITTargetMachineBuilderRef tm_builder;
1314 : LLVMOrcLLJITBuilderRef lljit_builder;
1315 : LLVMErrorRef error;
1316 : LLVMOrcDefinitionGeneratorRef main_gen;
1317 : LLVMOrcDefinitionGeneratorRef ref_gen;
1318 :
1319 : lljit_builder = LLVMOrcCreateLLJITBuilder();
1320 : tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1321 : LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1322 :
1323 : LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1324 : llvm_create_object_layer,
1325 : NULL);
1326 :
1327 : error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1328 : if (error)
1329 : elog(ERROR, "failed to create lljit instance: %s",
1330 : llvm_error_message(error));
1331 :
1332 : LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1333 : llvm_log_jit_error, NULL);
1334 :
1335 : /*
1336 : * Symbol resolution support for symbols in the postgres binary /
1337 : * libraries already loaded.
1338 : */
1339 : error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1340 : LLVMOrcLLJITGetGlobalPrefix(lljit),
1341 : 0, NULL);
1342 : if (error)
1343 : elog(ERROR, "failed to create generator: %s",
1344 : llvm_error_message(error));
1345 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1346 :
1347 : /*
1348 : * Symbol resolution support for "special" functions, e.g. a call into an
1349 : * SQL callable function.
1350 : */
1351 : #if LLVM_VERSION_MAJOR > 14
1352 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1353 : #else
1354 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1355 : #endif
1356 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1357 :
1358 : return lljit;
1359 : }
1360 :
1361 : static char *
1362 : llvm_error_message(LLVMErrorRef error)
1363 : {
1364 : char *orig = LLVMGetErrorMessage(error);
1365 : char *msg = pstrdup(orig);
1366 :
1367 : LLVMDisposeErrorMessage(orig);
1368 :
1369 : return msg;
1370 : }
1371 :
1372 : #endif /* LLVM_VERSION_MAJOR > 11 */
1373 :
1374 : /*
1375 : * ResourceOwner callbacks
1376 : */
1377 : static void
1378 14 : ResOwnerReleaseJitContext(Datum res)
1379 : {
1380 14 : LLVMJitContext *context = (LLVMJitContext *) DatumGetPointer(res);
1381 :
1382 14 : context->resowner = NULL;
1383 14 : jit_release_context(&context->base);
1384 14 : }
|
