Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit.c
4 : * Core part of the LLVM JIT provider.
5 : *
6 : * Copyright (c) 2016-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/jit/llvm/llvmjit.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 :
14 : #include "postgres.h"
15 :
16 : #include <llvm-c/Analysis.h>
17 : #include <llvm-c/BitReader.h>
18 : #include <llvm-c/BitWriter.h>
19 : #include <llvm-c/Core.h>
20 : #include <llvm-c/ExecutionEngine.h>
21 : #if LLVM_VERSION_MAJOR > 16
22 : #include <llvm-c/Transforms/PassBuilder.h>
23 : #endif
24 : #if LLVM_VERSION_MAJOR > 11
25 : #include <llvm-c/Orc.h>
26 : #include <llvm-c/OrcEE.h>
27 : #include <llvm-c/LLJIT.h>
28 : #else
29 : #include <llvm-c/OrcBindings.h>
30 : #endif
31 : #include <llvm-c/Support.h>
32 : #include <llvm-c/Target.h>
33 : #if LLVM_VERSION_MAJOR < 17
34 : #include <llvm-c/Transforms/IPO.h>
35 : #include <llvm-c/Transforms/PassManagerBuilder.h>
36 : #include <llvm-c/Transforms/Scalar.h>
37 : #include <llvm-c/Transforms/Utils.h>
38 : #endif
39 :
40 : #include "jit/llvmjit.h"
41 : #include "jit/llvmjit_emit.h"
42 : #include "miscadmin.h"
43 : #include "portability/instr_time.h"
44 : #include "storage/ipc.h"
45 : #include "utils/memutils.h"
46 : #include "utils/resowner.h"
47 :
48 : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
49 :
50 : /* Handle of a module emitted via ORC JIT */
51 : typedef struct LLVMJitHandle
52 : {
53 : #if LLVM_VERSION_MAJOR > 11
54 : LLVMOrcLLJITRef lljit;
55 : LLVMOrcResourceTrackerRef resource_tracker;
56 : #else
57 : LLVMOrcJITStackRef stack;
58 : LLVMOrcModuleHandle orc_handle;
59 : #endif
60 : } LLVMJitHandle;
61 :
62 :
63 : /* types & functions commonly needed for JITing */
64 : LLVMTypeRef TypeSizeT;
65 : LLVMTypeRef TypeParamBool;
66 : LLVMTypeRef TypeStorageBool;
67 : LLVMTypeRef TypePGFunction;
68 : LLVMTypeRef StructNullableDatum;
69 : LLVMTypeRef StructHeapTupleData;
70 : LLVMTypeRef StructMinimalTupleData;
71 : LLVMTypeRef StructTupleDescData;
72 : LLVMTypeRef StructTupleTableSlot;
73 : LLVMTypeRef StructHeapTupleHeaderData;
74 : LLVMTypeRef StructHeapTupleTableSlot;
75 : LLVMTypeRef StructMinimalTupleTableSlot;
76 : LLVMTypeRef StructMemoryContextData;
77 : LLVMTypeRef StructFunctionCallInfoData;
78 : LLVMTypeRef StructExprContext;
79 : LLVMTypeRef StructExprEvalStep;
80 : LLVMTypeRef StructExprState;
81 : LLVMTypeRef StructAggState;
82 : LLVMTypeRef StructAggStatePerGroupData;
83 : LLVMTypeRef StructAggStatePerTransData;
84 : LLVMTypeRef StructPlanState;
85 :
86 : LLVMValueRef AttributeTemplate;
87 : LLVMValueRef ExecEvalSubroutineTemplate;
88 : LLVMValueRef ExecEvalBoolSubroutineTemplate;
89 :
90 : static LLVMModuleRef llvm_types_module = NULL;
91 :
92 : static bool llvm_session_initialized = false;
93 : static size_t llvm_generation = 0;
94 :
95 : /* number of LLVMJitContexts that currently are in use */
96 : static size_t llvm_jit_context_in_use_count = 0;
97 :
98 : /* how many times has the current LLVMContextRef been used */
99 : static size_t llvm_llvm_context_reuse_count = 0;
100 : static const char *llvm_triple = NULL;
101 : static const char *llvm_layout = NULL;
102 : static LLVMContextRef llvm_context;
103 :
104 :
105 : static LLVMTargetRef llvm_targetref;
106 : #if LLVM_VERSION_MAJOR > 11
107 : static LLVMOrcThreadSafeContextRef llvm_ts_context;
108 : static LLVMOrcLLJITRef llvm_opt0_orc;
109 : static LLVMOrcLLJITRef llvm_opt3_orc;
110 : #else /* LLVM_VERSION_MAJOR > 11 */
111 : static LLVMOrcJITStackRef llvm_opt0_orc;
112 : static LLVMOrcJITStackRef llvm_opt3_orc;
113 : #endif /* LLVM_VERSION_MAJOR > 11 */
114 :
115 :
116 : static void llvm_release_context(JitContext *context);
117 : static void llvm_session_initialize(void);
118 : static void llvm_shutdown(int code, Datum arg);
119 : static void llvm_compile_module(LLVMJitContext *context);
120 : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
121 :
122 : static void llvm_create_types(void);
123 : static void llvm_set_target(void);
124 : static void llvm_recreate_llvm_context(void);
125 : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
126 :
127 : #if LLVM_VERSION_MAJOR > 11
128 : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
129 : static char *llvm_error_message(LLVMErrorRef error);
130 : #endif /* LLVM_VERSION_MAJOR > 11 */
131 :
132 : /* ResourceOwner callbacks to hold JitContexts */
133 : static void ResOwnerReleaseJitContext(Datum res);
134 :
135 : static const ResourceOwnerDesc jit_resowner_desc =
136 : {
137 : .name = "LLVM JIT context",
138 : .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
139 : .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
140 : .ReleaseResource = ResOwnerReleaseJitContext,
141 : .DebugPrint = NULL /* the default message is fine */
142 : };
143 :
144 : /* Convenience wrappers over ResourceOwnerRemember/Forget */
145 : static inline void
146 1768 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
147 : {
148 1768 : ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
149 1768 : }
150 : static inline void
151 1752 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
152 : {
153 1752 : ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
154 1752 : }
155 :
156 2010 : PG_MODULE_MAGIC;
157 :
158 :
159 : /*
160 : * Initialize LLVM JIT provider.
161 : */
162 : void
163 798 : _PG_jit_provider_init(JitProviderCallbacks *cb)
164 : {
165 798 : cb->reset_after_error = llvm_reset_after_error;
166 798 : cb->release_context = llvm_release_context;
167 798 : cb->compile_expr = llvm_compile_expr;
168 798 : }
169 :
170 :
171 : /*
172 : * Every now and then create a new LLVMContextRef. Unfortunately, during every
173 : * round of inlining, types may "leak" (they can still be found/used via the
174 : * context, but new types will be created the next time in inlining is
175 : * performed). To prevent that from slowly accumulating problematic amounts of
176 : * memory, recreate the LLVMContextRef we use. We don't want to do so too
177 : * often, as that implies some overhead (particularly re-loading the module
178 : * summaries / modules is fairly expensive). A future TODO would be to make
179 : * this more finegrained and only drop/recreate the LLVMContextRef when we know
180 : * there has been inlining. If we can get the size of the context from LLVM
181 : * then that might be a better way to determine when to drop/recreate rather
182 : * then the usagecount heuristic currently employed.
183 : */
184 : static void
185 1768 : llvm_recreate_llvm_context(void)
186 : {
187 1768 : if (!llvm_context)
188 0 : elog(ERROR, "Trying to recreate a non-existing context");
189 :
190 : /*
191 : * We can only safely recreate the LLVM context if no other code is being
192 : * JITed, otherwise we'd release the types in use for that.
193 : */
194 1768 : if (llvm_jit_context_in_use_count > 0)
195 : {
196 0 : llvm_llvm_context_reuse_count++;
197 0 : return;
198 : }
199 :
200 1768 : if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
201 : {
202 1768 : llvm_llvm_context_reuse_count++;
203 1768 : return;
204 : }
205 :
206 : /*
207 : * Need to reset the modules that the inlining code caches before
208 : * disposing of the context. LLVM modules exist within a specific LLVM
209 : * context, therefore disposing of the context before resetting the cache
210 : * would lead to dangling pointers to modules.
211 : */
212 0 : llvm_inline_reset_caches();
213 :
214 0 : LLVMContextDispose(llvm_context);
215 0 : llvm_context = LLVMContextCreate();
216 0 : llvm_llvm_context_reuse_count = 0;
217 :
218 : /*
219 : * Re-build cached type information, so code generation code can rely on
220 : * that information to be present (also prevents the variables to be
221 : * dangling references).
222 : */
223 0 : llvm_create_types();
224 : }
225 :
226 :
227 : /*
228 : * Create a context for JITing work.
229 : *
230 : * The context, including subsidiary resources, will be cleaned up either when
231 : * the context is explicitly released, or when the lifetime of
232 : * CurrentResourceOwner ends (usually the end of the current [sub]xact).
233 : */
234 : LLVMJitContext *
235 1768 : llvm_create_context(int jitFlags)
236 : {
237 : LLVMJitContext *context;
238 :
239 1768 : llvm_assert_in_fatal_section();
240 :
241 1768 : llvm_session_initialize();
242 :
243 1768 : llvm_recreate_llvm_context();
244 :
245 1768 : ResourceOwnerEnlarge(CurrentResourceOwner);
246 :
247 1768 : context = MemoryContextAllocZero(TopMemoryContext,
248 : sizeof(LLVMJitContext));
249 1768 : context->base.flags = jitFlags;
250 :
251 : /* ensure cleanup */
252 1768 : context->base.resowner = CurrentResourceOwner;
253 1768 : ResourceOwnerRememberJIT(CurrentResourceOwner, context);
254 :
255 1768 : llvm_jit_context_in_use_count++;
256 :
257 1768 : return context;
258 : }
259 :
260 : /*
261 : * Release resources required by one llvm context.
262 : */
263 : static void
264 1768 : llvm_release_context(JitContext *context)
265 : {
266 1768 : LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
267 : ListCell *lc;
268 :
269 : /*
270 : * Consider as cleaned up even if we skip doing so below, that way we can
271 : * verify the tracking is correct (see llvm_shutdown()).
272 : */
273 1768 : llvm_jit_context_in_use_count--;
274 :
275 : /*
276 : * When this backend is exiting, don't clean up LLVM. As an error might
277 : * have occurred from within LLVM, we do not want to risk reentering. All
278 : * resource cleanup is going to happen through process exit.
279 : */
280 1768 : if (proc_exit_inprogress)
281 0 : return;
282 :
283 1768 : llvm_enter_fatal_on_oom();
284 :
285 1768 : if (llvm_jit_context->module)
286 : {
287 576 : LLVMDisposeModule(llvm_jit_context->module);
288 576 : llvm_jit_context->module = NULL;
289 : }
290 :
291 3082 : foreach(lc, llvm_jit_context->handles)
292 : {
293 1314 : LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
294 :
295 : #if LLVM_VERSION_MAJOR > 11
296 : {
297 : LLVMOrcExecutionSessionRef ee;
298 : LLVMOrcSymbolStringPoolRef sp;
299 :
300 : LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
301 : LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
302 :
303 : /*
304 : * Without triggering cleanup of the string pool, we'd leak
305 : * memory. It'd be sufficient to do this far less often, but in
306 : * experiments the required time was small enough to just always
307 : * do it.
308 : */
309 : ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
310 : sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
311 : LLVMOrcSymbolStringPoolClearDeadEntries(sp);
312 : }
313 : #else /* LLVM_VERSION_MAJOR > 11 */
314 : {
315 1314 : LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
316 : }
317 : #endif /* LLVM_VERSION_MAJOR > 11 */
318 :
319 1314 : pfree(jit_handle);
320 : }
321 1768 : list_free(llvm_jit_context->handles);
322 1768 : llvm_jit_context->handles = NIL;
323 :
324 1768 : llvm_leave_fatal_on_oom();
325 :
326 1768 : if (context->resowner)
327 1752 : ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
328 : }
329 :
330 : /*
331 : * Return module which may be modified, e.g. by creating new functions.
332 : */
333 : LLVMModuleRef
334 17828 : llvm_mutable_module(LLVMJitContext *context)
335 : {
336 17828 : llvm_assert_in_fatal_section();
337 :
338 : /*
339 : * If there's no in-progress module, create a new one.
340 : */
341 17828 : if (!context->module)
342 : {
343 1890 : context->compiled = false;
344 1890 : context->module_generation = llvm_generation++;
345 1890 : context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
346 1890 : LLVMSetTarget(context->module, llvm_triple);
347 1890 : LLVMSetDataLayout(context->module, llvm_layout);
348 : }
349 :
350 17828 : return context->module;
351 : }
352 :
353 : /*
354 : * Expand function name to be non-conflicting. This should be used by code
355 : * generating code, when adding new externally visible function definitions to
356 : * a Module.
357 : */
358 : char *
359 17828 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
360 : {
361 : Assert(context->module != NULL);
362 :
363 17828 : context->base.instr.created_functions++;
364 :
365 : /*
366 : * Previously we used dots to separate, but turns out some tools, e.g.
367 : * GDB, don't like that and truncate name.
368 : */
369 35656 : return psprintf("%s_%zu_%d",
370 : basename,
371 : context->module_generation,
372 17828 : context->counter++);
373 : }
374 :
375 : /*
376 : * Return pointer to function funcname, which has to exist. If there's pending
377 : * code to be optimized and emitted, do so first.
378 : */
379 : void *
380 6264 : llvm_get_function(LLVMJitContext *context, const char *funcname)
381 : {
382 : ListCell *lc;
383 :
384 6264 : llvm_assert_in_fatal_section();
385 :
386 : /*
387 : * If there is a pending / not emitted module, compile and emit now.
388 : * Otherwise we might not find the [correct] function.
389 : */
390 6264 : if (!context->compiled)
391 : {
392 1314 : llvm_compile_module(context);
393 : }
394 :
395 : /*
396 : * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
397 : * to mangle here.
398 : */
399 :
400 : #if LLVM_VERSION_MAJOR > 11
401 : foreach(lc, context->handles)
402 : {
403 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
404 : instr_time starttime;
405 : instr_time endtime;
406 : LLVMErrorRef error;
407 : LLVMOrcJITTargetAddress addr;
408 :
409 : INSTR_TIME_SET_CURRENT(starttime);
410 :
411 : addr = 0;
412 : error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
413 : if (error)
414 : elog(ERROR, "failed to look up symbol \"%s\": %s",
415 : funcname, llvm_error_message(error));
416 :
417 : /*
418 : * LLJIT only actually emits code the first time a symbol is
419 : * referenced. Thus add lookup time to emission time. That's counting
420 : * a bit more than with older LLVM versions, but unlikely to ever
421 : * matter.
422 : */
423 : INSTR_TIME_SET_CURRENT(endtime);
424 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
425 : endtime, starttime);
426 :
427 : if (addr)
428 : return (void *) (uintptr_t) addr;
429 : }
430 : #else
431 6398 : foreach(lc, context->handles)
432 : {
433 : LLVMOrcTargetAddress addr;
434 6398 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
435 :
436 6398 : addr = 0;
437 6398 : if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
438 0 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
439 6398 : if (addr)
440 6264 : return (void *) (uintptr_t) addr;
441 : }
442 : #endif
443 :
444 0 : elog(ERROR, "failed to JIT: %s", funcname);
445 :
446 : return NULL;
447 : }
448 :
449 : /*
450 : * Return type of a variable in llvmjit_types.c. This is useful to keep types
451 : * in sync between plain C and JIT related code.
452 : */
453 : LLVMTypeRef
454 16078 : llvm_pg_var_type(const char *varname)
455 : {
456 : LLVMValueRef v_srcvar;
457 : LLVMTypeRef typ;
458 :
459 : /* this'll return a *pointer* to the global */
460 16078 : v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
461 16078 : if (!v_srcvar)
462 0 : elog(ERROR, "variable %s not in llvmjit_types.c", varname);
463 :
464 16078 : typ = LLVMGlobalGetValueType(v_srcvar);
465 :
466 16078 : return typ;
467 : }
468 :
469 : /*
470 : * Return function type of a variable in llvmjit_types.c. This is useful to
471 : * keep function types in sync between C and JITed code.
472 : */
473 : LLVMTypeRef
474 15634 : llvm_pg_var_func_type(const char *varname)
475 : {
476 : LLVMValueRef v_srcvar;
477 : LLVMTypeRef typ;
478 :
479 15634 : v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
480 15634 : if (!v_srcvar)
481 0 : elog(ERROR, "function %s not in llvmjit_types.c", varname);
482 :
483 15634 : typ = LLVMGetFunctionType(v_srcvar);
484 :
485 15634 : return typ;
486 : }
487 :
488 : /*
489 : * Return declaration for a function referenced in llvmjit_types.c, adding it
490 : * to the module if necessary.
491 : *
492 : * This is used to make functions discovered via llvm_create_types() known to
493 : * the module that's currently being worked on.
494 : */
495 : LLVMValueRef
496 14206 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
497 : {
498 : LLVMValueRef v_srcfn;
499 : LLVMValueRef v_fn;
500 :
501 : /* don't repeatedly add function */
502 14206 : v_fn = LLVMGetNamedFunction(mod, funcname);
503 14206 : if (v_fn)
504 10278 : return v_fn;
505 :
506 3928 : v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
507 :
508 3928 : if (!v_srcfn)
509 0 : elog(ERROR, "function %s not in llvmjit_types.c", funcname);
510 :
511 3928 : v_fn = LLVMAddFunction(mod,
512 : funcname,
513 : LLVMGetFunctionType(v_srcfn));
514 3928 : llvm_copy_attributes(v_srcfn, v_fn);
515 :
516 3928 : return v_fn;
517 : }
518 :
519 : /*
520 : * Copy attributes from one function to another, for a specific index (an
521 : * index can reference return value, function and parameter attributes).
522 : */
523 : static void
524 61290 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
525 : {
526 : int num_attributes;
527 : LLVMAttributeRef *attrs;
528 :
529 61290 : num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
530 :
531 61290 : if (num_attributes == 0)
532 21620 : return;
533 :
534 39670 : attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
535 39670 : LLVMGetAttributesAtIndex(v_from, index, attrs);
536 :
537 465208 : for (int attno = 0; attno < num_attributes; attno++)
538 425538 : LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
539 :
540 39670 : pfree(attrs);
541 : }
542 :
543 : /*
544 : * Copy all attributes from one function to another. I.e. function, return and
545 : * parameters will be copied.
546 : */
547 : void
548 21756 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
549 : {
550 : uint32 param_count;
551 :
552 : /* copy function attributes */
553 21756 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
554 :
555 21756 : if (LLVMGetTypeKind(LLVMGetFunctionReturnType(v_to)) != LLVMVoidTypeKind)
556 : {
557 : /* and the return value attributes */
558 11900 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
559 : }
560 :
561 : /* and each function parameter's attribute */
562 21756 : param_count = LLVMCountParams(v_from);
563 :
564 49390 : for (int paramidx = 1; paramidx <= param_count; paramidx++)
565 27634 : llvm_copy_attributes_at_index(v_from, v_to, paramidx);
566 21756 : }
567 :
568 : /*
569 : * Return a callable LLVMValueRef for fcinfo.
570 : */
571 : LLVMValueRef
572 5814 : llvm_function_reference(LLVMJitContext *context,
573 : LLVMBuilderRef builder,
574 : LLVMModuleRef mod,
575 : FunctionCallInfo fcinfo)
576 : {
577 : char *modname;
578 : char *basename;
579 : char *funcname;
580 :
581 : LLVMValueRef v_fn;
582 :
583 5814 : fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
584 :
585 5814 : if (modname != NULL && basename != NULL)
586 : {
587 : /* external function in loadable library */
588 20 : funcname = psprintf("pgextern.%s.%s", modname, basename);
589 : }
590 5794 : else if (basename != NULL)
591 : {
592 : /* internal function */
593 5782 : funcname = pstrdup(basename);
594 : }
595 : else
596 : {
597 : /*
598 : * Function we don't know to handle, return pointer. We do so by
599 : * creating a global constant containing a pointer to the function.
600 : * Makes IR more readable.
601 : */
602 : LLVMValueRef v_fn_addr;
603 :
604 12 : funcname = psprintf("pgoidextern.%u",
605 12 : fcinfo->flinfo->fn_oid);
606 12 : v_fn = LLVMGetNamedGlobal(mod, funcname);
607 12 : if (v_fn != 0)
608 0 : return l_load(builder, TypePGFunction, v_fn, "");
609 :
610 12 : v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
611 :
612 12 : v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
613 12 : LLVMSetInitializer(v_fn, v_fn_addr);
614 12 : LLVMSetGlobalConstant(v_fn, true);
615 12 : LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
616 12 : LLVMSetUnnamedAddr(v_fn, true);
617 :
618 12 : return l_load(builder, TypePGFunction, v_fn, "");
619 : }
620 :
621 : /* check if function already has been added */
622 5802 : v_fn = LLVMGetNamedFunction(mod, funcname);
623 5802 : if (v_fn != 0)
624 2504 : return v_fn;
625 :
626 3298 : v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
627 :
628 3298 : return v_fn;
629 : }
630 :
631 : /*
632 : * Optimize code in module using the flags set in context.
633 : */
634 : static void
635 1314 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
636 : {
637 : #if LLVM_VERSION_MAJOR < 17
638 : LLVMPassManagerBuilderRef llvm_pmb;
639 : LLVMPassManagerRef llvm_mpm;
640 : LLVMPassManagerRef llvm_fpm;
641 : LLVMValueRef func;
642 : int compile_optlevel;
643 :
644 1314 : if (context->base.flags & PGJIT_OPT3)
645 980 : compile_optlevel = 3;
646 : else
647 334 : compile_optlevel = 0;
648 :
649 : /*
650 : * Have to create a new pass manager builder every pass through, as the
651 : * inliner has some per-builder state. Otherwise one ends up only inlining
652 : * a function the first time though.
653 : */
654 1314 : llvm_pmb = LLVMPassManagerBuilderCreate();
655 1314 : LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
656 1314 : llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
657 :
658 1314 : if (context->base.flags & PGJIT_OPT3)
659 : {
660 : /* TODO: Unscientifically determined threshold */
661 980 : LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
662 : }
663 : else
664 : {
665 : /* we rely on mem2reg heavily, so emit even in the O0 case */
666 334 : LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
667 : }
668 :
669 1314 : LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
670 :
671 : /*
672 : * Do function level optimization. This could be moved to the point where
673 : * functions are emitted, to reduce memory usage a bit.
674 : */
675 1314 : LLVMInitializeFunctionPassManager(llvm_fpm);
676 31626 : for (func = LLVMGetFirstFunction(context->module);
677 : func != NULL;
678 30312 : func = LLVMGetNextFunction(func))
679 30312 : LLVMRunFunctionPassManager(llvm_fpm, func);
680 1314 : LLVMFinalizeFunctionPassManager(llvm_fpm);
681 1314 : LLVMDisposePassManager(llvm_fpm);
682 :
683 : /*
684 : * Perform module level optimization. We do so even in the non-optimized
685 : * case, so always-inline functions etc get inlined. It's cheap enough.
686 : */
687 1314 : llvm_mpm = LLVMCreatePassManager();
688 1314 : LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
689 : llvm_mpm);
690 : /* always use always-inliner pass */
691 1314 : if (!(context->base.flags & PGJIT_OPT3))
692 334 : LLVMAddAlwaysInlinerPass(llvm_mpm);
693 : /* if doing inlining, but no expensive optimization, add inlining pass */
694 1314 : if (context->base.flags & PGJIT_INLINE
695 980 : && !(context->base.flags & PGJIT_OPT3))
696 0 : LLVMAddFunctionInliningPass(llvm_mpm);
697 1314 : LLVMRunPassManager(llvm_mpm, context->module);
698 1314 : LLVMDisposePassManager(llvm_mpm);
699 :
700 1314 : LLVMPassManagerBuilderDispose(llvm_pmb);
701 : #else
702 : LLVMPassBuilderOptionsRef options;
703 : LLVMErrorRef err;
704 : const char *passes;
705 :
706 : if (context->base.flags & PGJIT_OPT3)
707 : passes = "default<O3>";
708 : else
709 : passes = "default<O0>,mem2reg";
710 :
711 : options = LLVMCreatePassBuilderOptions();
712 :
713 : #ifdef LLVM_PASS_DEBUG
714 : LLVMPassBuilderOptionsSetDebugLogging(options, 1);
715 : #endif
716 :
717 : LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
718 :
719 : err = LLVMRunPasses(module, passes, NULL, options);
720 :
721 : if (err)
722 : elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
723 :
724 : LLVMDisposePassBuilderOptions(options);
725 : #endif
726 1314 : }
727 :
728 : /*
729 : * Emit code for the currently pending module.
730 : */
731 : static void
732 1314 : llvm_compile_module(LLVMJitContext *context)
733 : {
734 : LLVMJitHandle *handle;
735 : MemoryContext oldcontext;
736 : instr_time starttime;
737 : instr_time endtime;
738 : #if LLVM_VERSION_MAJOR > 11
739 : LLVMOrcLLJITRef compile_orc;
740 : #else
741 : LLVMOrcJITStackRef compile_orc;
742 : #endif
743 :
744 1314 : if (context->base.flags & PGJIT_OPT3)
745 980 : compile_orc = llvm_opt3_orc;
746 : else
747 334 : compile_orc = llvm_opt0_orc;
748 :
749 : /* perform inlining */
750 1314 : if (context->base.flags & PGJIT_INLINE)
751 : {
752 980 : INSTR_TIME_SET_CURRENT(starttime);
753 980 : llvm_inline(context->module);
754 980 : INSTR_TIME_SET_CURRENT(endtime);
755 980 : INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
756 : endtime, starttime);
757 : }
758 :
759 1314 : if (jit_dump_bitcode)
760 : {
761 : char *filename;
762 :
763 0 : filename = psprintf("%d.%zu.bc",
764 : MyProcPid,
765 : context->module_generation);
766 0 : LLVMWriteBitcodeToFile(context->module, filename);
767 0 : pfree(filename);
768 : }
769 :
770 :
771 : /* optimize according to the chosen optimization settings */
772 1314 : INSTR_TIME_SET_CURRENT(starttime);
773 1314 : llvm_optimize_module(context, context->module);
774 1314 : INSTR_TIME_SET_CURRENT(endtime);
775 1314 : INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
776 : endtime, starttime);
777 :
778 1314 : if (jit_dump_bitcode)
779 : {
780 : char *filename;
781 :
782 0 : filename = psprintf("%d.%zu.optimized.bc",
783 : MyProcPid,
784 : context->module_generation);
785 0 : LLVMWriteBitcodeToFile(context->module, filename);
786 0 : pfree(filename);
787 : }
788 :
789 : handle = (LLVMJitHandle *)
790 1314 : MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
791 :
792 : /*
793 : * Emit the code. Note that this can, depending on the optimization
794 : * settings, take noticeable resources as code emission executes low-level
795 : * instruction combining/selection passes etc. Without optimization a
796 : * faster instruction selection mechanism is used.
797 : */
798 1314 : INSTR_TIME_SET_CURRENT(starttime);
799 : #if LLVM_VERSION_MAJOR > 11
800 : {
801 : LLVMOrcThreadSafeModuleRef ts_module;
802 : LLVMErrorRef error;
803 : LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
804 :
805 : ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
806 :
807 : handle->lljit = compile_orc;
808 : handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
809 :
810 : /*
811 : * NB: This doesn't actually emit code. That happens lazily the first
812 : * time a symbol defined in the module is requested. Due to that
813 : * llvm_get_function() also accounts for emission time.
814 : */
815 :
816 : context->module = NULL; /* will be owned by LLJIT */
817 : error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
818 : handle->resource_tracker,
819 : ts_module);
820 :
821 : if (error)
822 : elog(ERROR, "failed to JIT module: %s",
823 : llvm_error_message(error));
824 :
825 : /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
826 : }
827 : #else
828 : {
829 1314 : handle->stack = compile_orc;
830 1314 : if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
831 : llvm_resolve_symbol, NULL))
832 0 : elog(ERROR, "failed to JIT module");
833 :
834 : /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
835 : }
836 : #endif
837 :
838 1314 : INSTR_TIME_SET_CURRENT(endtime);
839 1314 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
840 : endtime, starttime);
841 :
842 1314 : context->module = NULL;
843 1314 : context->compiled = true;
844 :
845 : /* remember emitted code for cleanup and lookups */
846 1314 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
847 1314 : context->handles = lappend(context->handles, handle);
848 1314 : MemoryContextSwitchTo(oldcontext);
849 :
850 1314 : ereport(DEBUG1,
851 : (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
852 : INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
853 : INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
854 : INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
855 : errhidestmt(true),
856 : errhidecontext(true)));
857 1314 : }
858 :
859 : /*
860 : * Per session initialization.
861 : */
862 : static void
863 1768 : llvm_session_initialize(void)
864 : {
865 : MemoryContext oldcontext;
866 1768 : char *error = NULL;
867 1768 : char *cpu = NULL;
868 1768 : char *features = NULL;
869 : LLVMTargetMachineRef opt0_tm;
870 : LLVMTargetMachineRef opt3_tm;
871 :
872 1768 : if (llvm_session_initialized)
873 970 : return;
874 :
875 798 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
876 :
877 798 : LLVMInitializeNativeTarget();
878 798 : LLVMInitializeNativeAsmPrinter();
879 798 : LLVMInitializeNativeAsmParser();
880 :
881 798 : if (llvm_context == NULL)
882 : {
883 798 : llvm_context = LLVMContextCreate();
884 :
885 798 : llvm_jit_context_in_use_count = 0;
886 798 : llvm_llvm_context_reuse_count = 0;
887 : }
888 :
889 : /*
890 : * When targeting LLVM 15, turn off opaque pointers for the context we
891 : * build our code in. We don't need to do so for other contexts (e.g.
892 : * llvm_ts_context). Once the IR is generated, it carries the necessary
893 : * information.
894 : *
895 : * For 16 and above, opaque pointers must be used, and we have special
896 : * code for that.
897 : */
898 : #if LLVM_VERSION_MAJOR == 15
899 : LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
900 : #endif
901 :
902 : /*
903 : * Synchronize types early, as that also includes inferring the target
904 : * triple.
905 : */
906 798 : llvm_create_types();
907 :
908 : /*
909 : * Extract target information from loaded module.
910 : */
911 798 : llvm_set_target();
912 :
913 798 : if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
914 : {
915 0 : elog(FATAL, "failed to query triple %s", error);
916 : }
917 :
918 : /*
919 : * We want the generated code to use all available features. Therefore
920 : * grab the host CPU string and detect features of the current CPU. The
921 : * latter is needed because some CPU architectures default to enabling
922 : * features not all CPUs have (weird, huh).
923 : */
924 798 : cpu = LLVMGetHostCPUName();
925 798 : features = LLVMGetHostCPUFeatures();
926 798 : elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
927 : cpu, features);
928 :
929 : opt0_tm =
930 798 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
931 : LLVMCodeGenLevelNone,
932 : LLVMRelocDefault,
933 : LLVMCodeModelJITDefault);
934 : opt3_tm =
935 798 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
936 : LLVMCodeGenLevelAggressive,
937 : LLVMRelocDefault,
938 : LLVMCodeModelJITDefault);
939 :
940 798 : LLVMDisposeMessage(cpu);
941 798 : cpu = NULL;
942 798 : LLVMDisposeMessage(features);
943 798 : features = NULL;
944 :
945 : /* force symbols in main binary to be loaded */
946 798 : LLVMLoadLibraryPermanently(NULL);
947 :
948 : #if LLVM_VERSION_MAJOR > 11
949 : {
950 : llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
951 :
952 : llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
953 : opt0_tm = 0;
954 :
955 : llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
956 : opt3_tm = 0;
957 : }
958 : #else /* LLVM_VERSION_MAJOR > 11 */
959 : {
960 798 : llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
961 798 : llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
962 :
963 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
964 798 : if (jit_debugging_support)
965 : {
966 0 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
967 :
968 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
969 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
970 : }
971 : #endif
972 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
973 798 : if (jit_profiling_support)
974 : {
975 0 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
976 :
977 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
978 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
979 : }
980 : #endif
981 : }
982 : #endif /* LLVM_VERSION_MAJOR > 11 */
983 :
984 798 : on_proc_exit(llvm_shutdown, 0);
985 :
986 798 : llvm_session_initialized = true;
987 :
988 798 : MemoryContextSwitchTo(oldcontext);
989 : }
990 :
991 : static void
992 798 : llvm_shutdown(int code, Datum arg)
993 : {
994 : /*
995 : * If llvm_shutdown() is reached while in a fatal-on-oom section an error
996 : * has occurred in the middle of LLVM code. It is not safe to call back
997 : * into LLVM (which is why a FATAL error was thrown).
998 : *
999 : * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
1000 : * profiling data won't be written out.
1001 : */
1002 798 : if (llvm_in_fatal_on_oom())
1003 : {
1004 : Assert(proc_exit_inprogress);
1005 0 : return;
1006 : }
1007 :
1008 798 : if (llvm_jit_context_in_use_count != 0)
1009 0 : elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
1010 : llvm_jit_context_in_use_count);
1011 :
1012 : #if LLVM_VERSION_MAJOR > 11
1013 : {
1014 : if (llvm_opt3_orc)
1015 : {
1016 : LLVMOrcDisposeLLJIT(llvm_opt3_orc);
1017 : llvm_opt3_orc = NULL;
1018 : }
1019 : if (llvm_opt0_orc)
1020 : {
1021 : LLVMOrcDisposeLLJIT(llvm_opt0_orc);
1022 : llvm_opt0_orc = NULL;
1023 : }
1024 : if (llvm_ts_context)
1025 : {
1026 : LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
1027 : llvm_ts_context = NULL;
1028 : }
1029 : }
1030 : #else /* LLVM_VERSION_MAJOR > 11 */
1031 : {
1032 : /* unregister profiling support, needs to be flushed to be useful */
1033 :
1034 798 : if (llvm_opt3_orc)
1035 : {
1036 798 : LLVMOrcDisposeInstance(llvm_opt3_orc);
1037 798 : llvm_opt3_orc = NULL;
1038 : }
1039 :
1040 798 : if (llvm_opt0_orc)
1041 : {
1042 798 : LLVMOrcDisposeInstance(llvm_opt0_orc);
1043 798 : llvm_opt0_orc = NULL;
1044 : }
1045 : }
1046 : #endif /* LLVM_VERSION_MAJOR > 11 */
1047 : }
1048 :
1049 : /* helper for llvm_create_types, returning a function's return type */
1050 : static LLVMTypeRef
1051 798 : load_return_type(LLVMModuleRef mod, const char *name)
1052 : {
1053 : LLVMValueRef value;
1054 : LLVMTypeRef typ;
1055 :
1056 : /* this'll return a *pointer* to the function */
1057 798 : value = LLVMGetNamedFunction(mod, name);
1058 798 : if (!value)
1059 0 : elog(ERROR, "function %s is unknown", name);
1060 :
1061 798 : typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
1062 :
1063 798 : return typ;
1064 : }
1065 :
1066 : /*
1067 : * Load triple & layout from clang emitted file so we're guaranteed to be
1068 : * compatible.
1069 : */
1070 : static void
1071 798 : llvm_set_target(void)
1072 : {
1073 798 : if (!llvm_types_module)
1074 0 : elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
1075 :
1076 798 : if (llvm_triple == NULL)
1077 798 : llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1078 :
1079 798 : if (llvm_layout == NULL)
1080 798 : llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1081 798 : }
1082 :
1083 : /*
1084 : * Load required information, types, function signatures from llvmjit_types.c
1085 : * and make them available in global variables.
1086 : *
1087 : * Those global variables are then used while emitting code.
1088 : */
1089 : static void
1090 798 : llvm_create_types(void)
1091 : {
1092 : char path[MAXPGPATH];
1093 : LLVMMemoryBufferRef buf;
1094 : char *msg;
1095 :
1096 798 : snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
1097 :
1098 : /* open file */
1099 798 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1100 : {
1101 0 : elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1102 : path, msg);
1103 : }
1104 :
1105 : /* eagerly load contents, going to need it all */
1106 798 : if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1107 : {
1108 0 : elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1109 : }
1110 798 : LLVMDisposeMemoryBuffer(buf);
1111 :
1112 798 : TypeSizeT = llvm_pg_var_type("TypeSizeT");
1113 798 : TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1114 798 : TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1115 798 : TypePGFunction = llvm_pg_var_type("TypePGFunction");
1116 798 : StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1117 798 : StructExprContext = llvm_pg_var_type("StructExprContext");
1118 798 : StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1119 798 : StructExprState = llvm_pg_var_type("StructExprState");
1120 798 : StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1121 798 : StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1122 798 : StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1123 798 : StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1124 798 : StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1125 798 : StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1126 798 : StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1127 798 : StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1128 798 : StructAggState = llvm_pg_var_type("StructAggState");
1129 798 : StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1130 798 : StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1131 798 : StructPlanState = llvm_pg_var_type("StructPlanState");
1132 798 : StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1133 :
1134 798 : AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1135 798 : ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1136 798 : ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1137 798 : }
1138 :
1139 : /*
1140 : * Split a symbol into module / function parts. If the function is in the
1141 : * main binary (or an external library) *modname will be NULL.
1142 : */
1143 : void
1144 32604 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1145 : {
1146 32604 : *modname = NULL;
1147 32604 : *funcname = NULL;
1148 :
1149 : /*
1150 : * Module function names are pgextern.$module.$funcname
1151 : */
1152 32604 : if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1153 : {
1154 : /*
1155 : * Symbol names cannot contain a ., therefore we can split based on
1156 : * first and last occurrence of one.
1157 : */
1158 58 : *funcname = rindex(name, '.');
1159 58 : (*funcname)++; /* jump over . */
1160 :
1161 116 : *modname = pnstrdup(name + strlen("pgextern."),
1162 58 : *funcname - name - strlen("pgextern.") - 1);
1163 : Assert(funcname);
1164 :
1165 58 : *funcname = pstrdup(*funcname);
1166 : }
1167 : else
1168 : {
1169 32546 : *modname = NULL;
1170 32546 : *funcname = pstrdup(name);
1171 : }
1172 32604 : }
1173 :
1174 : /*
1175 : * Attempt to resolve symbol, so LLVM can emit a reference to it.
1176 : */
1177 : static uint64_t
1178 5212 : llvm_resolve_symbol(const char *symname, void *ctx)
1179 : {
1180 : uintptr_t addr;
1181 : char *funcname;
1182 : char *modname;
1183 :
1184 : /*
1185 : * macOS prefixes all object level symbols with an underscore. But neither
1186 : * dlsym() nor PG's inliner expect that. So undo.
1187 : */
1188 : #if defined(__darwin__)
1189 : if (symname[0] != '_')
1190 : elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1191 : symname++;
1192 : #endif
1193 :
1194 5212 : llvm_split_symbol_name(symname, &modname, &funcname);
1195 :
1196 : /* functions that aren't resolved to names shouldn't ever get here */
1197 : Assert(funcname);
1198 :
1199 5212 : if (modname)
1200 20 : addr = (uintptr_t) load_external_function(modname, funcname,
1201 : true, NULL);
1202 : else
1203 5192 : addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1204 :
1205 5212 : pfree(funcname);
1206 5212 : if (modname)
1207 20 : pfree(modname);
1208 :
1209 : /* let LLVM will error out - should never happen */
1210 5212 : if (!addr)
1211 0 : elog(WARNING, "failed to resolve name %s", symname);
1212 :
1213 5212 : return (uint64_t) addr;
1214 : }
1215 :
1216 : #if LLVM_VERSION_MAJOR > 11
1217 :
1218 : static LLVMErrorRef
1219 : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1220 : LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1221 : LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1222 : LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1223 : {
1224 : #if LLVM_VERSION_MAJOR > 14
1225 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1226 : #else
1227 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1228 : #endif
1229 : LLVMErrorRef error;
1230 : LLVMOrcMaterializationUnitRef mu;
1231 :
1232 : for (int i = 0; i < LookupSetSize; i++)
1233 : {
1234 : const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1235 :
1236 : #if LLVM_VERSION_MAJOR > 12
1237 : LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1238 : #endif
1239 : symbols[i].Name = LookupSet[i].Name;
1240 : symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1241 : symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1242 : }
1243 :
1244 : mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1245 : error = LLVMOrcJITDylibDefine(JD, mu);
1246 : if (error != LLVMErrorSuccess)
1247 : LLVMOrcDisposeMaterializationUnit(mu);
1248 :
1249 : pfree(symbols);
1250 :
1251 : return error;
1252 : }
1253 :
1254 : /*
1255 : * We cannot throw errors through LLVM (without causing a FATAL at least), so
1256 : * just use WARNING here. That's OK anyway, as the error is also reported at
1257 : * the top level action (with less detail) and there might be multiple
1258 : * invocations of errors with details.
1259 : *
1260 : * This doesn't really happen during normal operation, but in cases like
1261 : * symbol resolution breakage. So just using elog(WARNING) is fine.
1262 : */
1263 : static void
1264 : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1265 : {
1266 : elog(WARNING, "error during JITing: %s",
1267 : llvm_error_message(error));
1268 : }
1269 :
1270 : /*
1271 : * Create our own object layer, so we can add event listeners.
1272 : */
1273 : static LLVMOrcObjectLayerRef
1274 : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1275 : {
1276 : LLVMOrcObjectLayerRef objlayer =
1277 : LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1278 :
1279 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1280 : if (jit_debugging_support)
1281 : {
1282 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1283 :
1284 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1285 : }
1286 : #endif
1287 :
1288 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1289 : if (jit_profiling_support)
1290 : {
1291 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1292 :
1293 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1294 : }
1295 : #endif
1296 :
1297 : return objlayer;
1298 : }
1299 :
1300 : /*
1301 : * Create LLJIT instance, using the passed in target machine. Note that the
1302 : * target machine afterwards is owned by the LLJIT instance.
1303 : */
1304 : static LLVMOrcLLJITRef
1305 : llvm_create_jit_instance(LLVMTargetMachineRef tm)
1306 : {
1307 : LLVMOrcLLJITRef lljit;
1308 : LLVMOrcJITTargetMachineBuilderRef tm_builder;
1309 : LLVMOrcLLJITBuilderRef lljit_builder;
1310 : LLVMErrorRef error;
1311 : LLVMOrcDefinitionGeneratorRef main_gen;
1312 : LLVMOrcDefinitionGeneratorRef ref_gen;
1313 :
1314 : lljit_builder = LLVMOrcCreateLLJITBuilder();
1315 : tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1316 : LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1317 :
1318 : LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1319 : llvm_create_object_layer,
1320 : NULL);
1321 :
1322 : error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1323 : if (error)
1324 : elog(ERROR, "failed to create lljit instance: %s",
1325 : llvm_error_message(error));
1326 :
1327 : LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1328 : llvm_log_jit_error, NULL);
1329 :
1330 : /*
1331 : * Symbol resolution support for symbols in the postgres binary /
1332 : * libraries already loaded.
1333 : */
1334 : error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1335 : LLVMOrcLLJITGetGlobalPrefix(lljit),
1336 : 0, NULL);
1337 : if (error)
1338 : elog(ERROR, "failed to create generator: %s",
1339 : llvm_error_message(error));
1340 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1341 :
1342 : /*
1343 : * Symbol resolution support for "special" functions, e.g. a call into an
1344 : * SQL callable function.
1345 : */
1346 : #if LLVM_VERSION_MAJOR > 14
1347 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1348 : #else
1349 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1350 : #endif
1351 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1352 :
1353 : return lljit;
1354 : }
1355 :
1356 : static char *
1357 : llvm_error_message(LLVMErrorRef error)
1358 : {
1359 : char *orig = LLVMGetErrorMessage(error);
1360 : char *msg = pstrdup(orig);
1361 :
1362 : LLVMDisposeErrorMessage(orig);
1363 :
1364 : return msg;
1365 : }
1366 :
1367 : #endif /* LLVM_VERSION_MAJOR > 11 */
1368 :
1369 : /*
1370 : * ResourceOwner callbacks
1371 : */
1372 : static void
1373 16 : ResOwnerReleaseJitContext(Datum res)
1374 : {
1375 16 : JitContext *context = (JitContext *) DatumGetPointer(res);
1376 :
1377 16 : context->resowner = NULL;
1378 16 : jit_release_context(context);
1379 16 : }
|
