Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit.c
4 : * Core part of the LLVM JIT provider.
5 : *
6 : * Copyright (c) 2016-2024, PostgreSQL Global Development Group
7 : *
8 : * IDENTIFICATION
9 : * src/backend/jit/llvm/llvmjit.c
10 : *
11 : *-------------------------------------------------------------------------
12 : */
13 :
14 : #include "postgres.h"
15 :
16 : #include <llvm-c/Analysis.h>
17 : #include <llvm-c/BitReader.h>
18 : #include <llvm-c/BitWriter.h>
19 : #include <llvm-c/Core.h>
20 : #include <llvm-c/ExecutionEngine.h>
21 : #if LLVM_VERSION_MAJOR > 16
22 : #include <llvm-c/Transforms/PassBuilder.h>
23 : #endif
24 : #if LLVM_VERSION_MAJOR > 11
25 : #include <llvm-c/Orc.h>
26 : #include <llvm-c/OrcEE.h>
27 : #include <llvm-c/LLJIT.h>
28 : #else
29 : #include <llvm-c/OrcBindings.h>
30 : #endif
31 : #include <llvm-c/Support.h>
32 : #include <llvm-c/Target.h>
33 : #if LLVM_VERSION_MAJOR < 17
34 : #include <llvm-c/Transforms/IPO.h>
35 : #include <llvm-c/Transforms/PassManagerBuilder.h>
36 : #include <llvm-c/Transforms/Scalar.h>
37 : #include <llvm-c/Transforms/Utils.h>
38 : #endif
39 :
40 : #include "jit/llvmjit.h"
41 : #include "jit/llvmjit_emit.h"
42 : #include "miscadmin.h"
43 : #include "portability/instr_time.h"
44 : #include "storage/ipc.h"
45 : #include "utils/memutils.h"
46 : #include "utils/resowner.h"
47 :
48 : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
49 :
50 : /* Handle of a module emitted via ORC JIT */
51 : typedef struct LLVMJitHandle
52 : {
53 : #if LLVM_VERSION_MAJOR > 11
54 : LLVMOrcLLJITRef lljit;
55 : LLVMOrcResourceTrackerRef resource_tracker;
56 : #else
57 : LLVMOrcJITStackRef stack;
58 : LLVMOrcModuleHandle orc_handle;
59 : #endif
60 : } LLVMJitHandle;
61 :
62 :
63 : /* types & functions commonly needed for JITing */
64 : LLVMTypeRef TypeSizeT;
65 : LLVMTypeRef TypeParamBool;
66 : LLVMTypeRef TypeStorageBool;
67 : LLVMTypeRef TypePGFunction;
68 : LLVMTypeRef StructNullableDatum;
69 : LLVMTypeRef StructHeapTupleData;
70 : LLVMTypeRef StructMinimalTupleData;
71 : LLVMTypeRef StructTupleDescData;
72 : LLVMTypeRef StructTupleTableSlot;
73 : LLVMTypeRef StructHeapTupleHeaderData;
74 : LLVMTypeRef StructHeapTupleTableSlot;
75 : LLVMTypeRef StructMinimalTupleTableSlot;
76 : LLVMTypeRef StructMemoryContextData;
77 : LLVMTypeRef StructFunctionCallInfoData;
78 : LLVMTypeRef StructExprContext;
79 : LLVMTypeRef StructExprEvalStep;
80 : LLVMTypeRef StructExprState;
81 : LLVMTypeRef StructAggState;
82 : LLVMTypeRef StructAggStatePerGroupData;
83 : LLVMTypeRef StructAggStatePerTransData;
84 : LLVMTypeRef StructPlanState;
85 :
86 : LLVMValueRef AttributeTemplate;
87 : LLVMValueRef ExecEvalSubroutineTemplate;
88 : LLVMValueRef ExecEvalBoolSubroutineTemplate;
89 :
90 : static LLVMModuleRef llvm_types_module = NULL;
91 :
92 : static bool llvm_session_initialized = false;
93 : static size_t llvm_generation = 0;
94 :
95 : /* number of LLVMJitContexts that currently are in use */
96 : static size_t llvm_jit_context_in_use_count = 0;
97 :
98 : /* how many times has the current LLVMContextRef been used */
99 : static size_t llvm_llvm_context_reuse_count = 0;
100 : static const char *llvm_triple = NULL;
101 : static const char *llvm_layout = NULL;
102 : static LLVMContextRef llvm_context;
103 :
104 :
105 : static LLVMTargetRef llvm_targetref;
106 : #if LLVM_VERSION_MAJOR > 11
107 : static LLVMOrcThreadSafeContextRef llvm_ts_context;
108 : static LLVMOrcLLJITRef llvm_opt0_orc;
109 : static LLVMOrcLLJITRef llvm_opt3_orc;
110 : #else /* LLVM_VERSION_MAJOR > 11 */
111 : static LLVMOrcJITStackRef llvm_opt0_orc;
112 : static LLVMOrcJITStackRef llvm_opt3_orc;
113 : #endif /* LLVM_VERSION_MAJOR > 11 */
114 :
115 :
116 : static void llvm_release_context(JitContext *context);
117 : static void llvm_session_initialize(void);
118 : static void llvm_shutdown(int code, Datum arg);
119 : static void llvm_compile_module(LLVMJitContext *context);
120 : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
121 :
122 : static void llvm_create_types(void);
123 : static void llvm_set_target(void);
124 : static void llvm_recreate_llvm_context(void);
125 : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
126 :
127 : #if LLVM_VERSION_MAJOR > 11
128 : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
129 : static char *llvm_error_message(LLVMErrorRef error);
130 : #endif /* LLVM_VERSION_MAJOR > 11 */
131 :
132 : /* ResourceOwner callbacks to hold JitContexts */
133 : static void ResOwnerReleaseJitContext(Datum res);
134 :
135 : static const ResourceOwnerDesc jit_resowner_desc =
136 : {
137 : .name = "LLVM JIT context",
138 : .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
139 : .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
140 : .ReleaseResource = ResOwnerReleaseJitContext,
141 : .DebugPrint = NULL /* the default message is fine */
142 : };
143 :
144 : /* Convenience wrappers over ResourceOwnerRemember/Forget */
145 : static inline void
146 1774 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
147 : {
148 1774 : ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
149 1774 : }
150 : static inline void
151 1758 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
152 : {
153 1758 : ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
154 1758 : }
155 :
156 1988 : PG_MODULE_MAGIC;
157 :
158 :
159 : /*
160 : * Initialize LLVM JIT provider.
161 : */
162 : void
163 798 : _PG_jit_provider_init(JitProviderCallbacks *cb)
164 : {
165 798 : cb->reset_after_error = llvm_reset_after_error;
166 798 : cb->release_context = llvm_release_context;
167 798 : cb->compile_expr = llvm_compile_expr;
168 798 : }
169 :
170 :
171 : /*
172 : * Every now and then create a new LLVMContextRef. Unfortunately, during every
173 : * round of inlining, types may "leak" (they can still be found/used via the
174 : * context, but new types will be created the next time in inlining is
175 : * performed). To prevent that from slowly accumulating problematic amounts of
176 : * memory, recreate the LLVMContextRef we use. We don't want to do so too
177 : * often, as that implies some overhead (particularly re-loading the module
178 : * summaries / modules is fairly expensive). A future TODO would be to make
179 : * this more finegrained and only drop/recreate the LLVMContextRef when we know
180 : * there has been inlining. If we can get the size of the context from LLVM
181 : * then that might be a better way to determine when to drop/recreate rather
182 : * then the usagecount heuristic currently employed.
183 : */
184 : static void
185 1774 : llvm_recreate_llvm_context(void)
186 : {
187 1774 : if (!llvm_context)
188 0 : elog(ERROR, "Trying to recreate a non-existing context");
189 :
190 : /*
191 : * We can only safely recreate the LLVM context if no other code is being
192 : * JITed, otherwise we'd release the types in use for that.
193 : */
194 1774 : if (llvm_jit_context_in_use_count > 0)
195 : {
196 0 : llvm_llvm_context_reuse_count++;
197 0 : return;
198 : }
199 :
200 1774 : if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
201 : {
202 1774 : llvm_llvm_context_reuse_count++;
203 1774 : return;
204 : }
205 :
206 : /*
207 : * Need to reset the modules that the inlining code caches before
208 : * disposing of the context. LLVM modules exist within a specific LLVM
209 : * context, therefore disposing of the context before resetting the cache
210 : * would lead to dangling pointers to modules.
211 : */
212 0 : llvm_inline_reset_caches();
213 :
214 0 : LLVMContextDispose(llvm_context);
215 0 : llvm_context = LLVMContextCreate();
216 0 : llvm_llvm_context_reuse_count = 0;
217 :
218 : /*
219 : * Re-build cached type information, so code generation code can rely on
220 : * that information to be present (also prevents the variables to be
221 : * dangling references).
222 : */
223 0 : llvm_create_types();
224 : }
225 :
226 :
227 : /*
228 : * Create a context for JITing work.
229 : *
230 : * The context, including subsidiary resources, will be cleaned up either when
231 : * the context is explicitly released, or when the lifetime of
232 : * CurrentResourceOwner ends (usually the end of the current [sub]xact).
233 : */
234 : LLVMJitContext *
235 1774 : llvm_create_context(int jitFlags)
236 : {
237 : LLVMJitContext *context;
238 :
239 1774 : llvm_assert_in_fatal_section();
240 :
241 1774 : llvm_session_initialize();
242 :
243 1774 : llvm_recreate_llvm_context();
244 :
245 1774 : ResourceOwnerEnlarge(CurrentResourceOwner);
246 :
247 1774 : context = MemoryContextAllocZero(TopMemoryContext,
248 : sizeof(LLVMJitContext));
249 1774 : context->base.flags = jitFlags;
250 :
251 : /* ensure cleanup */
252 1774 : context->base.resowner = CurrentResourceOwner;
253 1774 : ResourceOwnerRememberJIT(CurrentResourceOwner, context);
254 :
255 1774 : llvm_jit_context_in_use_count++;
256 :
257 1774 : return context;
258 : }
259 :
260 : /*
261 : * Release resources required by one llvm context.
262 : */
263 : static void
264 1774 : llvm_release_context(JitContext *context)
265 : {
266 1774 : LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
267 : ListCell *lc;
268 :
269 : /*
270 : * Consider as cleaned up even if we skip doing so below, that way we can
271 : * verify the tracking is correct (see llvm_shutdown()).
272 : */
273 1774 : llvm_jit_context_in_use_count--;
274 :
275 : /*
276 : * When this backend is exiting, don't clean up LLVM. As an error might
277 : * have occurred from within LLVM, we do not want to risk reentering. All
278 : * resource cleanup is going to happen through process exit.
279 : */
280 1774 : if (proc_exit_inprogress)
281 0 : return;
282 :
283 1774 : llvm_enter_fatal_on_oom();
284 :
285 1774 : if (llvm_jit_context->module)
286 : {
287 582 : LLVMDisposeModule(llvm_jit_context->module);
288 582 : llvm_jit_context->module = NULL;
289 : }
290 :
291 3088 : foreach(lc, llvm_jit_context->handles)
292 : {
293 1314 : LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
294 :
295 : #if LLVM_VERSION_MAJOR > 11
296 : {
297 : LLVMOrcExecutionSessionRef ee;
298 : LLVMOrcSymbolStringPoolRef sp;
299 :
300 : LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
301 : LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
302 :
303 : /*
304 : * Without triggering cleanup of the string pool, we'd leak
305 : * memory. It'd be sufficient to do this far less often, but in
306 : * experiments the required time was small enough to just always
307 : * do it.
308 : */
309 : ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
310 : sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
311 : LLVMOrcSymbolStringPoolClearDeadEntries(sp);
312 : }
313 : #else /* LLVM_VERSION_MAJOR > 11 */
314 : {
315 1314 : LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
316 : }
317 : #endif /* LLVM_VERSION_MAJOR > 11 */
318 :
319 1314 : pfree(jit_handle);
320 : }
321 1774 : list_free(llvm_jit_context->handles);
322 1774 : llvm_jit_context->handles = NIL;
323 :
324 1774 : llvm_leave_fatal_on_oom();
325 :
326 1774 : if (context->resowner)
327 1758 : ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
328 : }
329 :
330 : /*
331 : * Return module which may be modified, e.g. by creating new functions.
332 : */
333 : LLVMModuleRef
334 17774 : llvm_mutable_module(LLVMJitContext *context)
335 : {
336 17774 : llvm_assert_in_fatal_section();
337 :
338 : /*
339 : * If there's no in-progress module, create a new one.
340 : */
341 17774 : if (!context->module)
342 : {
343 1896 : context->compiled = false;
344 1896 : context->module_generation = llvm_generation++;
345 1896 : context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
346 1896 : LLVMSetTarget(context->module, llvm_triple);
347 1896 : LLVMSetDataLayout(context->module, llvm_layout);
348 : }
349 :
350 17774 : return context->module;
351 : }
352 :
353 : /*
354 : * Expand function name to be non-conflicting. This should be used by code
355 : * generating code, when adding new externally visible function definitions to
356 : * a Module.
357 : */
358 : char *
359 17774 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
360 : {
361 : Assert(context->module != NULL);
362 :
363 17774 : context->base.instr.created_functions++;
364 :
365 : /*
366 : * Previously we used dots to separate, but turns out some tools, e.g.
367 : * GDB, don't like that and truncate name.
368 : */
369 35548 : return psprintf("%s_%zu_%d",
370 : basename,
371 : context->module_generation,
372 17774 : context->counter++);
373 : }
374 :
375 : /*
376 : * Return pointer to function funcname, which has to exist. If there's pending
377 : * code to be optimized and emitted, do so first.
378 : */
379 : void *
380 6196 : llvm_get_function(LLVMJitContext *context, const char *funcname)
381 : {
382 : ListCell *lc;
383 :
384 6196 : llvm_assert_in_fatal_section();
385 :
386 : /*
387 : * If there is a pending / not emitted module, compile and emit now.
388 : * Otherwise we might not find the [correct] function.
389 : */
390 6196 : if (!context->compiled)
391 : {
392 1314 : llvm_compile_module(context);
393 : }
394 :
395 : /*
396 : * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
397 : * to mangle here.
398 : */
399 :
400 : #if LLVM_VERSION_MAJOR > 11
401 : foreach(lc, context->handles)
402 : {
403 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
404 : instr_time starttime;
405 : instr_time endtime;
406 : LLVMErrorRef error;
407 : LLVMOrcJITTargetAddress addr;
408 :
409 : INSTR_TIME_SET_CURRENT(starttime);
410 :
411 : addr = 0;
412 : error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
413 : if (error)
414 : elog(ERROR, "failed to look up symbol \"%s\": %s",
415 : funcname, llvm_error_message(error));
416 :
417 : /*
418 : * LLJIT only actually emits code the first time a symbol is
419 : * referenced. Thus add lookup time to emission time. That's counting
420 : * a bit more than with older LLVM versions, but unlikely to ever
421 : * matter.
422 : */
423 : INSTR_TIME_SET_CURRENT(endtime);
424 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
425 : endtime, starttime);
426 :
427 : if (addr)
428 : return (void *) (uintptr_t) addr;
429 : }
430 : #else
431 6330 : foreach(lc, context->handles)
432 : {
433 : LLVMOrcTargetAddress addr;
434 6330 : LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
435 :
436 6330 : addr = 0;
437 6330 : if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
438 0 : elog(ERROR, "failed to look up symbol \"%s\"", funcname);
439 6330 : if (addr)
440 6196 : return (void *) (uintptr_t) addr;
441 : }
442 : #endif
443 :
444 0 : elog(ERROR, "failed to JIT: %s", funcname);
445 :
446 : return NULL;
447 : }
448 :
449 : /*
450 : * Return type of a variable in llvmjit_types.c. This is useful to keep types
451 : * in sync between plain C and JIT related code.
452 : */
453 : LLVMTypeRef
454 16078 : llvm_pg_var_type(const char *varname)
455 : {
456 : LLVMValueRef v_srcvar;
457 : LLVMTypeRef typ;
458 :
459 : /* this'll return a *pointer* to the global */
460 16078 : v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
461 16078 : if (!v_srcvar)
462 0 : elog(ERROR, "variable %s not in llvmjit_types.c", varname);
463 :
464 16078 : typ = LLVMGlobalGetValueType(v_srcvar);
465 :
466 16078 : return typ;
467 : }
468 :
469 : /*
470 : * Return function type of a variable in llvmjit_types.c. This is useful to
471 : * keep function types in sync between C and JITed code.
472 : */
473 : LLVMTypeRef
474 15604 : llvm_pg_var_func_type(const char *varname)
475 : {
476 : LLVMValueRef v_srcvar;
477 : LLVMTypeRef typ;
478 :
479 15604 : v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
480 15604 : if (!v_srcvar)
481 0 : elog(ERROR, "function %s not in llvmjit_types.c", varname);
482 :
483 15604 : typ = LLVMGetFunctionType(v_srcvar);
484 :
485 15604 : return typ;
486 : }
487 :
488 : /*
489 : * Return declaration for a function referenced in llvmjit_types.c, adding it
490 : * to the module if necessary.
491 : *
492 : * This is used to make functions discovered via llvm_create_types() known to
493 : * the module that's currently being worked on.
494 : */
495 : LLVMValueRef
496 14224 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
497 : {
498 : LLVMValueRef v_srcfn;
499 : LLVMValueRef v_fn;
500 :
501 : /* don't repeatedly add function */
502 14224 : v_fn = LLVMGetNamedFunction(mod, funcname);
503 14224 : if (v_fn)
504 10290 : return v_fn;
505 :
506 3934 : v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
507 :
508 3934 : if (!v_srcfn)
509 0 : elog(ERROR, "function %s not in llvmjit_types.c", funcname);
510 :
511 3934 : v_fn = LLVMAddFunction(mod,
512 : funcname,
513 : LLVMGetFunctionType(v_srcfn));
514 3934 : llvm_copy_attributes(v_srcfn, v_fn);
515 :
516 3934 : return v_fn;
517 : }
518 :
519 : /*
520 : * Copy attributes from one function to another, for a specific index (an
521 : * index can reference return value, function and parameter attributes).
522 : */
523 : static void
524 71014 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
525 : {
526 : int num_attributes;
527 : LLVMAttributeRef *attrs;
528 :
529 71014 : num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
530 :
531 71014 : if (num_attributes == 0)
532 31446 : return;
533 :
534 39568 : attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
535 39568 : LLVMGetAttributesAtIndex(v_from, index, attrs);
536 :
537 464050 : for (int attno = 0; attno < num_attributes; attno++)
538 424482 : LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
539 :
540 39568 : pfree(attrs);
541 : }
542 :
543 : /*
544 : * Copy all attributes from one function to another. I.e. function, return and
545 : * parameters will be copied.
546 : */
547 : void
548 21708 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
549 : {
550 : uint32 param_count;
551 :
552 : /* copy function attributes */
553 21708 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
554 :
555 : /* and the return value attributes */
556 21708 : llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
557 :
558 : /* and each function parameter's attribute */
559 21708 : param_count = LLVMCountParams(v_from);
560 :
561 49306 : for (int paramidx = 1; paramidx <= param_count; paramidx++)
562 27598 : llvm_copy_attributes_at_index(v_from, v_to, paramidx);
563 21708 : }
564 :
565 : /*
566 : * Return a callable LLVMValueRef for fcinfo.
567 : */
568 : LLVMValueRef
569 5814 : llvm_function_reference(LLVMJitContext *context,
570 : LLVMBuilderRef builder,
571 : LLVMModuleRef mod,
572 : FunctionCallInfo fcinfo)
573 : {
574 : char *modname;
575 : char *basename;
576 : char *funcname;
577 :
578 : LLVMValueRef v_fn;
579 :
580 5814 : fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
581 :
582 5814 : if (modname != NULL && basename != NULL)
583 : {
584 : /* external function in loadable library */
585 20 : funcname = psprintf("pgextern.%s.%s", modname, basename);
586 : }
587 5794 : else if (basename != NULL)
588 : {
589 : /* internal function */
590 5782 : funcname = pstrdup(basename);
591 : }
592 : else
593 : {
594 : /*
595 : * Function we don't know to handle, return pointer. We do so by
596 : * creating a global constant containing a pointer to the function.
597 : * Makes IR more readable.
598 : */
599 : LLVMValueRef v_fn_addr;
600 :
601 12 : funcname = psprintf("pgoidextern.%u",
602 12 : fcinfo->flinfo->fn_oid);
603 12 : v_fn = LLVMGetNamedGlobal(mod, funcname);
604 12 : if (v_fn != 0)
605 0 : return l_load(builder, TypePGFunction, v_fn, "");
606 :
607 12 : v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
608 :
609 12 : v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
610 12 : LLVMSetInitializer(v_fn, v_fn_addr);
611 12 : LLVMSetGlobalConstant(v_fn, true);
612 12 : LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
613 12 : LLVMSetUnnamedAddr(v_fn, true);
614 :
615 12 : return l_load(builder, TypePGFunction, v_fn, "");
616 : }
617 :
618 : /* check if function already has been added */
619 5802 : v_fn = LLVMGetNamedFunction(mod, funcname);
620 5802 : if (v_fn != 0)
621 2504 : return v_fn;
622 :
623 3298 : v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
624 :
625 3298 : return v_fn;
626 : }
627 :
628 : /*
629 : * Optimize code in module using the flags set in context.
630 : */
631 : static void
632 1314 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
633 : {
634 : #if LLVM_VERSION_MAJOR < 17
635 : LLVMPassManagerBuilderRef llvm_pmb;
636 : LLVMPassManagerRef llvm_mpm;
637 : LLVMPassManagerRef llvm_fpm;
638 : LLVMValueRef func;
639 : int compile_optlevel;
640 :
641 1314 : if (context->base.flags & PGJIT_OPT3)
642 980 : compile_optlevel = 3;
643 : else
644 334 : compile_optlevel = 0;
645 :
646 : /*
647 : * Have to create a new pass manager builder every pass through, as the
648 : * inliner has some per-builder state. Otherwise one ends up only inlining
649 : * a function the first time though.
650 : */
651 1314 : llvm_pmb = LLVMPassManagerBuilderCreate();
652 1314 : LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
653 1314 : llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
654 :
655 1314 : if (context->base.flags & PGJIT_OPT3)
656 : {
657 : /* TODO: Unscientifically determined threshold */
658 980 : LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
659 : }
660 : else
661 : {
662 : /* we rely on mem2reg heavily, so emit even in the O0 case */
663 334 : LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
664 : }
665 :
666 1314 : LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
667 :
668 : /*
669 : * Do function level optimization. This could be moved to the point where
670 : * functions are emitted, to reduce memory usage a bit.
671 : */
672 1314 : LLVMInitializeFunctionPassManager(llvm_fpm);
673 31484 : for (func = LLVMGetFirstFunction(context->module);
674 : func != NULL;
675 30170 : func = LLVMGetNextFunction(func))
676 30170 : LLVMRunFunctionPassManager(llvm_fpm, func);
677 1314 : LLVMFinalizeFunctionPassManager(llvm_fpm);
678 1314 : LLVMDisposePassManager(llvm_fpm);
679 :
680 : /*
681 : * Perform module level optimization. We do so even in the non-optimized
682 : * case, so always-inline functions etc get inlined. It's cheap enough.
683 : */
684 1314 : llvm_mpm = LLVMCreatePassManager();
685 1314 : LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
686 : llvm_mpm);
687 : /* always use always-inliner pass */
688 1314 : if (!(context->base.flags & PGJIT_OPT3))
689 334 : LLVMAddAlwaysInlinerPass(llvm_mpm);
690 : /* if doing inlining, but no expensive optimization, add inlining pass */
691 1314 : if (context->base.flags & PGJIT_INLINE
692 980 : && !(context->base.flags & PGJIT_OPT3))
693 0 : LLVMAddFunctionInliningPass(llvm_mpm);
694 1314 : LLVMRunPassManager(llvm_mpm, context->module);
695 1314 : LLVMDisposePassManager(llvm_mpm);
696 :
697 1314 : LLVMPassManagerBuilderDispose(llvm_pmb);
698 : #else
699 : LLVMPassBuilderOptionsRef options;
700 : LLVMErrorRef err;
701 : const char *passes;
702 :
703 : if (context->base.flags & PGJIT_OPT3)
704 : passes = "default<O3>";
705 : else
706 : passes = "default<O0>,mem2reg";
707 :
708 : options = LLVMCreatePassBuilderOptions();
709 :
710 : #ifdef LLVM_PASS_DEBUG
711 : LLVMPassBuilderOptionsSetDebugLogging(options, 1);
712 : #endif
713 :
714 : LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
715 :
716 : err = LLVMRunPasses(module, passes, NULL, options);
717 :
718 : if (err)
719 : elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
720 :
721 : LLVMDisposePassBuilderOptions(options);
722 : #endif
723 1314 : }
724 :
725 : /*
726 : * Emit code for the currently pending module.
727 : */
728 : static void
729 1314 : llvm_compile_module(LLVMJitContext *context)
730 : {
731 : LLVMJitHandle *handle;
732 : MemoryContext oldcontext;
733 : instr_time starttime;
734 : instr_time endtime;
735 : #if LLVM_VERSION_MAJOR > 11
736 : LLVMOrcLLJITRef compile_orc;
737 : #else
738 : LLVMOrcJITStackRef compile_orc;
739 : #endif
740 :
741 1314 : if (context->base.flags & PGJIT_OPT3)
742 980 : compile_orc = llvm_opt3_orc;
743 : else
744 334 : compile_orc = llvm_opt0_orc;
745 :
746 : /* perform inlining */
747 1314 : if (context->base.flags & PGJIT_INLINE)
748 : {
749 980 : INSTR_TIME_SET_CURRENT(starttime);
750 980 : llvm_inline(context->module);
751 980 : INSTR_TIME_SET_CURRENT(endtime);
752 980 : INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
753 : endtime, starttime);
754 : }
755 :
756 1314 : if (jit_dump_bitcode)
757 : {
758 : char *filename;
759 :
760 0 : filename = psprintf("%d.%zu.bc",
761 : MyProcPid,
762 : context->module_generation);
763 0 : LLVMWriteBitcodeToFile(context->module, filename);
764 0 : pfree(filename);
765 : }
766 :
767 :
768 : /* optimize according to the chosen optimization settings */
769 1314 : INSTR_TIME_SET_CURRENT(starttime);
770 1314 : llvm_optimize_module(context, context->module);
771 1314 : INSTR_TIME_SET_CURRENT(endtime);
772 1314 : INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
773 : endtime, starttime);
774 :
775 1314 : if (jit_dump_bitcode)
776 : {
777 : char *filename;
778 :
779 0 : filename = psprintf("%d.%zu.optimized.bc",
780 : MyProcPid,
781 : context->module_generation);
782 0 : LLVMWriteBitcodeToFile(context->module, filename);
783 0 : pfree(filename);
784 : }
785 :
786 : handle = (LLVMJitHandle *)
787 1314 : MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
788 :
789 : /*
790 : * Emit the code. Note that this can, depending on the optimization
791 : * settings, take noticeable resources as code emission executes low-level
792 : * instruction combining/selection passes etc. Without optimization a
793 : * faster instruction selection mechanism is used.
794 : */
795 1314 : INSTR_TIME_SET_CURRENT(starttime);
796 : #if LLVM_VERSION_MAJOR > 11
797 : {
798 : LLVMOrcThreadSafeModuleRef ts_module;
799 : LLVMErrorRef error;
800 : LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
801 :
802 : ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
803 :
804 : handle->lljit = compile_orc;
805 : handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
806 :
807 : /*
808 : * NB: This doesn't actually emit code. That happens lazily the first
809 : * time a symbol defined in the module is requested. Due to that
810 : * llvm_get_function() also accounts for emission time.
811 : */
812 :
813 : context->module = NULL; /* will be owned by LLJIT */
814 : error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
815 : handle->resource_tracker,
816 : ts_module);
817 :
818 : if (error)
819 : elog(ERROR, "failed to JIT module: %s",
820 : llvm_error_message(error));
821 :
822 : /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
823 : }
824 : #else
825 : {
826 1314 : handle->stack = compile_orc;
827 1314 : if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
828 : llvm_resolve_symbol, NULL))
829 0 : elog(ERROR, "failed to JIT module");
830 :
831 : /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
832 : }
833 : #endif
834 :
835 1314 : INSTR_TIME_SET_CURRENT(endtime);
836 1314 : INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
837 : endtime, starttime);
838 :
839 1314 : context->module = NULL;
840 1314 : context->compiled = true;
841 :
842 : /* remember emitted code for cleanup and lookups */
843 1314 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
844 1314 : context->handles = lappend(context->handles, handle);
845 1314 : MemoryContextSwitchTo(oldcontext);
846 :
847 1314 : ereport(DEBUG1,
848 : (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
849 : INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
850 : INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
851 : INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
852 : errhidestmt(true),
853 : errhidecontext(true)));
854 1314 : }
855 :
856 : /*
857 : * Per session initialization.
858 : */
859 : static void
860 1774 : llvm_session_initialize(void)
861 : {
862 : MemoryContext oldcontext;
863 1774 : char *error = NULL;
864 1774 : char *cpu = NULL;
865 1774 : char *features = NULL;
866 : LLVMTargetMachineRef opt0_tm;
867 : LLVMTargetMachineRef opt3_tm;
868 :
869 1774 : if (llvm_session_initialized)
870 976 : return;
871 :
872 798 : oldcontext = MemoryContextSwitchTo(TopMemoryContext);
873 :
874 798 : LLVMInitializeNativeTarget();
875 798 : LLVMInitializeNativeAsmPrinter();
876 798 : LLVMInitializeNativeAsmParser();
877 :
878 798 : if (llvm_context == NULL)
879 : {
880 798 : llvm_context = LLVMContextCreate();
881 :
882 798 : llvm_jit_context_in_use_count = 0;
883 798 : llvm_llvm_context_reuse_count = 0;
884 : }
885 :
886 : /*
887 : * When targeting LLVM 15, turn off opaque pointers for the context we
888 : * build our code in. We don't need to do so for other contexts (e.g.
889 : * llvm_ts_context). Once the IR is generated, it carries the necessary
890 : * information.
891 : *
892 : * For 16 and above, opaque pointers must be used, and we have special
893 : * code for that.
894 : */
895 : #if LLVM_VERSION_MAJOR == 15
896 : LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
897 : #endif
898 :
899 : /*
900 : * Synchronize types early, as that also includes inferring the target
901 : * triple.
902 : */
903 798 : llvm_create_types();
904 :
905 : /*
906 : * Extract target information from loaded module.
907 : */
908 798 : llvm_set_target();
909 :
910 798 : if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
911 : {
912 0 : elog(FATAL, "failed to query triple %s", error);
913 : }
914 :
915 : /*
916 : * We want the generated code to use all available features. Therefore
917 : * grab the host CPU string and detect features of the current CPU. The
918 : * latter is needed because some CPU architectures default to enabling
919 : * features not all CPUs have (weird, huh).
920 : */
921 798 : cpu = LLVMGetHostCPUName();
922 798 : features = LLVMGetHostCPUFeatures();
923 798 : elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
924 : cpu, features);
925 :
926 : opt0_tm =
927 798 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
928 : LLVMCodeGenLevelNone,
929 : LLVMRelocDefault,
930 : LLVMCodeModelJITDefault);
931 : opt3_tm =
932 798 : LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
933 : LLVMCodeGenLevelAggressive,
934 : LLVMRelocDefault,
935 : LLVMCodeModelJITDefault);
936 :
937 798 : LLVMDisposeMessage(cpu);
938 798 : cpu = NULL;
939 798 : LLVMDisposeMessage(features);
940 798 : features = NULL;
941 :
942 : /* force symbols in main binary to be loaded */
943 798 : LLVMLoadLibraryPermanently(NULL);
944 :
945 : #if LLVM_VERSION_MAJOR > 11
946 : {
947 : llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
948 :
949 : llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
950 : opt0_tm = 0;
951 :
952 : llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
953 : opt3_tm = 0;
954 : }
955 : #else /* LLVM_VERSION_MAJOR > 11 */
956 : {
957 798 : llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
958 798 : llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
959 :
960 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
961 798 : if (jit_debugging_support)
962 : {
963 0 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
964 :
965 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
966 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
967 : }
968 : #endif
969 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
970 798 : if (jit_profiling_support)
971 : {
972 0 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
973 :
974 0 : LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
975 0 : LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
976 : }
977 : #endif
978 : }
979 : #endif /* LLVM_VERSION_MAJOR > 11 */
980 :
981 798 : on_proc_exit(llvm_shutdown, 0);
982 :
983 798 : llvm_session_initialized = true;
984 :
985 798 : MemoryContextSwitchTo(oldcontext);
986 : }
987 :
988 : static void
989 798 : llvm_shutdown(int code, Datum arg)
990 : {
991 : /*
992 : * If llvm_shutdown() is reached while in a fatal-on-oom section an error
993 : * has occurred in the middle of LLVM code. It is not safe to call back
994 : * into LLVM (which is why a FATAL error was thrown).
995 : *
996 : * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
997 : * profiling data won't be written out.
998 : */
999 798 : if (llvm_in_fatal_on_oom())
1000 : {
1001 : Assert(proc_exit_inprogress);
1002 0 : return;
1003 : }
1004 :
1005 798 : if (llvm_jit_context_in_use_count != 0)
1006 0 : elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
1007 : llvm_jit_context_in_use_count);
1008 :
1009 : #if LLVM_VERSION_MAJOR > 11
1010 : {
1011 : if (llvm_opt3_orc)
1012 : {
1013 : LLVMOrcDisposeLLJIT(llvm_opt3_orc);
1014 : llvm_opt3_orc = NULL;
1015 : }
1016 : if (llvm_opt0_orc)
1017 : {
1018 : LLVMOrcDisposeLLJIT(llvm_opt0_orc);
1019 : llvm_opt0_orc = NULL;
1020 : }
1021 : if (llvm_ts_context)
1022 : {
1023 : LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
1024 : llvm_ts_context = NULL;
1025 : }
1026 : }
1027 : #else /* LLVM_VERSION_MAJOR > 11 */
1028 : {
1029 : /* unregister profiling support, needs to be flushed to be useful */
1030 :
1031 798 : if (llvm_opt3_orc)
1032 : {
1033 : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1034 : if (jit_profiling_support)
1035 : LLVMOrcUnregisterPerf(llvm_opt3_orc);
1036 : #endif
1037 798 : LLVMOrcDisposeInstance(llvm_opt3_orc);
1038 798 : llvm_opt3_orc = NULL;
1039 : }
1040 :
1041 798 : if (llvm_opt0_orc)
1042 : {
1043 : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
1044 : if (jit_profiling_support)
1045 : LLVMOrcUnregisterPerf(llvm_opt0_orc);
1046 : #endif
1047 798 : LLVMOrcDisposeInstance(llvm_opt0_orc);
1048 798 : llvm_opt0_orc = NULL;
1049 : }
1050 : }
1051 : #endif /* LLVM_VERSION_MAJOR > 11 */
1052 : }
1053 :
1054 : /* helper for llvm_create_types, returning a function's return type */
1055 : static LLVMTypeRef
1056 798 : load_return_type(LLVMModuleRef mod, const char *name)
1057 : {
1058 : LLVMValueRef value;
1059 : LLVMTypeRef typ;
1060 :
1061 : /* this'll return a *pointer* to the function */
1062 798 : value = LLVMGetNamedFunction(mod, name);
1063 798 : if (!value)
1064 0 : elog(ERROR, "function %s is unknown", name);
1065 :
1066 798 : typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
1067 :
1068 798 : return typ;
1069 : }
1070 :
1071 : /*
1072 : * Load triple & layout from clang emitted file so we're guaranteed to be
1073 : * compatible.
1074 : */
1075 : static void
1076 798 : llvm_set_target(void)
1077 : {
1078 798 : if (!llvm_types_module)
1079 0 : elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
1080 :
1081 798 : if (llvm_triple == NULL)
1082 798 : llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
1083 :
1084 798 : if (llvm_layout == NULL)
1085 798 : llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
1086 798 : }
1087 :
1088 : /*
1089 : * Load required information, types, function signatures from llvmjit_types.c
1090 : * and make them available in global variables.
1091 : *
1092 : * Those global variables are then used while emitting code.
1093 : */
1094 : static void
1095 798 : llvm_create_types(void)
1096 : {
1097 : char path[MAXPGPATH];
1098 : LLVMMemoryBufferRef buf;
1099 : char *msg;
1100 :
1101 798 : snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
1102 :
1103 : /* open file */
1104 798 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
1105 : {
1106 0 : elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
1107 : path, msg);
1108 : }
1109 :
1110 : /* eagerly load contents, going to need it all */
1111 798 : if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
1112 : {
1113 0 : elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
1114 : }
1115 798 : LLVMDisposeMemoryBuffer(buf);
1116 :
1117 798 : TypeSizeT = llvm_pg_var_type("TypeSizeT");
1118 798 : TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
1119 798 : TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
1120 798 : TypePGFunction = llvm_pg_var_type("TypePGFunction");
1121 798 : StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
1122 798 : StructExprContext = llvm_pg_var_type("StructExprContext");
1123 798 : StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
1124 798 : StructExprState = llvm_pg_var_type("StructExprState");
1125 798 : StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
1126 798 : StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
1127 798 : StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
1128 798 : StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
1129 798 : StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
1130 798 : StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
1131 798 : StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
1132 798 : StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
1133 798 : StructAggState = llvm_pg_var_type("StructAggState");
1134 798 : StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
1135 798 : StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
1136 798 : StructPlanState = llvm_pg_var_type("StructPlanState");
1137 798 : StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
1138 :
1139 798 : AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
1140 798 : ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
1141 798 : ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
1142 798 : }
1143 :
1144 : /*
1145 : * Split a symbol into module / function parts. If the function is in the
1146 : * main binary (or an external library) *modname will be NULL.
1147 : */
1148 : void
1149 31768 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
1150 : {
1151 31768 : *modname = NULL;
1152 31768 : *funcname = NULL;
1153 :
1154 : /*
1155 : * Module function names are pgextern.$module.$funcname
1156 : */
1157 31768 : if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
1158 : {
1159 : /*
1160 : * Symbol names cannot contain a ., therefore we can split based on
1161 : * first and last occurrence of one.
1162 : */
1163 58 : *funcname = rindex(name, '.');
1164 58 : (*funcname)++; /* jump over . */
1165 :
1166 116 : *modname = pnstrdup(name + strlen("pgextern."),
1167 58 : *funcname - name - strlen("pgextern.") - 1);
1168 : Assert(funcname);
1169 :
1170 58 : *funcname = pstrdup(*funcname);
1171 : }
1172 : else
1173 : {
1174 31710 : *modname = NULL;
1175 31710 : *funcname = pstrdup(name);
1176 : }
1177 31768 : }
1178 :
1179 : /*
1180 : * Attempt to resolve symbol, so LLVM can emit a reference to it.
1181 : */
1182 : static uint64_t
1183 5212 : llvm_resolve_symbol(const char *symname, void *ctx)
1184 : {
1185 : uintptr_t addr;
1186 : char *funcname;
1187 : char *modname;
1188 :
1189 : /*
1190 : * macOS prefixes all object level symbols with an underscore. But neither
1191 : * dlsym() nor PG's inliner expect that. So undo.
1192 : */
1193 : #if defined(__darwin__)
1194 : if (symname[0] != '_')
1195 : elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
1196 : symname++;
1197 : #endif
1198 :
1199 5212 : llvm_split_symbol_name(symname, &modname, &funcname);
1200 :
1201 : /* functions that aren't resolved to names shouldn't ever get here */
1202 : Assert(funcname);
1203 :
1204 5212 : if (modname)
1205 20 : addr = (uintptr_t) load_external_function(modname, funcname,
1206 : true, NULL);
1207 : else
1208 5192 : addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
1209 :
1210 5212 : pfree(funcname);
1211 5212 : if (modname)
1212 20 : pfree(modname);
1213 :
1214 : /* let LLVM will error out - should never happen */
1215 5212 : if (!addr)
1216 0 : elog(WARNING, "failed to resolve name %s", symname);
1217 :
1218 5212 : return (uint64_t) addr;
1219 : }
1220 :
1221 : #if LLVM_VERSION_MAJOR > 11
1222 :
1223 : static LLVMErrorRef
1224 : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
1225 : LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
1226 : LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
1227 : LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
1228 : {
1229 : #if LLVM_VERSION_MAJOR > 14
1230 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
1231 : #else
1232 : LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
1233 : #endif
1234 : LLVMErrorRef error;
1235 : LLVMOrcMaterializationUnitRef mu;
1236 :
1237 : for (int i = 0; i < LookupSetSize; i++)
1238 : {
1239 : const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
1240 :
1241 : #if LLVM_VERSION_MAJOR > 12
1242 : LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
1243 : #endif
1244 : symbols[i].Name = LookupSet[i].Name;
1245 : symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
1246 : symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
1247 : }
1248 :
1249 : mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
1250 : error = LLVMOrcJITDylibDefine(JD, mu);
1251 : if (error != LLVMErrorSuccess)
1252 : LLVMOrcDisposeMaterializationUnit(mu);
1253 :
1254 : pfree(symbols);
1255 :
1256 : return error;
1257 : }
1258 :
1259 : /*
1260 : * We cannot throw errors through LLVM (without causing a FATAL at least), so
1261 : * just use WARNING here. That's OK anyway, as the error is also reported at
1262 : * the top level action (with less detail) and there might be multiple
1263 : * invocations of errors with details.
1264 : *
1265 : * This doesn't really happen during normal operation, but in cases like
1266 : * symbol resolution breakage. So just using elog(WARNING) is fine.
1267 : */
1268 : static void
1269 : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
1270 : {
1271 : elog(WARNING, "error during JITing: %s",
1272 : llvm_error_message(error));
1273 : }
1274 :
1275 : /*
1276 : * Create our own object layer, so we can add event listeners.
1277 : */
1278 : static LLVMOrcObjectLayerRef
1279 : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
1280 : {
1281 : LLVMOrcObjectLayerRef objlayer =
1282 : LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
1283 :
1284 : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
1285 : if (jit_debugging_support)
1286 : {
1287 : LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
1288 :
1289 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1290 : }
1291 : #endif
1292 :
1293 : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
1294 : if (jit_profiling_support)
1295 : {
1296 : LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
1297 :
1298 : LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
1299 : }
1300 : #endif
1301 :
1302 : return objlayer;
1303 : }
1304 :
1305 : /*
1306 : * Create LLJIT instance, using the passed in target machine. Note that the
1307 : * target machine afterwards is owned by the LLJIT instance.
1308 : */
1309 : static LLVMOrcLLJITRef
1310 : llvm_create_jit_instance(LLVMTargetMachineRef tm)
1311 : {
1312 : LLVMOrcLLJITRef lljit;
1313 : LLVMOrcJITTargetMachineBuilderRef tm_builder;
1314 : LLVMOrcLLJITBuilderRef lljit_builder;
1315 : LLVMErrorRef error;
1316 : LLVMOrcDefinitionGeneratorRef main_gen;
1317 : LLVMOrcDefinitionGeneratorRef ref_gen;
1318 :
1319 : lljit_builder = LLVMOrcCreateLLJITBuilder();
1320 : tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
1321 : LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
1322 :
1323 : LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
1324 : llvm_create_object_layer,
1325 : NULL);
1326 :
1327 : error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
1328 : if (error)
1329 : elog(ERROR, "failed to create lljit instance: %s",
1330 : llvm_error_message(error));
1331 :
1332 : LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
1333 : llvm_log_jit_error, NULL);
1334 :
1335 : /*
1336 : * Symbol resolution support for symbols in the postgres binary /
1337 : * libraries already loaded.
1338 : */
1339 : error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
1340 : LLVMOrcLLJITGetGlobalPrefix(lljit),
1341 : 0, NULL);
1342 : if (error)
1343 : elog(ERROR, "failed to create generator: %s",
1344 : llvm_error_message(error));
1345 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
1346 :
1347 : /*
1348 : * Symbol resolution support for "special" functions, e.g. a call into an
1349 : * SQL callable function.
1350 : */
1351 : #if LLVM_VERSION_MAJOR > 14
1352 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
1353 : #else
1354 : ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
1355 : #endif
1356 : LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
1357 :
1358 : return lljit;
1359 : }
1360 :
1361 : static char *
1362 : llvm_error_message(LLVMErrorRef error)
1363 : {
1364 : char *orig = LLVMGetErrorMessage(error);
1365 : char *msg = pstrdup(orig);
1366 :
1367 : LLVMDisposeErrorMessage(orig);
1368 :
1369 : return msg;
1370 : }
1371 :
1372 : #endif /* LLVM_VERSION_MAJOR > 11 */
1373 :
1374 : /*
1375 : * ResourceOwner callbacks
1376 : */
1377 : static void
1378 16 : ResOwnerReleaseJitContext(Datum res)
1379 : {
1380 16 : JitContext *context = (JitContext *) DatumGetPointer(res);
1381 :
1382 16 : context->resowner = NULL;
1383 16 : jit_release_context(context);
1384 16 : }
|