LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 294 331 88.8 %
Date: 2024-04-25 01:13:18 Functions: 26 26 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit.c
       4             :  *    Core part of the LLVM JIT provider.
       5             :  *
       6             :  * Copyright (c) 2016-2024, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/jit/llvm/llvmjit.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres.h"
      15             : 
      16             : #include <llvm-c/Analysis.h>
      17             : #include <llvm-c/BitReader.h>
      18             : #include <llvm-c/BitWriter.h>
      19             : #include <llvm-c/Core.h>
      20             : #include <llvm-c/ExecutionEngine.h>
      21             : #if LLVM_VERSION_MAJOR > 16
      22             : #include <llvm-c/Transforms/PassBuilder.h>
      23             : #endif
      24             : #if LLVM_VERSION_MAJOR > 11
      25             : #include <llvm-c/Orc.h>
      26             : #include <llvm-c/OrcEE.h>
      27             : #include <llvm-c/LLJIT.h>
      28             : #else
      29             : #include <llvm-c/OrcBindings.h>
      30             : #endif
      31             : #include <llvm-c/Support.h>
      32             : #include <llvm-c/Target.h>
      33             : #if LLVM_VERSION_MAJOR < 17
      34             : #include <llvm-c/Transforms/IPO.h>
      35             : #include <llvm-c/Transforms/PassManagerBuilder.h>
      36             : #include <llvm-c/Transforms/Scalar.h>
      37             : #include <llvm-c/Transforms/Utils.h>
      38             : #endif
      39             : 
      40             : #include "jit/llvmjit.h"
      41             : #include "jit/llvmjit_emit.h"
      42             : #include "miscadmin.h"
      43             : #include "portability/instr_time.h"
      44             : #include "storage/ipc.h"
      45             : #include "utils/memutils.h"
      46             : #include "utils/resowner.h"
      47             : 
      48             : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
      49             : 
      50             : /* Handle of a module emitted via ORC JIT */
      51             : typedef struct LLVMJitHandle
      52             : {
      53             : #if LLVM_VERSION_MAJOR > 11
      54             :     LLVMOrcLLJITRef lljit;
      55             :     LLVMOrcResourceTrackerRef resource_tracker;
      56             : #else
      57             :     LLVMOrcJITStackRef stack;
      58             :     LLVMOrcModuleHandle orc_handle;
      59             : #endif
      60             : } LLVMJitHandle;
      61             : 
      62             : 
      63             : /* types & functions commonly needed for JITing */
      64             : LLVMTypeRef TypeSizeT;
      65             : LLVMTypeRef TypeParamBool;
      66             : LLVMTypeRef TypeStorageBool;
      67             : LLVMTypeRef TypePGFunction;
      68             : LLVMTypeRef StructNullableDatum;
      69             : LLVMTypeRef StructHeapTupleData;
      70             : LLVMTypeRef StructMinimalTupleData;
      71             : LLVMTypeRef StructTupleDescData;
      72             : LLVMTypeRef StructTupleTableSlot;
      73             : LLVMTypeRef StructHeapTupleHeaderData;
      74             : LLVMTypeRef StructHeapTupleTableSlot;
      75             : LLVMTypeRef StructMinimalTupleTableSlot;
      76             : LLVMTypeRef StructMemoryContextData;
      77             : LLVMTypeRef StructFunctionCallInfoData;
      78             : LLVMTypeRef StructExprContext;
      79             : LLVMTypeRef StructExprEvalStep;
      80             : LLVMTypeRef StructExprState;
      81             : LLVMTypeRef StructAggState;
      82             : LLVMTypeRef StructAggStatePerGroupData;
      83             : LLVMTypeRef StructAggStatePerTransData;
      84             : LLVMTypeRef StructPlanState;
      85             : 
      86             : LLVMValueRef AttributeTemplate;
      87             : LLVMValueRef ExecEvalSubroutineTemplate;
      88             : LLVMValueRef ExecEvalBoolSubroutineTemplate;
      89             : 
      90             : static LLVMModuleRef llvm_types_module = NULL;
      91             : 
      92             : static bool llvm_session_initialized = false;
      93             : static size_t llvm_generation = 0;
      94             : 
      95             : /* number of LLVMJitContexts that currently are in use */
      96             : static size_t llvm_jit_context_in_use_count = 0;
      97             : 
      98             : /* how many times has the current LLVMContextRef been used */
      99             : static size_t llvm_llvm_context_reuse_count = 0;
     100             : static const char *llvm_triple = NULL;
     101             : static const char *llvm_layout = NULL;
     102             : static LLVMContextRef llvm_context;
     103             : 
     104             : 
     105             : static LLVMTargetRef llvm_targetref;
     106             : #if LLVM_VERSION_MAJOR > 11
     107             : static LLVMOrcThreadSafeContextRef llvm_ts_context;
     108             : static LLVMOrcLLJITRef llvm_opt0_orc;
     109             : static LLVMOrcLLJITRef llvm_opt3_orc;
     110             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     111             : static LLVMOrcJITStackRef llvm_opt0_orc;
     112             : static LLVMOrcJITStackRef llvm_opt3_orc;
     113             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     114             : 
     115             : 
     116             : static void llvm_release_context(JitContext *context);
     117             : static void llvm_session_initialize(void);
     118             : static void llvm_shutdown(int code, Datum arg);
     119             : static void llvm_compile_module(LLVMJitContext *context);
     120             : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
     121             : 
     122             : static void llvm_create_types(void);
     123             : static void llvm_set_target(void);
     124             : static void llvm_recreate_llvm_context(void);
     125             : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
     126             : 
     127             : #if LLVM_VERSION_MAJOR > 11
     128             : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
     129             : static char *llvm_error_message(LLVMErrorRef error);
     130             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     131             : 
     132             : /* ResourceOwner callbacks to hold JitContexts  */
     133             : static void ResOwnerReleaseJitContext(Datum res);
     134             : 
     135             : static const ResourceOwnerDesc jit_resowner_desc =
     136             : {
     137             :     .name = "LLVM JIT context",
     138             :     .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
     139             :     .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
     140             :     .ReleaseResource = ResOwnerReleaseJitContext,
     141             :     .DebugPrint = NULL          /* the default message is fine */
     142             : };
     143             : 
     144             : /* Convenience wrappers over ResourceOwnerRemember/Forget */
     145             : static inline void
     146        1776 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
     147             : {
     148        1776 :     ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
     149        1776 : }
     150             : static inline void
     151        1760 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
     152             : {
     153        1760 :     ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
     154        1760 : }
     155             : 
     156        2012 : PG_MODULE_MAGIC;
     157             : 
     158             : 
     159             : /*
     160             :  * Initialize LLVM JIT provider.
     161             :  */
     162             : void
     163         798 : _PG_jit_provider_init(JitProviderCallbacks *cb)
     164             : {
     165         798 :     cb->reset_after_error = llvm_reset_after_error;
     166         798 :     cb->release_context = llvm_release_context;
     167         798 :     cb->compile_expr = llvm_compile_expr;
     168         798 : }
     169             : 
     170             : 
     171             : /*
     172             :  * Every now and then create a new LLVMContextRef. Unfortunately, during every
     173             :  * round of inlining, types may "leak" (they can still be found/used via the
     174             :  * context, but new types will be created the next time in inlining is
     175             :  * performed). To prevent that from slowly accumulating problematic amounts of
     176             :  * memory, recreate the LLVMContextRef we use. We don't want to do so too
     177             :  * often, as that implies some overhead (particularly re-loading the module
     178             :  * summaries / modules is fairly expensive). A future TODO would be to make
     179             :  * this more finegrained and only drop/recreate the LLVMContextRef when we know
     180             :  * there has been inlining. If we can get the size of the context from LLVM
     181             :  * then that might be a better way to determine when to drop/recreate rather
     182             :  * then the usagecount heuristic currently employed.
     183             :  */
     184             : static void
     185        1776 : llvm_recreate_llvm_context(void)
     186             : {
     187        1776 :     if (!llvm_context)
     188           0 :         elog(ERROR, "Trying to recreate a non-existing context");
     189             : 
     190             :     /*
     191             :      * We can only safely recreate the LLVM context if no other code is being
     192             :      * JITed, otherwise we'd release the types in use for that.
     193             :      */
     194        1776 :     if (llvm_jit_context_in_use_count > 0)
     195             :     {
     196           0 :         llvm_llvm_context_reuse_count++;
     197           0 :         return;
     198             :     }
     199             : 
     200        1776 :     if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
     201             :     {
     202        1776 :         llvm_llvm_context_reuse_count++;
     203        1776 :         return;
     204             :     }
     205             : 
     206             :     /*
     207             :      * Need to reset the modules that the inlining code caches before
     208             :      * disposing of the context. LLVM modules exist within a specific LLVM
     209             :      * context, therefore disposing of the context before resetting the cache
     210             :      * would lead to dangling pointers to modules.
     211             :      */
     212           0 :     llvm_inline_reset_caches();
     213             : 
     214           0 :     LLVMContextDispose(llvm_context);
     215           0 :     llvm_context = LLVMContextCreate();
     216           0 :     llvm_llvm_context_reuse_count = 0;
     217             : 
     218             :     /*
     219             :      * Re-build cached type information, so code generation code can rely on
     220             :      * that information to be present (also prevents the variables to be
     221             :      * dangling references).
     222             :      */
     223           0 :     llvm_create_types();
     224             : }
     225             : 
     226             : 
     227             : /*
     228             :  * Create a context for JITing work.
     229             :  *
     230             :  * The context, including subsidiary resources, will be cleaned up either when
     231             :  * the context is explicitly released, or when the lifetime of
     232             :  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
     233             :  */
     234             : LLVMJitContext *
     235        1776 : llvm_create_context(int jitFlags)
     236             : {
     237             :     LLVMJitContext *context;
     238             : 
     239        1776 :     llvm_assert_in_fatal_section();
     240             : 
     241        1776 :     llvm_session_initialize();
     242             : 
     243        1776 :     llvm_recreate_llvm_context();
     244             : 
     245        1776 :     ResourceOwnerEnlarge(CurrentResourceOwner);
     246             : 
     247        1776 :     context = MemoryContextAllocZero(TopMemoryContext,
     248             :                                      sizeof(LLVMJitContext));
     249        1776 :     context->base.flags = jitFlags;
     250             : 
     251             :     /* ensure cleanup */
     252        1776 :     context->base.resowner = CurrentResourceOwner;
     253        1776 :     ResourceOwnerRememberJIT(CurrentResourceOwner, context);
     254             : 
     255        1776 :     llvm_jit_context_in_use_count++;
     256             : 
     257        1776 :     return context;
     258             : }
     259             : 
     260             : /*
     261             :  * Release resources required by one llvm context.
     262             :  */
     263             : static void
     264        1776 : llvm_release_context(JitContext *context)
     265             : {
     266        1776 :     LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
     267             :     ListCell   *lc;
     268             : 
     269             :     /*
     270             :      * Consider as cleaned up even if we skip doing so below, that way we can
     271             :      * verify the tracking is correct (see llvm_shutdown()).
     272             :      */
     273        1776 :     llvm_jit_context_in_use_count--;
     274             : 
     275             :     /*
     276             :      * When this backend is exiting, don't clean up LLVM. As an error might
     277             :      * have occurred from within LLVM, we do not want to risk reentering. All
     278             :      * resource cleanup is going to happen through process exit.
     279             :      */
     280        1776 :     if (proc_exit_inprogress)
     281           0 :         return;
     282             : 
     283        1776 :     llvm_enter_fatal_on_oom();
     284             : 
     285        1776 :     if (llvm_jit_context->module)
     286             :     {
     287         582 :         LLVMDisposeModule(llvm_jit_context->module);
     288         582 :         llvm_jit_context->module = NULL;
     289             :     }
     290             : 
     291        3092 :     foreach(lc, llvm_jit_context->handles)
     292             :     {
     293        1316 :         LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
     294             : 
     295             : #if LLVM_VERSION_MAJOR > 11
     296             :         {
     297             :             LLVMOrcExecutionSessionRef ee;
     298             :             LLVMOrcSymbolStringPoolRef sp;
     299             : 
     300             :             LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
     301             :             LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
     302             : 
     303             :             /*
     304             :              * Without triggering cleanup of the string pool, we'd leak
     305             :              * memory. It'd be sufficient to do this far less often, but in
     306             :              * experiments the required time was small enough to just always
     307             :              * do it.
     308             :              */
     309             :             ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
     310             :             sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
     311             :             LLVMOrcSymbolStringPoolClearDeadEntries(sp);
     312             :         }
     313             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     314             :         {
     315        1316 :             LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
     316             :         }
     317             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     318             : 
     319        1316 :         pfree(jit_handle);
     320             :     }
     321        1776 :     list_free(llvm_jit_context->handles);
     322        1776 :     llvm_jit_context->handles = NIL;
     323             : 
     324        1776 :     llvm_leave_fatal_on_oom();
     325             : 
     326        1776 :     if (context->resowner)
     327        1760 :         ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
     328             : }
     329             : 
     330             : /*
     331             :  * Return module which may be modified, e.g. by creating new functions.
     332             :  */
     333             : LLVMModuleRef
     334       17866 : llvm_mutable_module(LLVMJitContext *context)
     335             : {
     336       17866 :     llvm_assert_in_fatal_section();
     337             : 
     338             :     /*
     339             :      * If there's no in-progress module, create a new one.
     340             :      */
     341       17866 :     if (!context->module)
     342             :     {
     343        1898 :         context->compiled = false;
     344        1898 :         context->module_generation = llvm_generation++;
     345        1898 :         context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
     346        1898 :         LLVMSetTarget(context->module, llvm_triple);
     347        1898 :         LLVMSetDataLayout(context->module, llvm_layout);
     348             :     }
     349             : 
     350       17866 :     return context->module;
     351             : }
     352             : 
     353             : /*
     354             :  * Expand function name to be non-conflicting. This should be used by code
     355             :  * generating code, when adding new externally visible function definitions to
     356             :  * a Module.
     357             :  */
     358             : char *
     359       17866 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
     360             : {
     361             :     Assert(context->module != NULL);
     362             : 
     363       17866 :     context->base.instr.created_functions++;
     364             : 
     365             :     /*
     366             :      * Previously we used dots to separate, but turns out some tools, e.g.
     367             :      * GDB, don't like that and truncate name.
     368             :      */
     369       35732 :     return psprintf("%s_%zu_%d",
     370             :                     basename,
     371             :                     context->module_generation,
     372       17866 :                     context->counter++);
     373             : }
     374             : 
     375             : /*
     376             :  * Return pointer to function funcname, which has to exist. If there's pending
     377             :  * code to be optimized and emitted, do so first.
     378             :  */
     379             : void *
     380        6264 : llvm_get_function(LLVMJitContext *context, const char *funcname)
     381             : {
     382             :     ListCell   *lc;
     383             : 
     384        6264 :     llvm_assert_in_fatal_section();
     385             : 
     386             :     /*
     387             :      * If there is a pending / not emitted module, compile and emit now.
     388             :      * Otherwise we might not find the [correct] function.
     389             :      */
     390        6264 :     if (!context->compiled)
     391             :     {
     392        1316 :         llvm_compile_module(context);
     393             :     }
     394             : 
     395             :     /*
     396             :      * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
     397             :      * to mangle here.
     398             :      */
     399             : 
     400             : #if LLVM_VERSION_MAJOR > 11
     401             :     foreach(lc, context->handles)
     402             :     {
     403             :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     404             :         instr_time  starttime;
     405             :         instr_time  endtime;
     406             :         LLVMErrorRef error;
     407             :         LLVMOrcJITTargetAddress addr;
     408             : 
     409             :         INSTR_TIME_SET_CURRENT(starttime);
     410             : 
     411             :         addr = 0;
     412             :         error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
     413             :         if (error)
     414             :             elog(ERROR, "failed to look up symbol \"%s\": %s",
     415             :                  funcname, llvm_error_message(error));
     416             : 
     417             :         /*
     418             :          * LLJIT only actually emits code the first time a symbol is
     419             :          * referenced. Thus add lookup time to emission time. That's counting
     420             :          * a bit more than with older LLVM versions, but unlikely to ever
     421             :          * matter.
     422             :          */
     423             :         INSTR_TIME_SET_CURRENT(endtime);
     424             :         INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     425             :                               endtime, starttime);
     426             : 
     427             :         if (addr)
     428             :             return (void *) (uintptr_t) addr;
     429             :     }
     430             : #else
     431        6398 :     foreach(lc, context->handles)
     432             :     {
     433             :         LLVMOrcTargetAddress addr;
     434        6398 :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     435             : 
     436        6398 :         addr = 0;
     437        6398 :         if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
     438           0 :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     439        6398 :         if (addr)
     440        6264 :             return (void *) (uintptr_t) addr;
     441             :     }
     442             : #endif
     443             : 
     444           0 :     elog(ERROR, "failed to JIT: %s", funcname);
     445             : 
     446             :     return NULL;
     447             : }
     448             : 
     449             : /*
     450             :  * Return type of a variable in llvmjit_types.c. This is useful to keep types
     451             :  * in sync between plain C and JIT related code.
     452             :  */
     453             : LLVMTypeRef
     454       16078 : llvm_pg_var_type(const char *varname)
     455             : {
     456             :     LLVMValueRef v_srcvar;
     457             :     LLVMTypeRef typ;
     458             : 
     459             :     /* this'll return a *pointer* to the global */
     460       16078 :     v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
     461       16078 :     if (!v_srcvar)
     462           0 :         elog(ERROR, "variable %s not in llvmjit_types.c", varname);
     463             : 
     464       16078 :     typ = LLVMGlobalGetValueType(v_srcvar);
     465             : 
     466       16078 :     return typ;
     467             : }
     468             : 
     469             : /*
     470             :  * Return function type of a variable in llvmjit_types.c. This is useful to
     471             :  * keep function types in sync between C and JITed code.
     472             :  */
     473             : LLVMTypeRef
     474       15658 : llvm_pg_var_func_type(const char *varname)
     475             : {
     476             :     LLVMValueRef v_srcvar;
     477             :     LLVMTypeRef typ;
     478             : 
     479       15658 :     v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
     480       15658 :     if (!v_srcvar)
     481           0 :         elog(ERROR, "function %s not in llvmjit_types.c", varname);
     482             : 
     483       15658 :     typ = LLVMGetFunctionType(v_srcvar);
     484             : 
     485       15658 :     return typ;
     486             : }
     487             : 
     488             : /*
     489             :  * Return declaration for a function referenced in llvmjit_types.c, adding it
     490             :  * to the module if necessary.
     491             :  *
     492             :  * This is used to make functions discovered via llvm_create_types() known to
     493             :  * the module that's currently being worked on.
     494             :  */
     495             : LLVMValueRef
     496       14232 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
     497             : {
     498             :     LLVMValueRef v_srcfn;
     499             :     LLVMValueRef v_fn;
     500             : 
     501             :     /* don't repeatedly add function */
     502       14232 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     503       14232 :     if (v_fn)
     504       10294 :         return v_fn;
     505             : 
     506        3938 :     v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
     507             : 
     508        3938 :     if (!v_srcfn)
     509           0 :         elog(ERROR, "function %s not in llvmjit_types.c", funcname);
     510             : 
     511        3938 :     v_fn = LLVMAddFunction(mod,
     512             :                            funcname,
     513             :                            LLVMGetFunctionType(v_srcfn));
     514        3938 :     llvm_copy_attributes(v_srcfn, v_fn);
     515             : 
     516        3938 :     return v_fn;
     517             : }
     518             : 
     519             : /*
     520             :  * Copy attributes from one function to another, for a specific index (an
     521             :  * index can reference return value, function and parameter attributes).
     522             :  */
     523             : static void
     524       61422 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
     525             : {
     526             :     int         num_attributes;
     527             :     LLVMAttributeRef *attrs;
     528             : 
     529       61422 :     num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
     530             : 
     531       61422 :     if (num_attributes == 0)
     532       21666 :         return;
     533             : 
     534       39756 :     attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
     535       39756 :     LLVMGetAttributesAtIndex(v_from, index, attrs);
     536             : 
     537      466222 :     for (int attno = 0; attno < num_attributes; attno++)
     538      426466 :         LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
     539             : 
     540       39756 :     pfree(attrs);
     541             : }
     542             : 
     543             : /*
     544             :  * Copy all attributes from one function to another. I.e. function, return and
     545             :  * parameters will be copied.
     546             :  */
     547             : void
     548       21804 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
     549             : {
     550             :     uint32      param_count;
     551             : 
     552             :     /* copy function attributes */
     553       21804 :     llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
     554             : 
     555       21804 :     if (LLVMGetTypeKind(LLVMGetFunctionReturnType(v_to)) != LLVMVoidTypeKind)
     556             :     {
     557             :         /* and the return value attributes */
     558       11920 :         llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
     559             :     }
     560             : 
     561             :     /* and each function parameter's attribute */
     562       21804 :     param_count = LLVMCountParams(v_from);
     563             : 
     564       49502 :     for (int paramidx = 1; paramidx <= param_count; paramidx++)
     565       27698 :         llvm_copy_attributes_at_index(v_from, v_to, paramidx);
     566       21804 : }
     567             : 
     568             : /*
     569             :  * Return a callable LLVMValueRef for fcinfo.
     570             :  */
     571             : LLVMValueRef
     572        5818 : llvm_function_reference(LLVMJitContext *context,
     573             :                         LLVMBuilderRef builder,
     574             :                         LLVMModuleRef mod,
     575             :                         FunctionCallInfo fcinfo)
     576             : {
     577             :     char       *modname;
     578             :     char       *basename;
     579             :     char       *funcname;
     580             : 
     581             :     LLVMValueRef v_fn;
     582             : 
     583        5818 :     fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
     584             : 
     585        5818 :     if (modname != NULL && basename != NULL)
     586             :     {
     587             :         /* external function in loadable library */
     588          20 :         funcname = psprintf("pgextern.%s.%s", modname, basename);
     589             :     }
     590        5798 :     else if (basename != NULL)
     591             :     {
     592             :         /* internal function */
     593        5786 :         funcname = pstrdup(basename);
     594             :     }
     595             :     else
     596             :     {
     597             :         /*
     598             :          * Function we don't know to handle, return pointer. We do so by
     599             :          * creating a global constant containing a pointer to the function.
     600             :          * Makes IR more readable.
     601             :          */
     602             :         LLVMValueRef v_fn_addr;
     603             : 
     604          12 :         funcname = psprintf("pgoidextern.%u",
     605          12 :                             fcinfo->flinfo->fn_oid);
     606          12 :         v_fn = LLVMGetNamedGlobal(mod, funcname);
     607          12 :         if (v_fn != 0)
     608           0 :             return l_load(builder, TypePGFunction, v_fn, "");
     609             : 
     610          12 :         v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
     611             : 
     612          12 :         v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
     613          12 :         LLVMSetInitializer(v_fn, v_fn_addr);
     614          12 :         LLVMSetGlobalConstant(v_fn, true);
     615          12 :         LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
     616          12 :         LLVMSetUnnamedAddr(v_fn, true);
     617             : 
     618          12 :         return l_load(builder, TypePGFunction, v_fn, "");
     619             :     }
     620             : 
     621             :     /* check if function already has been added */
     622        5806 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     623        5806 :     if (v_fn != 0)
     624        2504 :         return v_fn;
     625             : 
     626        3302 :     v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
     627             : 
     628        3302 :     return v_fn;
     629             : }
     630             : 
     631             : /*
     632             :  * Optimize code in module using the flags set in context.
     633             :  */
     634             : static void
     635        1316 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
     636             : {
     637             : #if LLVM_VERSION_MAJOR < 17
     638             :     LLVMPassManagerBuilderRef llvm_pmb;
     639             :     LLVMPassManagerRef llvm_mpm;
     640             :     LLVMPassManagerRef llvm_fpm;
     641             :     LLVMValueRef func;
     642             :     int         compile_optlevel;
     643             : 
     644        1316 :     if (context->base.flags & PGJIT_OPT3)
     645         982 :         compile_optlevel = 3;
     646             :     else
     647         334 :         compile_optlevel = 0;
     648             : 
     649             :     /*
     650             :      * Have to create a new pass manager builder every pass through, as the
     651             :      * inliner has some per-builder state. Otherwise one ends up only inlining
     652             :      * a function the first time though.
     653             :      */
     654        1316 :     llvm_pmb = LLVMPassManagerBuilderCreate();
     655        1316 :     LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
     656        1316 :     llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
     657             : 
     658        1316 :     if (context->base.flags & PGJIT_OPT3)
     659             :     {
     660             :         /* TODO: Unscientifically determined threshold */
     661         982 :         LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
     662             :     }
     663             :     else
     664             :     {
     665             :         /* we rely on mem2reg heavily, so emit even in the O0 case */
     666         334 :         LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
     667             :     }
     668             : 
     669        1316 :     LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
     670             : 
     671             :     /*
     672             :      * Do function level optimization. This could be moved to the point where
     673             :      * functions are emitted, to reduce memory usage a bit.
     674             :      */
     675        1316 :     LLVMInitializeFunctionPassManager(llvm_fpm);
     676       31926 :     for (func = LLVMGetFirstFunction(context->module);
     677             :          func != NULL;
     678       30610 :          func = LLVMGetNextFunction(func))
     679       30610 :         LLVMRunFunctionPassManager(llvm_fpm, func);
     680        1316 :     LLVMFinalizeFunctionPassManager(llvm_fpm);
     681        1316 :     LLVMDisposePassManager(llvm_fpm);
     682             : 
     683             :     /*
     684             :      * Perform module level optimization. We do so even in the non-optimized
     685             :      * case, so always-inline functions etc get inlined. It's cheap enough.
     686             :      */
     687        1316 :     llvm_mpm = LLVMCreatePassManager();
     688        1316 :     LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
     689             :                                                     llvm_mpm);
     690             :     /* always use always-inliner pass */
     691        1316 :     if (!(context->base.flags & PGJIT_OPT3))
     692         334 :         LLVMAddAlwaysInlinerPass(llvm_mpm);
     693             :     /* if doing inlining, but no expensive optimization, add inlining pass */
     694        1316 :     if (context->base.flags & PGJIT_INLINE
     695         982 :         && !(context->base.flags & PGJIT_OPT3))
     696           0 :         LLVMAddFunctionInliningPass(llvm_mpm);
     697        1316 :     LLVMRunPassManager(llvm_mpm, context->module);
     698        1316 :     LLVMDisposePassManager(llvm_mpm);
     699             : 
     700        1316 :     LLVMPassManagerBuilderDispose(llvm_pmb);
     701             : #else
     702             :     LLVMPassBuilderOptionsRef options;
     703             :     LLVMErrorRef err;
     704             :     const char *passes;
     705             : 
     706             :     if (context->base.flags & PGJIT_OPT3)
     707             :         passes = "default<O3>";
     708             :     else
     709             :         passes = "default<O0>,mem2reg";
     710             : 
     711             :     options = LLVMCreatePassBuilderOptions();
     712             : 
     713             : #ifdef LLVM_PASS_DEBUG
     714             :     LLVMPassBuilderOptionsSetDebugLogging(options, 1);
     715             : #endif
     716             : 
     717             :     LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
     718             : 
     719             :     err = LLVMRunPasses(module, passes, NULL, options);
     720             : 
     721             :     if (err)
     722             :         elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
     723             : 
     724             :     LLVMDisposePassBuilderOptions(options);
     725             : #endif
     726        1316 : }
     727             : 
     728             : /*
     729             :  * Emit code for the currently pending module.
     730             :  */
     731             : static void
     732        1316 : llvm_compile_module(LLVMJitContext *context)
     733             : {
     734             :     LLVMJitHandle *handle;
     735             :     MemoryContext oldcontext;
     736             :     instr_time  starttime;
     737             :     instr_time  endtime;
     738             : #if LLVM_VERSION_MAJOR > 11
     739             :     LLVMOrcLLJITRef compile_orc;
     740             : #else
     741             :     LLVMOrcJITStackRef compile_orc;
     742             : #endif
     743             : 
     744        1316 :     if (context->base.flags & PGJIT_OPT3)
     745         982 :         compile_orc = llvm_opt3_orc;
     746             :     else
     747         334 :         compile_orc = llvm_opt0_orc;
     748             : 
     749             :     /* perform inlining */
     750        1316 :     if (context->base.flags & PGJIT_INLINE)
     751             :     {
     752         982 :         INSTR_TIME_SET_CURRENT(starttime);
     753         982 :         llvm_inline(context->module);
     754         982 :         INSTR_TIME_SET_CURRENT(endtime);
     755         982 :         INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
     756             :                               endtime, starttime);
     757             :     }
     758             : 
     759        1316 :     if (jit_dump_bitcode)
     760             :     {
     761             :         char       *filename;
     762             : 
     763           0 :         filename = psprintf("%d.%zu.bc",
     764             :                             MyProcPid,
     765             :                             context->module_generation);
     766           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     767           0 :         pfree(filename);
     768             :     }
     769             : 
     770             : 
     771             :     /* optimize according to the chosen optimization settings */
     772        1316 :     INSTR_TIME_SET_CURRENT(starttime);
     773        1316 :     llvm_optimize_module(context, context->module);
     774        1316 :     INSTR_TIME_SET_CURRENT(endtime);
     775        1316 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
     776             :                           endtime, starttime);
     777             : 
     778        1316 :     if (jit_dump_bitcode)
     779             :     {
     780             :         char       *filename;
     781             : 
     782           0 :         filename = psprintf("%d.%zu.optimized.bc",
     783             :                             MyProcPid,
     784             :                             context->module_generation);
     785           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     786           0 :         pfree(filename);
     787             :     }
     788             : 
     789             :     handle = (LLVMJitHandle *)
     790        1316 :         MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
     791             : 
     792             :     /*
     793             :      * Emit the code. Note that this can, depending on the optimization
     794             :      * settings, take noticeable resources as code emission executes low-level
     795             :      * instruction combining/selection passes etc. Without optimization a
     796             :      * faster instruction selection mechanism is used.
     797             :      */
     798        1316 :     INSTR_TIME_SET_CURRENT(starttime);
     799             : #if LLVM_VERSION_MAJOR > 11
     800             :     {
     801             :         LLVMOrcThreadSafeModuleRef ts_module;
     802             :         LLVMErrorRef error;
     803             :         LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
     804             : 
     805             :         ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
     806             : 
     807             :         handle->lljit = compile_orc;
     808             :         handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
     809             : 
     810             :         /*
     811             :          * NB: This doesn't actually emit code. That happens lazily the first
     812             :          * time a symbol defined in the module is requested. Due to that
     813             :          * llvm_get_function() also accounts for emission time.
     814             :          */
     815             : 
     816             :         context->module = NULL; /* will be owned by LLJIT */
     817             :         error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
     818             :                                                   handle->resource_tracker,
     819             :                                                   ts_module);
     820             : 
     821             :         if (error)
     822             :             elog(ERROR, "failed to JIT module: %s",
     823             :                  llvm_error_message(error));
     824             : 
     825             :         /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
     826             :     }
     827             : #else
     828             :     {
     829        1316 :         handle->stack = compile_orc;
     830        1316 :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
     831             :                                         llvm_resolve_symbol, NULL))
     832           0 :             elog(ERROR, "failed to JIT module");
     833             : 
     834             :         /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
     835             :     }
     836             : #endif
     837             : 
     838        1316 :     INSTR_TIME_SET_CURRENT(endtime);
     839        1316 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     840             :                           endtime, starttime);
     841             : 
     842        1316 :     context->module = NULL;
     843        1316 :     context->compiled = true;
     844             : 
     845             :     /* remember emitted code for cleanup and lookups */
     846        1316 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     847        1316 :     context->handles = lappend(context->handles, handle);
     848        1316 :     MemoryContextSwitchTo(oldcontext);
     849             : 
     850        1316 :     ereport(DEBUG1,
     851             :             (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
     852             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
     853             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
     854             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
     855             :              errhidestmt(true),
     856             :              errhidecontext(true)));
     857        1316 : }
     858             : 
     859             : /*
     860             :  * Per session initialization.
     861             :  */
     862             : static void
     863        1776 : llvm_session_initialize(void)
     864             : {
     865             :     MemoryContext oldcontext;
     866        1776 :     char       *error = NULL;
     867        1776 :     char       *cpu = NULL;
     868        1776 :     char       *features = NULL;
     869             :     LLVMTargetMachineRef opt0_tm;
     870             :     LLVMTargetMachineRef opt3_tm;
     871             : 
     872        1776 :     if (llvm_session_initialized)
     873         978 :         return;
     874             : 
     875         798 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     876             : 
     877         798 :     LLVMInitializeNativeTarget();
     878         798 :     LLVMInitializeNativeAsmPrinter();
     879         798 :     LLVMInitializeNativeAsmParser();
     880             : 
     881         798 :     if (llvm_context == NULL)
     882             :     {
     883         798 :         llvm_context = LLVMContextCreate();
     884             : 
     885         798 :         llvm_jit_context_in_use_count = 0;
     886         798 :         llvm_llvm_context_reuse_count = 0;
     887             :     }
     888             : 
     889             :     /*
     890             :      * When targeting LLVM 15, turn off opaque pointers for the context we
     891             :      * build our code in.  We don't need to do so for other contexts (e.g.
     892             :      * llvm_ts_context).  Once the IR is generated, it carries the necessary
     893             :      * information.
     894             :      *
     895             :      * For 16 and above, opaque pointers must be used, and we have special
     896             :      * code for that.
     897             :      */
     898             : #if LLVM_VERSION_MAJOR == 15
     899             :     LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
     900             : #endif
     901             : 
     902             :     /*
     903             :      * Synchronize types early, as that also includes inferring the target
     904             :      * triple.
     905             :      */
     906         798 :     llvm_create_types();
     907             : 
     908             :     /*
     909             :      * Extract target information from loaded module.
     910             :      */
     911         798 :     llvm_set_target();
     912             : 
     913         798 :     if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
     914             :     {
     915           0 :         elog(FATAL, "failed to query triple %s", error);
     916             :     }
     917             : 
     918             :     /*
     919             :      * We want the generated code to use all available features. Therefore
     920             :      * grab the host CPU string and detect features of the current CPU. The
     921             :      * latter is needed because some CPU architectures default to enabling
     922             :      * features not all CPUs have (weird, huh).
     923             :      */
     924         798 :     cpu = LLVMGetHostCPUName();
     925         798 :     features = LLVMGetHostCPUFeatures();
     926         798 :     elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
     927             :          cpu, features);
     928             : 
     929             :     opt0_tm =
     930         798 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     931             :                                 LLVMCodeGenLevelNone,
     932             :                                 LLVMRelocDefault,
     933             :                                 LLVMCodeModelJITDefault);
     934             :     opt3_tm =
     935         798 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     936             :                                 LLVMCodeGenLevelAggressive,
     937             :                                 LLVMRelocDefault,
     938             :                                 LLVMCodeModelJITDefault);
     939             : 
     940         798 :     LLVMDisposeMessage(cpu);
     941         798 :     cpu = NULL;
     942         798 :     LLVMDisposeMessage(features);
     943         798 :     features = NULL;
     944             : 
     945             :     /* force symbols in main binary to be loaded */
     946         798 :     LLVMLoadLibraryPermanently(NULL);
     947             : 
     948             : #if LLVM_VERSION_MAJOR > 11
     949             :     {
     950             :         llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
     951             : 
     952             :         llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
     953             :         opt0_tm = 0;
     954             : 
     955             :         llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
     956             :         opt3_tm = 0;
     957             :     }
     958             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     959             :     {
     960         798 :         llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
     961         798 :         llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
     962             : 
     963             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
     964         798 :         if (jit_debugging_support)
     965             :         {
     966           0 :             LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
     967             : 
     968           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     969           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     970             :         }
     971             : #endif
     972             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
     973         798 :         if (jit_profiling_support)
     974             :         {
     975           0 :             LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
     976             : 
     977           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     978           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     979             :         }
     980             : #endif
     981             :     }
     982             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     983             : 
     984         798 :     on_proc_exit(llvm_shutdown, 0);
     985             : 
     986         798 :     llvm_session_initialized = true;
     987             : 
     988         798 :     MemoryContextSwitchTo(oldcontext);
     989             : }
     990             : 
     991             : static void
     992         798 : llvm_shutdown(int code, Datum arg)
     993             : {
     994             :     /*
     995             :      * If llvm_shutdown() is reached while in a fatal-on-oom section an error
     996             :      * has occurred in the middle of LLVM code. It is not safe to call back
     997             :      * into LLVM (which is why a FATAL error was thrown).
     998             :      *
     999             :      * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
    1000             :      * profiling data won't be written out.
    1001             :      */
    1002         798 :     if (llvm_in_fatal_on_oom())
    1003             :     {
    1004             :         Assert(proc_exit_inprogress);
    1005           0 :         return;
    1006             :     }
    1007             : 
    1008         798 :     if (llvm_jit_context_in_use_count != 0)
    1009           0 :         elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
    1010             :              llvm_jit_context_in_use_count);
    1011             : 
    1012             : #if LLVM_VERSION_MAJOR > 11
    1013             :     {
    1014             :         if (llvm_opt3_orc)
    1015             :         {
    1016             :             LLVMOrcDisposeLLJIT(llvm_opt3_orc);
    1017             :             llvm_opt3_orc = NULL;
    1018             :         }
    1019             :         if (llvm_opt0_orc)
    1020             :         {
    1021             :             LLVMOrcDisposeLLJIT(llvm_opt0_orc);
    1022             :             llvm_opt0_orc = NULL;
    1023             :         }
    1024             :         if (llvm_ts_context)
    1025             :         {
    1026             :             LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
    1027             :             llvm_ts_context = NULL;
    1028             :         }
    1029             :     }
    1030             : #else                           /* LLVM_VERSION_MAJOR > 11 */
    1031             :     {
    1032             :         /* unregister profiling support, needs to be flushed to be useful */
    1033             : 
    1034         798 :         if (llvm_opt3_orc)
    1035             :         {
    1036         798 :             LLVMOrcDisposeInstance(llvm_opt3_orc);
    1037         798 :             llvm_opt3_orc = NULL;
    1038             :         }
    1039             : 
    1040         798 :         if (llvm_opt0_orc)
    1041             :         {
    1042         798 :             LLVMOrcDisposeInstance(llvm_opt0_orc);
    1043         798 :             llvm_opt0_orc = NULL;
    1044             :         }
    1045             :     }
    1046             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1047             : }
    1048             : 
    1049             : /* helper for llvm_create_types, returning a function's return type */
    1050             : static LLVMTypeRef
    1051         798 : load_return_type(LLVMModuleRef mod, const char *name)
    1052             : {
    1053             :     LLVMValueRef value;
    1054             :     LLVMTypeRef typ;
    1055             : 
    1056             :     /* this'll return a *pointer* to the function */
    1057         798 :     value = LLVMGetNamedFunction(mod, name);
    1058         798 :     if (!value)
    1059           0 :         elog(ERROR, "function %s is unknown", name);
    1060             : 
    1061         798 :     typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
    1062             : 
    1063         798 :     return typ;
    1064             : }
    1065             : 
    1066             : /*
    1067             :  * Load triple & layout from clang emitted file so we're guaranteed to be
    1068             :  * compatible.
    1069             :  */
    1070             : static void
    1071         798 : llvm_set_target(void)
    1072             : {
    1073         798 :     if (!llvm_types_module)
    1074           0 :         elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
    1075             : 
    1076         798 :     if (llvm_triple == NULL)
    1077         798 :         llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
    1078             : 
    1079         798 :     if (llvm_layout == NULL)
    1080         798 :         llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
    1081         798 : }
    1082             : 
    1083             : /*
    1084             :  * Load required information, types, function signatures from llvmjit_types.c
    1085             :  * and make them available in global variables.
    1086             :  *
    1087             :  * Those global variables are then used while emitting code.
    1088             :  */
    1089             : static void
    1090         798 : llvm_create_types(void)
    1091             : {
    1092             :     char        path[MAXPGPATH];
    1093             :     LLVMMemoryBufferRef buf;
    1094             :     char       *msg;
    1095             : 
    1096         798 :     snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
    1097             : 
    1098             :     /* open file */
    1099         798 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
    1100             :     {
    1101           0 :         elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
    1102             :              path, msg);
    1103             :     }
    1104             : 
    1105             :     /* eagerly load contents, going to need it all */
    1106         798 :     if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
    1107             :     {
    1108           0 :         elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
    1109             :     }
    1110         798 :     LLVMDisposeMemoryBuffer(buf);
    1111             : 
    1112         798 :     TypeSizeT = llvm_pg_var_type("TypeSizeT");
    1113         798 :     TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
    1114         798 :     TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
    1115         798 :     TypePGFunction = llvm_pg_var_type("TypePGFunction");
    1116         798 :     StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
    1117         798 :     StructExprContext = llvm_pg_var_type("StructExprContext");
    1118         798 :     StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
    1119         798 :     StructExprState = llvm_pg_var_type("StructExprState");
    1120         798 :     StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
    1121         798 :     StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
    1122         798 :     StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
    1123         798 :     StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
    1124         798 :     StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
    1125         798 :     StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
    1126         798 :     StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
    1127         798 :     StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
    1128         798 :     StructAggState = llvm_pg_var_type("StructAggState");
    1129         798 :     StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
    1130         798 :     StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
    1131         798 :     StructPlanState = llvm_pg_var_type("StructPlanState");
    1132         798 :     StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
    1133             : 
    1134         798 :     AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
    1135         798 :     ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
    1136         798 :     ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
    1137         798 : }
    1138             : 
    1139             : /*
    1140             :  * Split a symbol into module / function parts.  If the function is in the
    1141             :  * main binary (or an external library) *modname will be NULL.
    1142             :  */
    1143             : void
    1144       33446 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
    1145             : {
    1146       33446 :     *modname = NULL;
    1147       33446 :     *funcname = NULL;
    1148             : 
    1149             :     /*
    1150             :      * Module function names are pgextern.$module.$funcname
    1151             :      */
    1152       33446 :     if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
    1153             :     {
    1154             :         /*
    1155             :          * Symbol names cannot contain a ., therefore we can split based on
    1156             :          * first and last occurrence of one.
    1157             :          */
    1158          58 :         *funcname = rindex(name, '.');
    1159          58 :         (*funcname)++;          /* jump over . */
    1160             : 
    1161         116 :         *modname = pnstrdup(name + strlen("pgextern."),
    1162          58 :                             *funcname - name - strlen("pgextern.") - 1);
    1163             :         Assert(funcname);
    1164             : 
    1165          58 :         *funcname = pstrdup(*funcname);
    1166             :     }
    1167             :     else
    1168             :     {
    1169       33388 :         *modname = NULL;
    1170       33388 :         *funcname = pstrdup(name);
    1171             :     }
    1172       33446 : }
    1173             : 
    1174             : /*
    1175             :  * Attempt to resolve symbol, so LLVM can emit a reference to it.
    1176             :  */
    1177             : static uint64_t
    1178        5226 : llvm_resolve_symbol(const char *symname, void *ctx)
    1179             : {
    1180             :     uintptr_t   addr;
    1181             :     char       *funcname;
    1182             :     char       *modname;
    1183             : 
    1184             :     /*
    1185             :      * macOS prefixes all object level symbols with an underscore. But neither
    1186             :      * dlsym() nor PG's inliner expect that. So undo.
    1187             :      */
    1188             : #if defined(__darwin__)
    1189             :     if (symname[0] != '_')
    1190             :         elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
    1191             :     symname++;
    1192             : #endif
    1193             : 
    1194        5226 :     llvm_split_symbol_name(symname, &modname, &funcname);
    1195             : 
    1196             :     /* functions that aren't resolved to names shouldn't ever get here */
    1197             :     Assert(funcname);
    1198             : 
    1199        5226 :     if (modname)
    1200          20 :         addr = (uintptr_t) load_external_function(modname, funcname,
    1201             :                                                   true, NULL);
    1202             :     else
    1203        5206 :         addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
    1204             : 
    1205        5226 :     pfree(funcname);
    1206        5226 :     if (modname)
    1207          20 :         pfree(modname);
    1208             : 
    1209             :     /* let LLVM will error out - should never happen */
    1210        5226 :     if (!addr)
    1211           0 :         elog(WARNING, "failed to resolve name %s", symname);
    1212             : 
    1213        5226 :     return (uint64_t) addr;
    1214             : }
    1215             : 
    1216             : #if LLVM_VERSION_MAJOR > 11
    1217             : 
    1218             : static LLVMErrorRef
    1219             : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
    1220             :                      LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
    1221             :                      LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
    1222             :                      LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
    1223             : {
    1224             : #if LLVM_VERSION_MAJOR > 14
    1225             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
    1226             : #else
    1227             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
    1228             : #endif
    1229             :     LLVMErrorRef error;
    1230             :     LLVMOrcMaterializationUnitRef mu;
    1231             : 
    1232             :     for (int i = 0; i < LookupSetSize; i++)
    1233             :     {
    1234             :         const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
    1235             : 
    1236             : #if LLVM_VERSION_MAJOR > 12
    1237             :         LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
    1238             : #endif
    1239             :         symbols[i].Name = LookupSet[i].Name;
    1240             :         symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
    1241             :         symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
    1242             :     }
    1243             : 
    1244             :     mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
    1245             :     error = LLVMOrcJITDylibDefine(JD, mu);
    1246             :     if (error != LLVMErrorSuccess)
    1247             :         LLVMOrcDisposeMaterializationUnit(mu);
    1248             : 
    1249             :     pfree(symbols);
    1250             : 
    1251             :     return error;
    1252             : }
    1253             : 
    1254             : /*
    1255             :  * We cannot throw errors through LLVM (without causing a FATAL at least), so
    1256             :  * just use WARNING here. That's OK anyway, as the error is also reported at
    1257             :  * the top level action (with less detail) and there might be multiple
    1258             :  * invocations of errors with details.
    1259             :  *
    1260             :  * This doesn't really happen during normal operation, but in cases like
    1261             :  * symbol resolution breakage. So just using elog(WARNING) is fine.
    1262             :  */
    1263             : static void
    1264             : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
    1265             : {
    1266             :     elog(WARNING, "error during JITing: %s",
    1267             :          llvm_error_message(error));
    1268             : }
    1269             : 
    1270             : /*
    1271             :  * Create our own object layer, so we can add event listeners.
    1272             :  */
    1273             : static LLVMOrcObjectLayerRef
    1274             : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
    1275             : {
    1276             :     LLVMOrcObjectLayerRef objlayer =
    1277             :         LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
    1278             : 
    1279             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
    1280             :     if (jit_debugging_support)
    1281             :     {
    1282             :         LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
    1283             : 
    1284             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1285             :     }
    1286             : #endif
    1287             : 
    1288             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
    1289             :     if (jit_profiling_support)
    1290             :     {
    1291             :         LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
    1292             : 
    1293             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1294             :     }
    1295             : #endif
    1296             : 
    1297             :     return objlayer;
    1298             : }
    1299             : 
    1300             : /*
    1301             :  * Create LLJIT instance, using the passed in target machine. Note that the
    1302             :  * target machine afterwards is owned by the LLJIT instance.
    1303             :  */
    1304             : static LLVMOrcLLJITRef
    1305             : llvm_create_jit_instance(LLVMTargetMachineRef tm)
    1306             : {
    1307             :     LLVMOrcLLJITRef lljit;
    1308             :     LLVMOrcJITTargetMachineBuilderRef tm_builder;
    1309             :     LLVMOrcLLJITBuilderRef lljit_builder;
    1310             :     LLVMErrorRef error;
    1311             :     LLVMOrcDefinitionGeneratorRef main_gen;
    1312             :     LLVMOrcDefinitionGeneratorRef ref_gen;
    1313             : 
    1314             :     lljit_builder = LLVMOrcCreateLLJITBuilder();
    1315             :     tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
    1316             :     LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
    1317             : 
    1318             :     LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
    1319             :                                                     llvm_create_object_layer,
    1320             :                                                     NULL);
    1321             : 
    1322             :     error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
    1323             :     if (error)
    1324             :         elog(ERROR, "failed to create lljit instance: %s",
    1325             :              llvm_error_message(error));
    1326             : 
    1327             :     LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
    1328             :                                             llvm_log_jit_error, NULL);
    1329             : 
    1330             :     /*
    1331             :      * Symbol resolution support for symbols in the postgres binary /
    1332             :      * libraries already loaded.
    1333             :      */
    1334             :     error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
    1335             :                                                                  LLVMOrcLLJITGetGlobalPrefix(lljit),
    1336             :                                                                  0, NULL);
    1337             :     if (error)
    1338             :         elog(ERROR, "failed to create generator: %s",
    1339             :              llvm_error_message(error));
    1340             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
    1341             : 
    1342             :     /*
    1343             :      * Symbol resolution support for "special" functions, e.g. a call into an
    1344             :      * SQL callable function.
    1345             :      */
    1346             : #if LLVM_VERSION_MAJOR > 14
    1347             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
    1348             : #else
    1349             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
    1350             : #endif
    1351             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
    1352             : 
    1353             :     return lljit;
    1354             : }
    1355             : 
    1356             : static char *
    1357             : llvm_error_message(LLVMErrorRef error)
    1358             : {
    1359             :     char       *orig = LLVMGetErrorMessage(error);
    1360             :     char       *msg = pstrdup(orig);
    1361             : 
    1362             :     LLVMDisposeErrorMessage(orig);
    1363             : 
    1364             :     return msg;
    1365             : }
    1366             : 
    1367             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1368             : 
    1369             : /*
    1370             :  * ResourceOwner callbacks
    1371             :  */
    1372             : static void
    1373          16 : ResOwnerReleaseJitContext(Datum res)
    1374             : {
    1375          16 :     JitContext *context = (JitContext *) DatumGetPointer(res);
    1376             : 
    1377          16 :     context->resowner = NULL;
    1378          16 :     jit_release_context(context);
    1379          16 : }

Generated by: LCOV version 1.14