LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit.c (source / functions) Hit Total Coverage
Test: PostgreSQL 18devel Lines: 294 331 88.8 %
Date: 2024-07-18 17:11:54 Functions: 26 26 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit.c
       4             :  *    Core part of the LLVM JIT provider.
       5             :  *
       6             :  * Copyright (c) 2016-2024, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/jit/llvm/llvmjit.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres.h"
      15             : 
      16             : #include <llvm-c/Analysis.h>
      17             : #include <llvm-c/BitReader.h>
      18             : #include <llvm-c/BitWriter.h>
      19             : #include <llvm-c/Core.h>
      20             : #include <llvm-c/ExecutionEngine.h>
      21             : #if LLVM_VERSION_MAJOR > 16
      22             : #include <llvm-c/Transforms/PassBuilder.h>
      23             : #endif
      24             : #if LLVM_VERSION_MAJOR > 11
      25             : #include <llvm-c/Orc.h>
      26             : #include <llvm-c/OrcEE.h>
      27             : #include <llvm-c/LLJIT.h>
      28             : #else
      29             : #include <llvm-c/OrcBindings.h>
      30             : #endif
      31             : #include <llvm-c/Support.h>
      32             : #include <llvm-c/Target.h>
      33             : #if LLVM_VERSION_MAJOR < 17
      34             : #include <llvm-c/Transforms/IPO.h>
      35             : #include <llvm-c/Transforms/PassManagerBuilder.h>
      36             : #include <llvm-c/Transforms/Scalar.h>
      37             : #include <llvm-c/Transforms/Utils.h>
      38             : #endif
      39             : 
      40             : #include "jit/llvmjit.h"
      41             : #include "jit/llvmjit_emit.h"
      42             : #include "miscadmin.h"
      43             : #include "portability/instr_time.h"
      44             : #include "storage/ipc.h"
      45             : #include "utils/memutils.h"
      46             : #include "utils/resowner.h"
      47             : 
      48             : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
      49             : 
      50             : /* Handle of a module emitted via ORC JIT */
      51             : typedef struct LLVMJitHandle
      52             : {
      53             : #if LLVM_VERSION_MAJOR > 11
      54             :     LLVMOrcLLJITRef lljit;
      55             :     LLVMOrcResourceTrackerRef resource_tracker;
      56             : #else
      57             :     LLVMOrcJITStackRef stack;
      58             :     LLVMOrcModuleHandle orc_handle;
      59             : #endif
      60             : } LLVMJitHandle;
      61             : 
      62             : 
      63             : /* types & functions commonly needed for JITing */
      64             : LLVMTypeRef TypeSizeT;
      65             : LLVMTypeRef TypeParamBool;
      66             : LLVMTypeRef TypeStorageBool;
      67             : LLVMTypeRef TypePGFunction;
      68             : LLVMTypeRef StructNullableDatum;
      69             : LLVMTypeRef StructHeapTupleData;
      70             : LLVMTypeRef StructMinimalTupleData;
      71             : LLVMTypeRef StructTupleDescData;
      72             : LLVMTypeRef StructTupleTableSlot;
      73             : LLVMTypeRef StructHeapTupleHeaderData;
      74             : LLVMTypeRef StructHeapTupleTableSlot;
      75             : LLVMTypeRef StructMinimalTupleTableSlot;
      76             : LLVMTypeRef StructMemoryContextData;
      77             : LLVMTypeRef StructFunctionCallInfoData;
      78             : LLVMTypeRef StructExprContext;
      79             : LLVMTypeRef StructExprEvalStep;
      80             : LLVMTypeRef StructExprState;
      81             : LLVMTypeRef StructAggState;
      82             : LLVMTypeRef StructAggStatePerGroupData;
      83             : LLVMTypeRef StructAggStatePerTransData;
      84             : LLVMTypeRef StructPlanState;
      85             : 
      86             : LLVMValueRef AttributeTemplate;
      87             : LLVMValueRef ExecEvalSubroutineTemplate;
      88             : LLVMValueRef ExecEvalBoolSubroutineTemplate;
      89             : 
      90             : static LLVMModuleRef llvm_types_module = NULL;
      91             : 
      92             : static bool llvm_session_initialized = false;
      93             : static size_t llvm_generation = 0;
      94             : 
      95             : /* number of LLVMJitContexts that currently are in use */
      96             : static size_t llvm_jit_context_in_use_count = 0;
      97             : 
      98             : /* how many times has the current LLVMContextRef been used */
      99             : static size_t llvm_llvm_context_reuse_count = 0;
     100             : static const char *llvm_triple = NULL;
     101             : static const char *llvm_layout = NULL;
     102             : static LLVMContextRef llvm_context;
     103             : 
     104             : 
     105             : static LLVMTargetRef llvm_targetref;
     106             : #if LLVM_VERSION_MAJOR > 11
     107             : static LLVMOrcThreadSafeContextRef llvm_ts_context;
     108             : static LLVMOrcLLJITRef llvm_opt0_orc;
     109             : static LLVMOrcLLJITRef llvm_opt3_orc;
     110             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     111             : static LLVMOrcJITStackRef llvm_opt0_orc;
     112             : static LLVMOrcJITStackRef llvm_opt3_orc;
     113             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     114             : 
     115             : 
     116             : static void llvm_release_context(JitContext *context);
     117             : static void llvm_session_initialize(void);
     118             : static void llvm_shutdown(int code, Datum arg);
     119             : static void llvm_compile_module(LLVMJitContext *context);
     120             : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
     121             : 
     122             : static void llvm_create_types(void);
     123             : static void llvm_set_target(void);
     124             : static void llvm_recreate_llvm_context(void);
     125             : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
     126             : 
     127             : #if LLVM_VERSION_MAJOR > 11
     128             : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
     129             : static char *llvm_error_message(LLVMErrorRef error);
     130             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     131             : 
     132             : /* ResourceOwner callbacks to hold JitContexts  */
     133             : static void ResOwnerReleaseJitContext(Datum res);
     134             : 
     135             : static const ResourceOwnerDesc jit_resowner_desc =
     136             : {
     137             :     .name = "LLVM JIT context",
     138             :     .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
     139             :     .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
     140             :     .ReleaseResource = ResOwnerReleaseJitContext,
     141             :     .DebugPrint = NULL          /* the default message is fine */
     142             : };
     143             : 
     144             : /* Convenience wrappers over ResourceOwnerRemember/Forget */
     145             : static inline void
     146        1838 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
     147             : {
     148        1838 :     ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
     149        1838 : }
     150             : static inline void
     151        1822 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
     152             : {
     153        1822 :     ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
     154        1822 : }
     155             : 
     156        2054 : PG_MODULE_MAGIC;
     157             : 
     158             : 
     159             : /*
     160             :  * Initialize LLVM JIT provider.
     161             :  */
     162             : void
     163         840 : _PG_jit_provider_init(JitProviderCallbacks *cb)
     164             : {
     165         840 :     cb->reset_after_error = llvm_reset_after_error;
     166         840 :     cb->release_context = llvm_release_context;
     167         840 :     cb->compile_expr = llvm_compile_expr;
     168         840 : }
     169             : 
     170             : 
     171             : /*
     172             :  * Every now and then create a new LLVMContextRef. Unfortunately, during every
     173             :  * round of inlining, types may "leak" (they can still be found/used via the
     174             :  * context, but new types will be created the next time in inlining is
     175             :  * performed). To prevent that from slowly accumulating problematic amounts of
     176             :  * memory, recreate the LLVMContextRef we use. We don't want to do so too
     177             :  * often, as that implies some overhead (particularly re-loading the module
     178             :  * summaries / modules is fairly expensive). A future TODO would be to make
     179             :  * this more finegrained and only drop/recreate the LLVMContextRef when we know
     180             :  * there has been inlining. If we can get the size of the context from LLVM
     181             :  * then that might be a better way to determine when to drop/recreate rather
     182             :  * then the usagecount heuristic currently employed.
     183             :  */
     184             : static void
     185        1838 : llvm_recreate_llvm_context(void)
     186             : {
     187        1838 :     if (!llvm_context)
     188           0 :         elog(ERROR, "Trying to recreate a non-existing context");
     189             : 
     190             :     /*
     191             :      * We can only safely recreate the LLVM context if no other code is being
     192             :      * JITed, otherwise we'd release the types in use for that.
     193             :      */
     194        1838 :     if (llvm_jit_context_in_use_count > 0)
     195             :     {
     196           0 :         llvm_llvm_context_reuse_count++;
     197           0 :         return;
     198             :     }
     199             : 
     200        1838 :     if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
     201             :     {
     202        1838 :         llvm_llvm_context_reuse_count++;
     203        1838 :         return;
     204             :     }
     205             : 
     206             :     /*
     207             :      * Need to reset the modules that the inlining code caches before
     208             :      * disposing of the context. LLVM modules exist within a specific LLVM
     209             :      * context, therefore disposing of the context before resetting the cache
     210             :      * would lead to dangling pointers to modules.
     211             :      */
     212           0 :     llvm_inline_reset_caches();
     213             : 
     214           0 :     LLVMContextDispose(llvm_context);
     215           0 :     llvm_context = LLVMContextCreate();
     216           0 :     llvm_llvm_context_reuse_count = 0;
     217             : 
     218             :     /*
     219             :      * Re-build cached type information, so code generation code can rely on
     220             :      * that information to be present (also prevents the variables to be
     221             :      * dangling references).
     222             :      */
     223           0 :     llvm_create_types();
     224             : }
     225             : 
     226             : 
     227             : /*
     228             :  * Create a context for JITing work.
     229             :  *
     230             :  * The context, including subsidiary resources, will be cleaned up either when
     231             :  * the context is explicitly released, or when the lifetime of
     232             :  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
     233             :  */
     234             : LLVMJitContext *
     235        1838 : llvm_create_context(int jitFlags)
     236             : {
     237             :     LLVMJitContext *context;
     238             : 
     239        1838 :     llvm_assert_in_fatal_section();
     240             : 
     241        1838 :     llvm_session_initialize();
     242             : 
     243        1838 :     llvm_recreate_llvm_context();
     244             : 
     245        1838 :     ResourceOwnerEnlarge(CurrentResourceOwner);
     246             : 
     247        1838 :     context = MemoryContextAllocZero(TopMemoryContext,
     248             :                                      sizeof(LLVMJitContext));
     249        1838 :     context->base.flags = jitFlags;
     250             : 
     251             :     /* ensure cleanup */
     252        1838 :     context->base.resowner = CurrentResourceOwner;
     253        1838 :     ResourceOwnerRememberJIT(CurrentResourceOwner, context);
     254             : 
     255        1838 :     llvm_jit_context_in_use_count++;
     256             : 
     257        1838 :     return context;
     258             : }
     259             : 
     260             : /*
     261             :  * Release resources required by one llvm context.
     262             :  */
     263             : static void
     264        1838 : llvm_release_context(JitContext *context)
     265             : {
     266        1838 :     LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
     267             :     ListCell   *lc;
     268             : 
     269             :     /*
     270             :      * Consider as cleaned up even if we skip doing so below, that way we can
     271             :      * verify the tracking is correct (see llvm_shutdown()).
     272             :      */
     273        1838 :     llvm_jit_context_in_use_count--;
     274             : 
     275             :     /*
     276             :      * When this backend is exiting, don't clean up LLVM. As an error might
     277             :      * have occurred from within LLVM, we do not want to risk reentering. All
     278             :      * resource cleanup is going to happen through process exit.
     279             :      */
     280        1838 :     if (proc_exit_inprogress)
     281           0 :         return;
     282             : 
     283        1838 :     llvm_enter_fatal_on_oom();
     284             : 
     285        1838 :     if (llvm_jit_context->module)
     286             :     {
     287         628 :         LLVMDisposeModule(llvm_jit_context->module);
     288         628 :         llvm_jit_context->module = NULL;
     289             :     }
     290             : 
     291        3170 :     foreach(lc, llvm_jit_context->handles)
     292             :     {
     293        1332 :         LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
     294             : 
     295             : #if LLVM_VERSION_MAJOR > 11
     296             :         {
     297             :             LLVMOrcExecutionSessionRef ee;
     298             :             LLVMOrcSymbolStringPoolRef sp;
     299             : 
     300             :             LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
     301             :             LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
     302             : 
     303             :             /*
     304             :              * Without triggering cleanup of the string pool, we'd leak
     305             :              * memory. It'd be sufficient to do this far less often, but in
     306             :              * experiments the required time was small enough to just always
     307             :              * do it.
     308             :              */
     309             :             ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
     310             :             sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
     311             :             LLVMOrcSymbolStringPoolClearDeadEntries(sp);
     312             :         }
     313             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     314             :         {
     315        1332 :             LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
     316             :         }
     317             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     318             : 
     319        1332 :         pfree(jit_handle);
     320             :     }
     321        1838 :     list_free(llvm_jit_context->handles);
     322        1838 :     llvm_jit_context->handles = NIL;
     323             : 
     324        1838 :     llvm_leave_fatal_on_oom();
     325             : 
     326        1838 :     if (context->resowner)
     327        1822 :         ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
     328             : }
     329             : 
     330             : /*
     331             :  * Return module which may be modified, e.g. by creating new functions.
     332             :  */
     333             : LLVMModuleRef
     334       17878 : llvm_mutable_module(LLVMJitContext *context)
     335             : {
     336       17878 :     llvm_assert_in_fatal_section();
     337             : 
     338             :     /*
     339             :      * If there's no in-progress module, create a new one.
     340             :      */
     341       17878 :     if (!context->module)
     342             :     {
     343        1960 :         context->compiled = false;
     344        1960 :         context->module_generation = llvm_generation++;
     345        1960 :         context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
     346        1960 :         LLVMSetTarget(context->module, llvm_triple);
     347        1960 :         LLVMSetDataLayout(context->module, llvm_layout);
     348             :     }
     349             : 
     350       17878 :     return context->module;
     351             : }
     352             : 
     353             : /*
     354             :  * Expand function name to be non-conflicting. This should be used by code
     355             :  * generating code, when adding new externally visible function definitions to
     356             :  * a Module.
     357             :  */
     358             : char *
     359       17878 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
     360             : {
     361             :     Assert(context->module != NULL);
     362             : 
     363       17878 :     context->base.instr.created_functions++;
     364             : 
     365             :     /*
     366             :      * Previously we used dots to separate, but turns out some tools, e.g.
     367             :      * GDB, don't like that and truncate name.
     368             :      */
     369       35756 :     return psprintf("%s_%zu_%d",
     370             :                     basename,
     371             :                     context->module_generation,
     372       17878 :                     context->counter++);
     373             : }
     374             : 
     375             : /*
     376             :  * Return pointer to function funcname, which has to exist. If there's pending
     377             :  * code to be optimized and emitted, do so first.
     378             :  */
     379             : void *
     380        6240 : llvm_get_function(LLVMJitContext *context, const char *funcname)
     381             : {
     382             :     ListCell   *lc;
     383             : 
     384        6240 :     llvm_assert_in_fatal_section();
     385             : 
     386             :     /*
     387             :      * If there is a pending / not emitted module, compile and emit now.
     388             :      * Otherwise we might not find the [correct] function.
     389             :      */
     390        6240 :     if (!context->compiled)
     391             :     {
     392        1332 :         llvm_compile_module(context);
     393             :     }
     394             : 
     395             :     /*
     396             :      * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
     397             :      * to mangle here.
     398             :      */
     399             : 
     400             : #if LLVM_VERSION_MAJOR > 11
     401             :     foreach(lc, context->handles)
     402             :     {
     403             :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     404             :         instr_time  starttime;
     405             :         instr_time  endtime;
     406             :         LLVMErrorRef error;
     407             :         LLVMOrcJITTargetAddress addr;
     408             : 
     409             :         INSTR_TIME_SET_CURRENT(starttime);
     410             : 
     411             :         addr = 0;
     412             :         error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
     413             :         if (error)
     414             :             elog(ERROR, "failed to look up symbol \"%s\": %s",
     415             :                  funcname, llvm_error_message(error));
     416             : 
     417             :         /*
     418             :          * LLJIT only actually emits code the first time a symbol is
     419             :          * referenced. Thus add lookup time to emission time. That's counting
     420             :          * a bit more than with older LLVM versions, but unlikely to ever
     421             :          * matter.
     422             :          */
     423             :         INSTR_TIME_SET_CURRENT(endtime);
     424             :         INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     425             :                               endtime, starttime);
     426             : 
     427             :         if (addr)
     428             :             return (void *) (uintptr_t) addr;
     429             :     }
     430             : #else
     431        6374 :     foreach(lc, context->handles)
     432             :     {
     433             :         LLVMOrcTargetAddress addr;
     434        6374 :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     435             : 
     436        6374 :         addr = 0;
     437        6374 :         if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
     438           0 :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     439        6374 :         if (addr)
     440        6240 :             return (void *) (uintptr_t) addr;
     441             :     }
     442             : #endif
     443             : 
     444           0 :     elog(ERROR, "failed to JIT: %s", funcname);
     445             : 
     446             :     return NULL;
     447             : }
     448             : 
     449             : /*
     450             :  * Return type of a variable in llvmjit_types.c. This is useful to keep types
     451             :  * in sync between plain C and JIT related code.
     452             :  */
     453             : LLVMTypeRef
     454       16918 : llvm_pg_var_type(const char *varname)
     455             : {
     456             :     LLVMValueRef v_srcvar;
     457             :     LLVMTypeRef typ;
     458             : 
     459             :     /* this'll return a *pointer* to the global */
     460       16918 :     v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
     461       16918 :     if (!v_srcvar)
     462           0 :         elog(ERROR, "variable %s not in llvmjit_types.c", varname);
     463             : 
     464       16918 :     typ = LLVMGlobalGetValueType(v_srcvar);
     465             : 
     466       16918 :     return typ;
     467             : }
     468             : 
     469             : /*
     470             :  * Return function type of a variable in llvmjit_types.c. This is useful to
     471             :  * keep function types in sync between C and JITed code.
     472             :  */
     473             : LLVMTypeRef
     474       15740 : llvm_pg_var_func_type(const char *varname)
     475             : {
     476             :     LLVMValueRef v_srcvar;
     477             :     LLVMTypeRef typ;
     478             : 
     479       15740 :     v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
     480       15740 :     if (!v_srcvar)
     481           0 :         elog(ERROR, "function %s not in llvmjit_types.c", varname);
     482             : 
     483       15740 :     typ = LLVMGetFunctionType(v_srcvar);
     484             : 
     485       15740 :     return typ;
     486             : }
     487             : 
     488             : /*
     489             :  * Return declaration for a function referenced in llvmjit_types.c, adding it
     490             :  * to the module if necessary.
     491             :  *
     492             :  * This is used to make functions discovered via llvm_create_types() known to
     493             :  * the module that's currently being worked on.
     494             :  */
     495             : LLVMValueRef
     496       14168 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
     497             : {
     498             :     LLVMValueRef v_srcfn;
     499             :     LLVMValueRef v_fn;
     500             : 
     501             :     /* don't repeatedly add function */
     502       14168 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     503       14168 :     if (v_fn)
     504       10132 :         return v_fn;
     505             : 
     506        4036 :     v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
     507             : 
     508        4036 :     if (!v_srcfn)
     509           0 :         elog(ERROR, "function %s not in llvmjit_types.c", funcname);
     510             : 
     511        4036 :     v_fn = LLVMAddFunction(mod,
     512             :                            funcname,
     513             :                            LLVMGetFunctionType(v_srcfn));
     514        4036 :     llvm_copy_attributes(v_srcfn, v_fn);
     515             : 
     516        4036 :     return v_fn;
     517             : }
     518             : 
     519             : /*
     520             :  * Copy attributes from one function to another, for a specific index (an
     521             :  * index can reference return value, function and parameter attributes).
     522             :  */
     523             : static void
     524       61748 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
     525             : {
     526             :     int         num_attributes;
     527             :     LLVMAttributeRef *attrs;
     528             : 
     529       61748 :     num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
     530             : 
     531       61748 :     if (num_attributes == 0)
     532       21870 :         return;
     533             : 
     534       39878 :     attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
     535       39878 :     LLVMGetAttributesAtIndex(v_from, index, attrs);
     536             : 
     537      467870 :     for (int attno = 0; attno < num_attributes; attno++)
     538      427992 :         LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
     539             : 
     540       39878 :     pfree(attrs);
     541             : }
     542             : 
     543             : /*
     544             :  * Copy all attributes from one function to another. I.e. function, return and
     545             :  * parameters will be copied.
     546             :  */
     547             : void
     548       21914 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
     549             : {
     550             :     uint32      param_count;
     551             : 
     552             :     /* copy function attributes */
     553       21914 :     llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
     554             : 
     555       21914 :     if (LLVMGetTypeKind(LLVMGetFunctionReturnType(v_to)) != LLVMVoidTypeKind)
     556             :     {
     557             :         /* and the return value attributes */
     558       11874 :         llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
     559             :     }
     560             : 
     561             :     /* and each function parameter's attribute */
     562       21914 :     param_count = LLVMCountParams(v_from);
     563             : 
     564       49874 :     for (int paramidx = 1; paramidx <= param_count; paramidx++)
     565       27960 :         llvm_copy_attributes_at_index(v_from, v_to, paramidx);
     566       21914 : }
     567             : 
     568             : /*
     569             :  * Return a callable LLVMValueRef for fcinfo.
     570             :  */
     571             : LLVMValueRef
     572        5806 : llvm_function_reference(LLVMJitContext *context,
     573             :                         LLVMBuilderRef builder,
     574             :                         LLVMModuleRef mod,
     575             :                         FunctionCallInfo fcinfo)
     576             : {
     577             :     char       *modname;
     578             :     char       *basename;
     579             :     char       *funcname;
     580             : 
     581             :     LLVMValueRef v_fn;
     582             : 
     583        5806 :     fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
     584             : 
     585        5806 :     if (modname != NULL && basename != NULL)
     586             :     {
     587             :         /* external function in loadable library */
     588          20 :         funcname = psprintf("pgextern.%s.%s", modname, basename);
     589             :     }
     590        5786 :     else if (basename != NULL)
     591             :     {
     592             :         /* internal function */
     593        5774 :         funcname = pstrdup(basename);
     594             :     }
     595             :     else
     596             :     {
     597             :         /*
     598             :          * Function we don't know to handle, return pointer. We do so by
     599             :          * creating a global constant containing a pointer to the function.
     600             :          * Makes IR more readable.
     601             :          */
     602             :         LLVMValueRef v_fn_addr;
     603             : 
     604          12 :         funcname = psprintf("pgoidextern.%u",
     605          12 :                             fcinfo->flinfo->fn_oid);
     606          12 :         v_fn = LLVMGetNamedGlobal(mod, funcname);
     607          12 :         if (v_fn != 0)
     608           0 :             return l_load(builder, TypePGFunction, v_fn, "");
     609             : 
     610          12 :         v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
     611             : 
     612          12 :         v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
     613          12 :         LLVMSetInitializer(v_fn, v_fn_addr);
     614          12 :         LLVMSetGlobalConstant(v_fn, true);
     615          12 :         LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
     616          12 :         LLVMSetUnnamedAddr(v_fn, true);
     617             : 
     618          12 :         return l_load(builder, TypePGFunction, v_fn, "");
     619             :     }
     620             : 
     621             :     /* check if function already has been added */
     622        5794 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     623        5794 :     if (v_fn != 0)
     624        2426 :         return v_fn;
     625             : 
     626        3368 :     v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
     627             : 
     628        3368 :     return v_fn;
     629             : }
     630             : 
     631             : /*
     632             :  * Optimize code in module using the flags set in context.
     633             :  */
     634             : static void
     635        1332 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
     636             : {
     637             : #if LLVM_VERSION_MAJOR < 17
     638             :     LLVMPassManagerBuilderRef llvm_pmb;
     639             :     LLVMPassManagerRef llvm_mpm;
     640             :     LLVMPassManagerRef llvm_fpm;
     641             :     LLVMValueRef func;
     642             :     int         compile_optlevel;
     643             : 
     644        1332 :     if (context->base.flags & PGJIT_OPT3)
     645         998 :         compile_optlevel = 3;
     646             :     else
     647         334 :         compile_optlevel = 0;
     648             : 
     649             :     /*
     650             :      * Have to create a new pass manager builder every pass through, as the
     651             :      * inliner has some per-builder state. Otherwise one ends up only inlining
     652             :      * a function the first time though.
     653             :      */
     654        1332 :     llvm_pmb = LLVMPassManagerBuilderCreate();
     655        1332 :     LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
     656        1332 :     llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
     657             : 
     658        1332 :     if (context->base.flags & PGJIT_OPT3)
     659             :     {
     660             :         /* TODO: Unscientifically determined threshold */
     661         998 :         LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
     662             :     }
     663             :     else
     664             :     {
     665             :         /* we rely on mem2reg heavily, so emit even in the O0 case */
     666         334 :         LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
     667             :     }
     668             : 
     669        1332 :     LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
     670             : 
     671             :     /*
     672             :      * Do function level optimization. This could be moved to the point where
     673             :      * functions are emitted, to reduce memory usage a bit.
     674             :      */
     675        1332 :     LLVMInitializeFunctionPassManager(llvm_fpm);
     676       31992 :     for (func = LLVMGetFirstFunction(context->module);
     677             :          func != NULL;
     678       30660 :          func = LLVMGetNextFunction(func))
     679       30660 :         LLVMRunFunctionPassManager(llvm_fpm, func);
     680        1332 :     LLVMFinalizeFunctionPassManager(llvm_fpm);
     681        1332 :     LLVMDisposePassManager(llvm_fpm);
     682             : 
     683             :     /*
     684             :      * Perform module level optimization. We do so even in the non-optimized
     685             :      * case, so always-inline functions etc get inlined. It's cheap enough.
     686             :      */
     687        1332 :     llvm_mpm = LLVMCreatePassManager();
     688        1332 :     LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
     689             :                                                     llvm_mpm);
     690             :     /* always use always-inliner pass */
     691        1332 :     if (!(context->base.flags & PGJIT_OPT3))
     692         334 :         LLVMAddAlwaysInlinerPass(llvm_mpm);
     693             :     /* if doing inlining, but no expensive optimization, add inlining pass */
     694        1332 :     if (context->base.flags & PGJIT_INLINE
     695         998 :         && !(context->base.flags & PGJIT_OPT3))
     696           0 :         LLVMAddFunctionInliningPass(llvm_mpm);
     697        1332 :     LLVMRunPassManager(llvm_mpm, context->module);
     698        1332 :     LLVMDisposePassManager(llvm_mpm);
     699             : 
     700        1332 :     LLVMPassManagerBuilderDispose(llvm_pmb);
     701             : #else
     702             :     LLVMPassBuilderOptionsRef options;
     703             :     LLVMErrorRef err;
     704             :     const char *passes;
     705             : 
     706             :     if (context->base.flags & PGJIT_OPT3)
     707             :         passes = "default<O3>";
     708             :     else
     709             :         passes = "default<O0>,mem2reg";
     710             : 
     711             :     options = LLVMCreatePassBuilderOptions();
     712             : 
     713             : #ifdef LLVM_PASS_DEBUG
     714             :     LLVMPassBuilderOptionsSetDebugLogging(options, 1);
     715             : #endif
     716             : 
     717             :     /* In assertion builds, run the LLVM verify pass. */
     718             : #ifdef USE_ASSERT_CHECKING
     719             :     LLVMPassBuilderOptionsSetVerifyEach(options, true);
     720             : #endif
     721             : 
     722             :     LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
     723             : 
     724             :     err = LLVMRunPasses(module, passes, NULL, options);
     725             : 
     726             :     if (err)
     727             :         elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
     728             : 
     729             :     LLVMDisposePassBuilderOptions(options);
     730             : #endif
     731        1332 : }
     732             : 
     733             : /*
     734             :  * Emit code for the currently pending module.
     735             :  */
     736             : static void
     737        1332 : llvm_compile_module(LLVMJitContext *context)
     738             : {
     739             :     LLVMJitHandle *handle;
     740             :     MemoryContext oldcontext;
     741             :     instr_time  starttime;
     742             :     instr_time  endtime;
     743             : #if LLVM_VERSION_MAJOR > 11
     744             :     LLVMOrcLLJITRef compile_orc;
     745             : #else
     746             :     LLVMOrcJITStackRef compile_orc;
     747             : #endif
     748             : 
     749        1332 :     if (context->base.flags & PGJIT_OPT3)
     750         998 :         compile_orc = llvm_opt3_orc;
     751             :     else
     752         334 :         compile_orc = llvm_opt0_orc;
     753             : 
     754             :     /* perform inlining */
     755        1332 :     if (context->base.flags & PGJIT_INLINE)
     756             :     {
     757         998 :         INSTR_TIME_SET_CURRENT(starttime);
     758         998 :         llvm_inline(context->module);
     759         998 :         INSTR_TIME_SET_CURRENT(endtime);
     760         998 :         INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
     761             :                               endtime, starttime);
     762             :     }
     763             : 
     764        1332 :     if (jit_dump_bitcode)
     765             :     {
     766             :         char       *filename;
     767             : 
     768           0 :         filename = psprintf("%d.%zu.bc",
     769             :                             MyProcPid,
     770             :                             context->module_generation);
     771           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     772           0 :         pfree(filename);
     773             :     }
     774             : 
     775             : 
     776             :     /* optimize according to the chosen optimization settings */
     777        1332 :     INSTR_TIME_SET_CURRENT(starttime);
     778        1332 :     llvm_optimize_module(context, context->module);
     779        1332 :     INSTR_TIME_SET_CURRENT(endtime);
     780        1332 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
     781             :                           endtime, starttime);
     782             : 
     783        1332 :     if (jit_dump_bitcode)
     784             :     {
     785             :         char       *filename;
     786             : 
     787           0 :         filename = psprintf("%d.%zu.optimized.bc",
     788             :                             MyProcPid,
     789             :                             context->module_generation);
     790           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     791           0 :         pfree(filename);
     792             :     }
     793             : 
     794             :     handle = (LLVMJitHandle *)
     795        1332 :         MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
     796             : 
     797             :     /*
     798             :      * Emit the code. Note that this can, depending on the optimization
     799             :      * settings, take noticeable resources as code emission executes low-level
     800             :      * instruction combining/selection passes etc. Without optimization a
     801             :      * faster instruction selection mechanism is used.
     802             :      */
     803        1332 :     INSTR_TIME_SET_CURRENT(starttime);
     804             : #if LLVM_VERSION_MAJOR > 11
     805             :     {
     806             :         LLVMOrcThreadSafeModuleRef ts_module;
     807             :         LLVMErrorRef error;
     808             :         LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
     809             : 
     810             :         ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
     811             : 
     812             :         handle->lljit = compile_orc;
     813             :         handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
     814             : 
     815             :         /*
     816             :          * NB: This doesn't actually emit code. That happens lazily the first
     817             :          * time a symbol defined in the module is requested. Due to that
     818             :          * llvm_get_function() also accounts for emission time.
     819             :          */
     820             : 
     821             :         context->module = NULL; /* will be owned by LLJIT */
     822             :         error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
     823             :                                                   handle->resource_tracker,
     824             :                                                   ts_module);
     825             : 
     826             :         if (error)
     827             :             elog(ERROR, "failed to JIT module: %s",
     828             :                  llvm_error_message(error));
     829             : 
     830             :         /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
     831             :     }
     832             : #else
     833             :     {
     834        1332 :         handle->stack = compile_orc;
     835        1332 :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
     836             :                                         llvm_resolve_symbol, NULL))
     837           0 :             elog(ERROR, "failed to JIT module");
     838             : 
     839             :         /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
     840             :     }
     841             : #endif
     842             : 
     843        1332 :     INSTR_TIME_SET_CURRENT(endtime);
     844        1332 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     845             :                           endtime, starttime);
     846             : 
     847        1332 :     context->module = NULL;
     848        1332 :     context->compiled = true;
     849             : 
     850             :     /* remember emitted code for cleanup and lookups */
     851        1332 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     852        1332 :     context->handles = lappend(context->handles, handle);
     853        1332 :     MemoryContextSwitchTo(oldcontext);
     854             : 
     855        1332 :     ereport(DEBUG1,
     856             :             (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
     857             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
     858             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
     859             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
     860             :              errhidestmt(true),
     861             :              errhidecontext(true)));
     862        1332 : }
     863             : 
     864             : /*
     865             :  * Per session initialization.
     866             :  */
     867             : static void
     868        1838 : llvm_session_initialize(void)
     869             : {
     870             :     MemoryContext oldcontext;
     871        1838 :     char       *error = NULL;
     872        1838 :     char       *cpu = NULL;
     873        1838 :     char       *features = NULL;
     874             :     LLVMTargetMachineRef opt0_tm;
     875             :     LLVMTargetMachineRef opt3_tm;
     876             : 
     877        1838 :     if (llvm_session_initialized)
     878         998 :         return;
     879             : 
     880         840 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     881             : 
     882         840 :     LLVMInitializeNativeTarget();
     883         840 :     LLVMInitializeNativeAsmPrinter();
     884         840 :     LLVMInitializeNativeAsmParser();
     885             : 
     886         840 :     if (llvm_context == NULL)
     887             :     {
     888         840 :         llvm_context = LLVMContextCreate();
     889             : 
     890         840 :         llvm_jit_context_in_use_count = 0;
     891         840 :         llvm_llvm_context_reuse_count = 0;
     892             :     }
     893             : 
     894             :     /*
     895             :      * When targeting LLVM 15, turn off opaque pointers for the context we
     896             :      * build our code in.  We don't need to do so for other contexts (e.g.
     897             :      * llvm_ts_context).  Once the IR is generated, it carries the necessary
     898             :      * information.
     899             :      *
     900             :      * For 16 and above, opaque pointers must be used, and we have special
     901             :      * code for that.
     902             :      */
     903             : #if LLVM_VERSION_MAJOR == 15
     904             :     LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
     905             : #endif
     906             : 
     907             :     /*
     908             :      * Synchronize types early, as that also includes inferring the target
     909             :      * triple.
     910             :      */
     911         840 :     llvm_create_types();
     912             : 
     913             :     /*
     914             :      * Extract target information from loaded module.
     915             :      */
     916         840 :     llvm_set_target();
     917             : 
     918         840 :     if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
     919             :     {
     920           0 :         elog(FATAL, "failed to query triple %s", error);
     921             :     }
     922             : 
     923             :     /*
     924             :      * We want the generated code to use all available features. Therefore
     925             :      * grab the host CPU string and detect features of the current CPU. The
     926             :      * latter is needed because some CPU architectures default to enabling
     927             :      * features not all CPUs have (weird, huh).
     928             :      */
     929         840 :     cpu = LLVMGetHostCPUName();
     930         840 :     features = LLVMGetHostCPUFeatures();
     931         840 :     elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
     932             :          cpu, features);
     933             : 
     934             :     opt0_tm =
     935         840 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     936             :                                 LLVMCodeGenLevelNone,
     937             :                                 LLVMRelocDefault,
     938             :                                 LLVMCodeModelJITDefault);
     939             :     opt3_tm =
     940         840 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     941             :                                 LLVMCodeGenLevelAggressive,
     942             :                                 LLVMRelocDefault,
     943             :                                 LLVMCodeModelJITDefault);
     944             : 
     945         840 :     LLVMDisposeMessage(cpu);
     946         840 :     cpu = NULL;
     947         840 :     LLVMDisposeMessage(features);
     948         840 :     features = NULL;
     949             : 
     950             :     /* force symbols in main binary to be loaded */
     951         840 :     LLVMLoadLibraryPermanently(NULL);
     952             : 
     953             : #if LLVM_VERSION_MAJOR > 11
     954             :     {
     955             :         llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
     956             : 
     957             :         llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
     958             :         opt0_tm = 0;
     959             : 
     960             :         llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
     961             :         opt3_tm = 0;
     962             :     }
     963             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     964             :     {
     965         840 :         llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
     966         840 :         llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
     967             : 
     968             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
     969         840 :         if (jit_debugging_support)
     970             :         {
     971           0 :             LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
     972             : 
     973           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     974           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     975             :         }
     976             : #endif
     977             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
     978         840 :         if (jit_profiling_support)
     979             :         {
     980           0 :             LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
     981             : 
     982           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     983           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     984             :         }
     985             : #endif
     986             :     }
     987             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     988             : 
     989         840 :     on_proc_exit(llvm_shutdown, 0);
     990             : 
     991         840 :     llvm_session_initialized = true;
     992             : 
     993         840 :     MemoryContextSwitchTo(oldcontext);
     994             : }
     995             : 
     996             : static void
     997         840 : llvm_shutdown(int code, Datum arg)
     998             : {
     999             :     /*
    1000             :      * If llvm_shutdown() is reached while in a fatal-on-oom section an error
    1001             :      * has occurred in the middle of LLVM code. It is not safe to call back
    1002             :      * into LLVM (which is why a FATAL error was thrown).
    1003             :      *
    1004             :      * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
    1005             :      * profiling data won't be written out.
    1006             :      */
    1007         840 :     if (llvm_in_fatal_on_oom())
    1008             :     {
    1009             :         Assert(proc_exit_inprogress);
    1010           0 :         return;
    1011             :     }
    1012             : 
    1013         840 :     if (llvm_jit_context_in_use_count != 0)
    1014           0 :         elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
    1015             :              llvm_jit_context_in_use_count);
    1016             : 
    1017             : #if LLVM_VERSION_MAJOR > 11
    1018             :     {
    1019             :         if (llvm_opt3_orc)
    1020             :         {
    1021             :             LLVMOrcDisposeLLJIT(llvm_opt3_orc);
    1022             :             llvm_opt3_orc = NULL;
    1023             :         }
    1024             :         if (llvm_opt0_orc)
    1025             :         {
    1026             :             LLVMOrcDisposeLLJIT(llvm_opt0_orc);
    1027             :             llvm_opt0_orc = NULL;
    1028             :         }
    1029             :         if (llvm_ts_context)
    1030             :         {
    1031             :             LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
    1032             :             llvm_ts_context = NULL;
    1033             :         }
    1034             :     }
    1035             : #else                           /* LLVM_VERSION_MAJOR > 11 */
    1036             :     {
    1037             :         /* unregister profiling support, needs to be flushed to be useful */
    1038             : 
    1039         840 :         if (llvm_opt3_orc)
    1040             :         {
    1041         840 :             LLVMOrcDisposeInstance(llvm_opt3_orc);
    1042         840 :             llvm_opt3_orc = NULL;
    1043             :         }
    1044             : 
    1045         840 :         if (llvm_opt0_orc)
    1046             :         {
    1047         840 :             LLVMOrcDisposeInstance(llvm_opt0_orc);
    1048         840 :             llvm_opt0_orc = NULL;
    1049             :         }
    1050             :     }
    1051             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1052             : }
    1053             : 
    1054             : /* helper for llvm_create_types, returning a function's return type */
    1055             : static LLVMTypeRef
    1056         840 : load_return_type(LLVMModuleRef mod, const char *name)
    1057             : {
    1058             :     LLVMValueRef value;
    1059             :     LLVMTypeRef typ;
    1060             : 
    1061             :     /* this'll return a *pointer* to the function */
    1062         840 :     value = LLVMGetNamedFunction(mod, name);
    1063         840 :     if (!value)
    1064           0 :         elog(ERROR, "function %s is unknown", name);
    1065             : 
    1066         840 :     typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
    1067             : 
    1068         840 :     return typ;
    1069             : }
    1070             : 
    1071             : /*
    1072             :  * Load triple & layout from clang emitted file so we're guaranteed to be
    1073             :  * compatible.
    1074             :  */
    1075             : static void
    1076         840 : llvm_set_target(void)
    1077             : {
    1078         840 :     if (!llvm_types_module)
    1079           0 :         elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
    1080             : 
    1081         840 :     if (llvm_triple == NULL)
    1082         840 :         llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
    1083             : 
    1084         840 :     if (llvm_layout == NULL)
    1085         840 :         llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
    1086         840 : }
    1087             : 
    1088             : /*
    1089             :  * Load required information, types, function signatures from llvmjit_types.c
    1090             :  * and make them available in global variables.
    1091             :  *
    1092             :  * Those global variables are then used while emitting code.
    1093             :  */
    1094             : static void
    1095         840 : llvm_create_types(void)
    1096             : {
    1097             :     char        path[MAXPGPATH];
    1098             :     LLVMMemoryBufferRef buf;
    1099             :     char       *msg;
    1100             : 
    1101         840 :     snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
    1102             : 
    1103             :     /* open file */
    1104         840 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
    1105             :     {
    1106           0 :         elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
    1107             :              path, msg);
    1108             :     }
    1109             : 
    1110             :     /* eagerly load contents, going to need it all */
    1111         840 :     if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
    1112             :     {
    1113           0 :         elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
    1114             :     }
    1115         840 :     LLVMDisposeMemoryBuffer(buf);
    1116             : 
    1117         840 :     TypeSizeT = llvm_pg_var_type("TypeSizeT");
    1118         840 :     TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
    1119         840 :     TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
    1120         840 :     TypePGFunction = llvm_pg_var_type("TypePGFunction");
    1121         840 :     StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
    1122         840 :     StructExprContext = llvm_pg_var_type("StructExprContext");
    1123         840 :     StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
    1124         840 :     StructExprState = llvm_pg_var_type("StructExprState");
    1125         840 :     StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
    1126         840 :     StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
    1127         840 :     StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
    1128         840 :     StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
    1129         840 :     StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
    1130         840 :     StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
    1131         840 :     StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
    1132         840 :     StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
    1133         840 :     StructAggState = llvm_pg_var_type("StructAggState");
    1134         840 :     StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
    1135         840 :     StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
    1136         840 :     StructPlanState = llvm_pg_var_type("StructPlanState");
    1137         840 :     StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
    1138             : 
    1139         840 :     AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
    1140         840 :     ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
    1141         840 :     ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
    1142         840 : }
    1143             : 
    1144             : /*
    1145             :  * Split a symbol into module / function parts.  If the function is in the
    1146             :  * main binary (or an external library) *modname will be NULL.
    1147             :  */
    1148             : void
    1149       34156 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
    1150             : {
    1151       34156 :     *modname = NULL;
    1152       34156 :     *funcname = NULL;
    1153             : 
    1154             :     /*
    1155             :      * Module function names are pgextern.$module.$funcname
    1156             :      */
    1157       34156 :     if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
    1158             :     {
    1159             :         /*
    1160             :          * Symbol names cannot contain a ., therefore we can split based on
    1161             :          * first and last occurrence of one.
    1162             :          */
    1163          58 :         *funcname = rindex(name, '.');
    1164          58 :         (*funcname)++;          /* jump over . */
    1165             : 
    1166         116 :         *modname = pnstrdup(name + strlen("pgextern."),
    1167          58 :                             *funcname - name - strlen("pgextern.") - 1);
    1168             :         Assert(funcname);
    1169             : 
    1170          58 :         *funcname = pstrdup(*funcname);
    1171             :     }
    1172             :     else
    1173             :     {
    1174       34098 :         *modname = NULL;
    1175       34098 :         *funcname = pstrdup(name);
    1176             :     }
    1177       34156 : }
    1178             : 
    1179             : /*
    1180             :  * Attempt to resolve symbol, so LLVM can emit a reference to it.
    1181             :  */
    1182             : static uint64_t
    1183        5242 : llvm_resolve_symbol(const char *symname, void *ctx)
    1184             : {
    1185             :     uintptr_t   addr;
    1186             :     char       *funcname;
    1187             :     char       *modname;
    1188             : 
    1189             :     /*
    1190             :      * macOS prefixes all object level symbols with an underscore. But neither
    1191             :      * dlsym() nor PG's inliner expect that. So undo.
    1192             :      */
    1193             : #if defined(__darwin__)
    1194             :     if (symname[0] != '_')
    1195             :         elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
    1196             :     symname++;
    1197             : #endif
    1198             : 
    1199        5242 :     llvm_split_symbol_name(symname, &modname, &funcname);
    1200             : 
    1201             :     /* functions that aren't resolved to names shouldn't ever get here */
    1202             :     Assert(funcname);
    1203             : 
    1204        5242 :     if (modname)
    1205          20 :         addr = (uintptr_t) load_external_function(modname, funcname,
    1206             :                                                   true, NULL);
    1207             :     else
    1208        5222 :         addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
    1209             : 
    1210        5242 :     pfree(funcname);
    1211        5242 :     if (modname)
    1212          20 :         pfree(modname);
    1213             : 
    1214             :     /* let LLVM will error out - should never happen */
    1215        5242 :     if (!addr)
    1216           0 :         elog(WARNING, "failed to resolve name %s", symname);
    1217             : 
    1218        5242 :     return (uint64_t) addr;
    1219             : }
    1220             : 
    1221             : #if LLVM_VERSION_MAJOR > 11
    1222             : 
    1223             : static LLVMErrorRef
    1224             : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
    1225             :                      LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
    1226             :                      LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
    1227             :                      LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
    1228             : {
    1229             : #if LLVM_VERSION_MAJOR > 14
    1230             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
    1231             : #else
    1232             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
    1233             : #endif
    1234             :     LLVMErrorRef error;
    1235             :     LLVMOrcMaterializationUnitRef mu;
    1236             : 
    1237             :     for (int i = 0; i < LookupSetSize; i++)
    1238             :     {
    1239             :         const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
    1240             : 
    1241             : #if LLVM_VERSION_MAJOR > 12
    1242             :         LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
    1243             : #endif
    1244             :         symbols[i].Name = LookupSet[i].Name;
    1245             :         symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
    1246             :         symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
    1247             :     }
    1248             : 
    1249             :     mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
    1250             :     error = LLVMOrcJITDylibDefine(JD, mu);
    1251             :     if (error != LLVMErrorSuccess)
    1252             :         LLVMOrcDisposeMaterializationUnit(mu);
    1253             : 
    1254             :     pfree(symbols);
    1255             : 
    1256             :     return error;
    1257             : }
    1258             : 
    1259             : /*
    1260             :  * We cannot throw errors through LLVM (without causing a FATAL at least), so
    1261             :  * just use WARNING here. That's OK anyway, as the error is also reported at
    1262             :  * the top level action (with less detail) and there might be multiple
    1263             :  * invocations of errors with details.
    1264             :  *
    1265             :  * This doesn't really happen during normal operation, but in cases like
    1266             :  * symbol resolution breakage. So just using elog(WARNING) is fine.
    1267             :  */
    1268             : static void
    1269             : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
    1270             : {
    1271             :     elog(WARNING, "error during JITing: %s",
    1272             :          llvm_error_message(error));
    1273             : }
    1274             : 
    1275             : /*
    1276             :  * Create our own object layer, so we can add event listeners.
    1277             :  */
    1278             : static LLVMOrcObjectLayerRef
    1279             : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
    1280             : {
    1281             :     LLVMOrcObjectLayerRef objlayer =
    1282             :         LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
    1283             : 
    1284             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
    1285             :     if (jit_debugging_support)
    1286             :     {
    1287             :         LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
    1288             : 
    1289             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1290             :     }
    1291             : #endif
    1292             : 
    1293             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
    1294             :     if (jit_profiling_support)
    1295             :     {
    1296             :         LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
    1297             : 
    1298             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1299             :     }
    1300             : #endif
    1301             : 
    1302             :     return objlayer;
    1303             : }
    1304             : 
    1305             : /*
    1306             :  * Create LLJIT instance, using the passed in target machine. Note that the
    1307             :  * target machine afterwards is owned by the LLJIT instance.
    1308             :  */
    1309             : static LLVMOrcLLJITRef
    1310             : llvm_create_jit_instance(LLVMTargetMachineRef tm)
    1311             : {
    1312             :     LLVMOrcLLJITRef lljit;
    1313             :     LLVMOrcJITTargetMachineBuilderRef tm_builder;
    1314             :     LLVMOrcLLJITBuilderRef lljit_builder;
    1315             :     LLVMErrorRef error;
    1316             :     LLVMOrcDefinitionGeneratorRef main_gen;
    1317             :     LLVMOrcDefinitionGeneratorRef ref_gen;
    1318             : 
    1319             :     lljit_builder = LLVMOrcCreateLLJITBuilder();
    1320             :     tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
    1321             :     LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
    1322             : 
    1323             :     LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
    1324             :                                                     llvm_create_object_layer,
    1325             :                                                     NULL);
    1326             : 
    1327             :     error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
    1328             :     if (error)
    1329             :         elog(ERROR, "failed to create lljit instance: %s",
    1330             :              llvm_error_message(error));
    1331             : 
    1332             :     LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
    1333             :                                             llvm_log_jit_error, NULL);
    1334             : 
    1335             :     /*
    1336             :      * Symbol resolution support for symbols in the postgres binary /
    1337             :      * libraries already loaded.
    1338             :      */
    1339             :     error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
    1340             :                                                                  LLVMOrcLLJITGetGlobalPrefix(lljit),
    1341             :                                                                  0, NULL);
    1342             :     if (error)
    1343             :         elog(ERROR, "failed to create generator: %s",
    1344             :              llvm_error_message(error));
    1345             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
    1346             : 
    1347             :     /*
    1348             :      * Symbol resolution support for "special" functions, e.g. a call into an
    1349             :      * SQL callable function.
    1350             :      */
    1351             : #if LLVM_VERSION_MAJOR > 14
    1352             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
    1353             : #else
    1354             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
    1355             : #endif
    1356             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
    1357             : 
    1358             :     return lljit;
    1359             : }
    1360             : 
    1361             : static char *
    1362             : llvm_error_message(LLVMErrorRef error)
    1363             : {
    1364             :     char       *orig = LLVMGetErrorMessage(error);
    1365             :     char       *msg = pstrdup(orig);
    1366             : 
    1367             :     LLVMDisposeErrorMessage(orig);
    1368             : 
    1369             :     return msg;
    1370             : }
    1371             : 
    1372             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1373             : 
    1374             : /*
    1375             :  * ResourceOwner callbacks
    1376             :  */
    1377             : static void
    1378          16 : ResOwnerReleaseJitContext(Datum res)
    1379             : {
    1380          16 :     JitContext *context = (JitContext *) DatumGetPointer(res);
    1381             : 
    1382          16 :     context->resowner = NULL;
    1383          16 :     jit_release_context(context);
    1384          16 : }

Generated by: LCOV version 1.14