LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 293 330 88.8 %
Date: 2024-03-28 15:11:36 Functions: 26 26 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit.c
       4             :  *    Core part of the LLVM JIT provider.
       5             :  *
       6             :  * Copyright (c) 2016-2024, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/jit/llvm/llvmjit.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres.h"
      15             : 
      16             : #include <llvm-c/Analysis.h>
      17             : #include <llvm-c/BitReader.h>
      18             : #include <llvm-c/BitWriter.h>
      19             : #include <llvm-c/Core.h>
      20             : #include <llvm-c/ExecutionEngine.h>
      21             : #if LLVM_VERSION_MAJOR > 16
      22             : #include <llvm-c/Transforms/PassBuilder.h>
      23             : #endif
      24             : #if LLVM_VERSION_MAJOR > 11
      25             : #include <llvm-c/Orc.h>
      26             : #include <llvm-c/OrcEE.h>
      27             : #include <llvm-c/LLJIT.h>
      28             : #else
      29             : #include <llvm-c/OrcBindings.h>
      30             : #endif
      31             : #include <llvm-c/Support.h>
      32             : #include <llvm-c/Target.h>
      33             : #if LLVM_VERSION_MAJOR < 17
      34             : #include <llvm-c/Transforms/IPO.h>
      35             : #include <llvm-c/Transforms/PassManagerBuilder.h>
      36             : #include <llvm-c/Transforms/Scalar.h>
      37             : #include <llvm-c/Transforms/Utils.h>
      38             : #endif
      39             : 
      40             : #include "jit/llvmjit.h"
      41             : #include "jit/llvmjit_emit.h"
      42             : #include "miscadmin.h"
      43             : #include "portability/instr_time.h"
      44             : #include "storage/ipc.h"
      45             : #include "utils/memutils.h"
      46             : #include "utils/resowner.h"
      47             : 
      48             : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
      49             : 
      50             : /* Handle of a module emitted via ORC JIT */
      51             : typedef struct LLVMJitHandle
      52             : {
      53             : #if LLVM_VERSION_MAJOR > 11
      54             :     LLVMOrcLLJITRef lljit;
      55             :     LLVMOrcResourceTrackerRef resource_tracker;
      56             : #else
      57             :     LLVMOrcJITStackRef stack;
      58             :     LLVMOrcModuleHandle orc_handle;
      59             : #endif
      60             : } LLVMJitHandle;
      61             : 
      62             : 
      63             : /* types & functions commonly needed for JITing */
      64             : LLVMTypeRef TypeSizeT;
      65             : LLVMTypeRef TypeParamBool;
      66             : LLVMTypeRef TypeStorageBool;
      67             : LLVMTypeRef TypePGFunction;
      68             : LLVMTypeRef StructNullableDatum;
      69             : LLVMTypeRef StructHeapTupleData;
      70             : LLVMTypeRef StructMinimalTupleData;
      71             : LLVMTypeRef StructTupleDescData;
      72             : LLVMTypeRef StructTupleTableSlot;
      73             : LLVMTypeRef StructHeapTupleHeaderData;
      74             : LLVMTypeRef StructHeapTupleTableSlot;
      75             : LLVMTypeRef StructMinimalTupleTableSlot;
      76             : LLVMTypeRef StructMemoryContextData;
      77             : LLVMTypeRef StructFunctionCallInfoData;
      78             : LLVMTypeRef StructExprContext;
      79             : LLVMTypeRef StructExprEvalStep;
      80             : LLVMTypeRef StructExprState;
      81             : LLVMTypeRef StructAggState;
      82             : LLVMTypeRef StructAggStatePerGroupData;
      83             : LLVMTypeRef StructAggStatePerTransData;
      84             : LLVMTypeRef StructPlanState;
      85             : 
      86             : LLVMValueRef AttributeTemplate;
      87             : LLVMValueRef ExecEvalSubroutineTemplate;
      88             : LLVMValueRef ExecEvalBoolSubroutineTemplate;
      89             : 
      90             : static LLVMModuleRef llvm_types_module = NULL;
      91             : 
      92             : static bool llvm_session_initialized = false;
      93             : static size_t llvm_generation = 0;
      94             : 
      95             : /* number of LLVMJitContexts that currently are in use */
      96             : static size_t llvm_jit_context_in_use_count = 0;
      97             : 
      98             : /* how many times has the current LLVMContextRef been used */
      99             : static size_t llvm_llvm_context_reuse_count = 0;
     100             : static const char *llvm_triple = NULL;
     101             : static const char *llvm_layout = NULL;
     102             : static LLVMContextRef llvm_context;
     103             : 
     104             : 
     105             : static LLVMTargetRef llvm_targetref;
     106             : #if LLVM_VERSION_MAJOR > 11
     107             : static LLVMOrcThreadSafeContextRef llvm_ts_context;
     108             : static LLVMOrcLLJITRef llvm_opt0_orc;
     109             : static LLVMOrcLLJITRef llvm_opt3_orc;
     110             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     111             : static LLVMOrcJITStackRef llvm_opt0_orc;
     112             : static LLVMOrcJITStackRef llvm_opt3_orc;
     113             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     114             : 
     115             : 
     116             : static void llvm_release_context(JitContext *context);
     117             : static void llvm_session_initialize(void);
     118             : static void llvm_shutdown(int code, Datum arg);
     119             : static void llvm_compile_module(LLVMJitContext *context);
     120             : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
     121             : 
     122             : static void llvm_create_types(void);
     123             : static void llvm_set_target(void);
     124             : static void llvm_recreate_llvm_context(void);
     125             : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
     126             : 
     127             : #if LLVM_VERSION_MAJOR > 11
     128             : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
     129             : static char *llvm_error_message(LLVMErrorRef error);
     130             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     131             : 
     132             : /* ResourceOwner callbacks to hold JitContexts  */
     133             : static void ResOwnerReleaseJitContext(Datum res);
     134             : 
     135             : static const ResourceOwnerDesc jit_resowner_desc =
     136             : {
     137             :     .name = "LLVM JIT context",
     138             :     .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
     139             :     .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
     140             :     .ReleaseResource = ResOwnerReleaseJitContext,
     141             :     .DebugPrint = NULL          /* the default message is fine */
     142             : };
     143             : 
     144             : /* Convenience wrappers over ResourceOwnerRemember/Forget */
     145             : static inline void
     146        1774 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
     147             : {
     148        1774 :     ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
     149        1774 : }
     150             : static inline void
     151        1758 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
     152             : {
     153        1758 :     ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
     154        1758 : }
     155             : 
     156        1988 : PG_MODULE_MAGIC;
     157             : 
     158             : 
     159             : /*
     160             :  * Initialize LLVM JIT provider.
     161             :  */
     162             : void
     163         798 : _PG_jit_provider_init(JitProviderCallbacks *cb)
     164             : {
     165         798 :     cb->reset_after_error = llvm_reset_after_error;
     166         798 :     cb->release_context = llvm_release_context;
     167         798 :     cb->compile_expr = llvm_compile_expr;
     168         798 : }
     169             : 
     170             : 
     171             : /*
     172             :  * Every now and then create a new LLVMContextRef. Unfortunately, during every
     173             :  * round of inlining, types may "leak" (they can still be found/used via the
     174             :  * context, but new types will be created the next time in inlining is
     175             :  * performed). To prevent that from slowly accumulating problematic amounts of
     176             :  * memory, recreate the LLVMContextRef we use. We don't want to do so too
     177             :  * often, as that implies some overhead (particularly re-loading the module
     178             :  * summaries / modules is fairly expensive). A future TODO would be to make
     179             :  * this more finegrained and only drop/recreate the LLVMContextRef when we know
     180             :  * there has been inlining. If we can get the size of the context from LLVM
     181             :  * then that might be a better way to determine when to drop/recreate rather
     182             :  * then the usagecount heuristic currently employed.
     183             :  */
     184             : static void
     185        1774 : llvm_recreate_llvm_context(void)
     186             : {
     187        1774 :     if (!llvm_context)
     188           0 :         elog(ERROR, "Trying to recreate a non-existing context");
     189             : 
     190             :     /*
     191             :      * We can only safely recreate the LLVM context if no other code is being
     192             :      * JITed, otherwise we'd release the types in use for that.
     193             :      */
     194        1774 :     if (llvm_jit_context_in_use_count > 0)
     195             :     {
     196           0 :         llvm_llvm_context_reuse_count++;
     197           0 :         return;
     198             :     }
     199             : 
     200        1774 :     if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
     201             :     {
     202        1774 :         llvm_llvm_context_reuse_count++;
     203        1774 :         return;
     204             :     }
     205             : 
     206             :     /*
     207             :      * Need to reset the modules that the inlining code caches before
     208             :      * disposing of the context. LLVM modules exist within a specific LLVM
     209             :      * context, therefore disposing of the context before resetting the cache
     210             :      * would lead to dangling pointers to modules.
     211             :      */
     212           0 :     llvm_inline_reset_caches();
     213             : 
     214           0 :     LLVMContextDispose(llvm_context);
     215           0 :     llvm_context = LLVMContextCreate();
     216           0 :     llvm_llvm_context_reuse_count = 0;
     217             : 
     218             :     /*
     219             :      * Re-build cached type information, so code generation code can rely on
     220             :      * that information to be present (also prevents the variables to be
     221             :      * dangling references).
     222             :      */
     223           0 :     llvm_create_types();
     224             : }
     225             : 
     226             : 
     227             : /*
     228             :  * Create a context for JITing work.
     229             :  *
     230             :  * The context, including subsidiary resources, will be cleaned up either when
     231             :  * the context is explicitly released, or when the lifetime of
     232             :  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
     233             :  */
     234             : LLVMJitContext *
     235        1774 : llvm_create_context(int jitFlags)
     236             : {
     237             :     LLVMJitContext *context;
     238             : 
     239        1774 :     llvm_assert_in_fatal_section();
     240             : 
     241        1774 :     llvm_session_initialize();
     242             : 
     243        1774 :     llvm_recreate_llvm_context();
     244             : 
     245        1774 :     ResourceOwnerEnlarge(CurrentResourceOwner);
     246             : 
     247        1774 :     context = MemoryContextAllocZero(TopMemoryContext,
     248             :                                      sizeof(LLVMJitContext));
     249        1774 :     context->base.flags = jitFlags;
     250             : 
     251             :     /* ensure cleanup */
     252        1774 :     context->base.resowner = CurrentResourceOwner;
     253        1774 :     ResourceOwnerRememberJIT(CurrentResourceOwner, context);
     254             : 
     255        1774 :     llvm_jit_context_in_use_count++;
     256             : 
     257        1774 :     return context;
     258             : }
     259             : 
     260             : /*
     261             :  * Release resources required by one llvm context.
     262             :  */
     263             : static void
     264        1774 : llvm_release_context(JitContext *context)
     265             : {
     266        1774 :     LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
     267             :     ListCell   *lc;
     268             : 
     269             :     /*
     270             :      * Consider as cleaned up even if we skip doing so below, that way we can
     271             :      * verify the tracking is correct (see llvm_shutdown()).
     272             :      */
     273        1774 :     llvm_jit_context_in_use_count--;
     274             : 
     275             :     /*
     276             :      * When this backend is exiting, don't clean up LLVM. As an error might
     277             :      * have occurred from within LLVM, we do not want to risk reentering. All
     278             :      * resource cleanup is going to happen through process exit.
     279             :      */
     280        1774 :     if (proc_exit_inprogress)
     281           0 :         return;
     282             : 
     283        1774 :     llvm_enter_fatal_on_oom();
     284             : 
     285        1774 :     if (llvm_jit_context->module)
     286             :     {
     287         582 :         LLVMDisposeModule(llvm_jit_context->module);
     288         582 :         llvm_jit_context->module = NULL;
     289             :     }
     290             : 
     291        3088 :     foreach(lc, llvm_jit_context->handles)
     292             :     {
     293        1314 :         LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
     294             : 
     295             : #if LLVM_VERSION_MAJOR > 11
     296             :         {
     297             :             LLVMOrcExecutionSessionRef ee;
     298             :             LLVMOrcSymbolStringPoolRef sp;
     299             : 
     300             :             LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
     301             :             LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
     302             : 
     303             :             /*
     304             :              * Without triggering cleanup of the string pool, we'd leak
     305             :              * memory. It'd be sufficient to do this far less often, but in
     306             :              * experiments the required time was small enough to just always
     307             :              * do it.
     308             :              */
     309             :             ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
     310             :             sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
     311             :             LLVMOrcSymbolStringPoolClearDeadEntries(sp);
     312             :         }
     313             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     314             :         {
     315        1314 :             LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
     316             :         }
     317             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     318             : 
     319        1314 :         pfree(jit_handle);
     320             :     }
     321        1774 :     list_free(llvm_jit_context->handles);
     322        1774 :     llvm_jit_context->handles = NIL;
     323             : 
     324        1774 :     llvm_leave_fatal_on_oom();
     325             : 
     326        1774 :     if (context->resowner)
     327        1758 :         ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
     328             : }
     329             : 
     330             : /*
     331             :  * Return module which may be modified, e.g. by creating new functions.
     332             :  */
     333             : LLVMModuleRef
     334       17774 : llvm_mutable_module(LLVMJitContext *context)
     335             : {
     336       17774 :     llvm_assert_in_fatal_section();
     337             : 
     338             :     /*
     339             :      * If there's no in-progress module, create a new one.
     340             :      */
     341       17774 :     if (!context->module)
     342             :     {
     343        1896 :         context->compiled = false;
     344        1896 :         context->module_generation = llvm_generation++;
     345        1896 :         context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
     346        1896 :         LLVMSetTarget(context->module, llvm_triple);
     347        1896 :         LLVMSetDataLayout(context->module, llvm_layout);
     348             :     }
     349             : 
     350       17774 :     return context->module;
     351             : }
     352             : 
     353             : /*
     354             :  * Expand function name to be non-conflicting. This should be used by code
     355             :  * generating code, when adding new externally visible function definitions to
     356             :  * a Module.
     357             :  */
     358             : char *
     359       17774 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
     360             : {
     361             :     Assert(context->module != NULL);
     362             : 
     363       17774 :     context->base.instr.created_functions++;
     364             : 
     365             :     /*
     366             :      * Previously we used dots to separate, but turns out some tools, e.g.
     367             :      * GDB, don't like that and truncate name.
     368             :      */
     369       35548 :     return psprintf("%s_%zu_%d",
     370             :                     basename,
     371             :                     context->module_generation,
     372       17774 :                     context->counter++);
     373             : }
     374             : 
     375             : /*
     376             :  * Return pointer to function funcname, which has to exist. If there's pending
     377             :  * code to be optimized and emitted, do so first.
     378             :  */
     379             : void *
     380        6196 : llvm_get_function(LLVMJitContext *context, const char *funcname)
     381             : {
     382             :     ListCell   *lc;
     383             : 
     384        6196 :     llvm_assert_in_fatal_section();
     385             : 
     386             :     /*
     387             :      * If there is a pending / not emitted module, compile and emit now.
     388             :      * Otherwise we might not find the [correct] function.
     389             :      */
     390        6196 :     if (!context->compiled)
     391             :     {
     392        1314 :         llvm_compile_module(context);
     393             :     }
     394             : 
     395             :     /*
     396             :      * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
     397             :      * to mangle here.
     398             :      */
     399             : 
     400             : #if LLVM_VERSION_MAJOR > 11
     401             :     foreach(lc, context->handles)
     402             :     {
     403             :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     404             :         instr_time  starttime;
     405             :         instr_time  endtime;
     406             :         LLVMErrorRef error;
     407             :         LLVMOrcJITTargetAddress addr;
     408             : 
     409             :         INSTR_TIME_SET_CURRENT(starttime);
     410             : 
     411             :         addr = 0;
     412             :         error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
     413             :         if (error)
     414             :             elog(ERROR, "failed to look up symbol \"%s\": %s",
     415             :                  funcname, llvm_error_message(error));
     416             : 
     417             :         /*
     418             :          * LLJIT only actually emits code the first time a symbol is
     419             :          * referenced. Thus add lookup time to emission time. That's counting
     420             :          * a bit more than with older LLVM versions, but unlikely to ever
     421             :          * matter.
     422             :          */
     423             :         INSTR_TIME_SET_CURRENT(endtime);
     424             :         INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     425             :                               endtime, starttime);
     426             : 
     427             :         if (addr)
     428             :             return (void *) (uintptr_t) addr;
     429             :     }
     430             : #else
     431        6330 :     foreach(lc, context->handles)
     432             :     {
     433             :         LLVMOrcTargetAddress addr;
     434        6330 :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     435             : 
     436        6330 :         addr = 0;
     437        6330 :         if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
     438           0 :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     439        6330 :         if (addr)
     440        6196 :             return (void *) (uintptr_t) addr;
     441             :     }
     442             : #endif
     443             : 
     444           0 :     elog(ERROR, "failed to JIT: %s", funcname);
     445             : 
     446             :     return NULL;
     447             : }
     448             : 
     449             : /*
     450             :  * Return type of a variable in llvmjit_types.c. This is useful to keep types
     451             :  * in sync between plain C and JIT related code.
     452             :  */
     453             : LLVMTypeRef
     454       16078 : llvm_pg_var_type(const char *varname)
     455             : {
     456             :     LLVMValueRef v_srcvar;
     457             :     LLVMTypeRef typ;
     458             : 
     459             :     /* this'll return a *pointer* to the global */
     460       16078 :     v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
     461       16078 :     if (!v_srcvar)
     462           0 :         elog(ERROR, "variable %s not in llvmjit_types.c", varname);
     463             : 
     464       16078 :     typ = LLVMGlobalGetValueType(v_srcvar);
     465             : 
     466       16078 :     return typ;
     467             : }
     468             : 
     469             : /*
     470             :  * Return function type of a variable in llvmjit_types.c. This is useful to
     471             :  * keep function types in sync between C and JITed code.
     472             :  */
     473             : LLVMTypeRef
     474       15604 : llvm_pg_var_func_type(const char *varname)
     475             : {
     476             :     LLVMValueRef v_srcvar;
     477             :     LLVMTypeRef typ;
     478             : 
     479       15604 :     v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
     480       15604 :     if (!v_srcvar)
     481           0 :         elog(ERROR, "function %s not in llvmjit_types.c", varname);
     482             : 
     483       15604 :     typ = LLVMGetFunctionType(v_srcvar);
     484             : 
     485       15604 :     return typ;
     486             : }
     487             : 
     488             : /*
     489             :  * Return declaration for a function referenced in llvmjit_types.c, adding it
     490             :  * to the module if necessary.
     491             :  *
     492             :  * This is used to make functions discovered via llvm_create_types() known to
     493             :  * the module that's currently being worked on.
     494             :  */
     495             : LLVMValueRef
     496       14224 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
     497             : {
     498             :     LLVMValueRef v_srcfn;
     499             :     LLVMValueRef v_fn;
     500             : 
     501             :     /* don't repeatedly add function */
     502       14224 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     503       14224 :     if (v_fn)
     504       10290 :         return v_fn;
     505             : 
     506        3934 :     v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
     507             : 
     508        3934 :     if (!v_srcfn)
     509           0 :         elog(ERROR, "function %s not in llvmjit_types.c", funcname);
     510             : 
     511        3934 :     v_fn = LLVMAddFunction(mod,
     512             :                            funcname,
     513             :                            LLVMGetFunctionType(v_srcfn));
     514        3934 :     llvm_copy_attributes(v_srcfn, v_fn);
     515             : 
     516        3934 :     return v_fn;
     517             : }
     518             : 
     519             : /*
     520             :  * Copy attributes from one function to another, for a specific index (an
     521             :  * index can reference return value, function and parameter attributes).
     522             :  */
     523             : static void
     524       71014 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
     525             : {
     526             :     int         num_attributes;
     527             :     LLVMAttributeRef *attrs;
     528             : 
     529       71014 :     num_attributes = LLVMGetAttributeCountAtIndex(v_from, index);
     530             : 
     531       71014 :     if (num_attributes == 0)
     532       31446 :         return;
     533             : 
     534       39568 :     attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
     535       39568 :     LLVMGetAttributesAtIndex(v_from, index, attrs);
     536             : 
     537      464050 :     for (int attno = 0; attno < num_attributes; attno++)
     538      424482 :         LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
     539             : 
     540       39568 :     pfree(attrs);
     541             : }
     542             : 
     543             : /*
     544             :  * Copy all attributes from one function to another. I.e. function, return and
     545             :  * parameters will be copied.
     546             :  */
     547             : void
     548       21708 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
     549             : {
     550             :     uint32      param_count;
     551             : 
     552             :     /* copy function attributes */
     553       21708 :     llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
     554             : 
     555             :     /* and the return value attributes */
     556       21708 :     llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
     557             : 
     558             :     /* and each function parameter's attribute */
     559       21708 :     param_count = LLVMCountParams(v_from);
     560             : 
     561       49306 :     for (int paramidx = 1; paramidx <= param_count; paramidx++)
     562       27598 :         llvm_copy_attributes_at_index(v_from, v_to, paramidx);
     563       21708 : }
     564             : 
     565             : /*
     566             :  * Return a callable LLVMValueRef for fcinfo.
     567             :  */
     568             : LLVMValueRef
     569        5814 : llvm_function_reference(LLVMJitContext *context,
     570             :                         LLVMBuilderRef builder,
     571             :                         LLVMModuleRef mod,
     572             :                         FunctionCallInfo fcinfo)
     573             : {
     574             :     char       *modname;
     575             :     char       *basename;
     576             :     char       *funcname;
     577             : 
     578             :     LLVMValueRef v_fn;
     579             : 
     580        5814 :     fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
     581             : 
     582        5814 :     if (modname != NULL && basename != NULL)
     583             :     {
     584             :         /* external function in loadable library */
     585          20 :         funcname = psprintf("pgextern.%s.%s", modname, basename);
     586             :     }
     587        5794 :     else if (basename != NULL)
     588             :     {
     589             :         /* internal function */
     590        5782 :         funcname = pstrdup(basename);
     591             :     }
     592             :     else
     593             :     {
     594             :         /*
     595             :          * Function we don't know to handle, return pointer. We do so by
     596             :          * creating a global constant containing a pointer to the function.
     597             :          * Makes IR more readable.
     598             :          */
     599             :         LLVMValueRef v_fn_addr;
     600             : 
     601          12 :         funcname = psprintf("pgoidextern.%u",
     602          12 :                             fcinfo->flinfo->fn_oid);
     603          12 :         v_fn = LLVMGetNamedGlobal(mod, funcname);
     604          12 :         if (v_fn != 0)
     605           0 :             return l_load(builder, TypePGFunction, v_fn, "");
     606             : 
     607          12 :         v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
     608             : 
     609          12 :         v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
     610          12 :         LLVMSetInitializer(v_fn, v_fn_addr);
     611          12 :         LLVMSetGlobalConstant(v_fn, true);
     612          12 :         LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
     613          12 :         LLVMSetUnnamedAddr(v_fn, true);
     614             : 
     615          12 :         return l_load(builder, TypePGFunction, v_fn, "");
     616             :     }
     617             : 
     618             :     /* check if function already has been added */
     619        5802 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     620        5802 :     if (v_fn != 0)
     621        2504 :         return v_fn;
     622             : 
     623        3298 :     v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
     624             : 
     625        3298 :     return v_fn;
     626             : }
     627             : 
     628             : /*
     629             :  * Optimize code in module using the flags set in context.
     630             :  */
     631             : static void
     632        1314 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
     633             : {
     634             : #if LLVM_VERSION_MAJOR < 17
     635             :     LLVMPassManagerBuilderRef llvm_pmb;
     636             :     LLVMPassManagerRef llvm_mpm;
     637             :     LLVMPassManagerRef llvm_fpm;
     638             :     LLVMValueRef func;
     639             :     int         compile_optlevel;
     640             : 
     641        1314 :     if (context->base.flags & PGJIT_OPT3)
     642         980 :         compile_optlevel = 3;
     643             :     else
     644         334 :         compile_optlevel = 0;
     645             : 
     646             :     /*
     647             :      * Have to create a new pass manager builder every pass through, as the
     648             :      * inliner has some per-builder state. Otherwise one ends up only inlining
     649             :      * a function the first time though.
     650             :      */
     651        1314 :     llvm_pmb = LLVMPassManagerBuilderCreate();
     652        1314 :     LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
     653        1314 :     llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
     654             : 
     655        1314 :     if (context->base.flags & PGJIT_OPT3)
     656             :     {
     657             :         /* TODO: Unscientifically determined threshold */
     658         980 :         LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
     659             :     }
     660             :     else
     661             :     {
     662             :         /* we rely on mem2reg heavily, so emit even in the O0 case */
     663         334 :         LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
     664             :     }
     665             : 
     666        1314 :     LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
     667             : 
     668             :     /*
     669             :      * Do function level optimization. This could be moved to the point where
     670             :      * functions are emitted, to reduce memory usage a bit.
     671             :      */
     672        1314 :     LLVMInitializeFunctionPassManager(llvm_fpm);
     673       31484 :     for (func = LLVMGetFirstFunction(context->module);
     674             :          func != NULL;
     675       30170 :          func = LLVMGetNextFunction(func))
     676       30170 :         LLVMRunFunctionPassManager(llvm_fpm, func);
     677        1314 :     LLVMFinalizeFunctionPassManager(llvm_fpm);
     678        1314 :     LLVMDisposePassManager(llvm_fpm);
     679             : 
     680             :     /*
     681             :      * Perform module level optimization. We do so even in the non-optimized
     682             :      * case, so always-inline functions etc get inlined. It's cheap enough.
     683             :      */
     684        1314 :     llvm_mpm = LLVMCreatePassManager();
     685        1314 :     LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
     686             :                                                     llvm_mpm);
     687             :     /* always use always-inliner pass */
     688        1314 :     if (!(context->base.flags & PGJIT_OPT3))
     689         334 :         LLVMAddAlwaysInlinerPass(llvm_mpm);
     690             :     /* if doing inlining, but no expensive optimization, add inlining pass */
     691        1314 :     if (context->base.flags & PGJIT_INLINE
     692         980 :         && !(context->base.flags & PGJIT_OPT3))
     693           0 :         LLVMAddFunctionInliningPass(llvm_mpm);
     694        1314 :     LLVMRunPassManager(llvm_mpm, context->module);
     695        1314 :     LLVMDisposePassManager(llvm_mpm);
     696             : 
     697        1314 :     LLVMPassManagerBuilderDispose(llvm_pmb);
     698             : #else
     699             :     LLVMPassBuilderOptionsRef options;
     700             :     LLVMErrorRef err;
     701             :     const char *passes;
     702             : 
     703             :     if (context->base.flags & PGJIT_OPT3)
     704             :         passes = "default<O3>";
     705             :     else
     706             :         passes = "default<O0>,mem2reg";
     707             : 
     708             :     options = LLVMCreatePassBuilderOptions();
     709             : 
     710             : #ifdef LLVM_PASS_DEBUG
     711             :     LLVMPassBuilderOptionsSetDebugLogging(options, 1);
     712             : #endif
     713             : 
     714             :     LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
     715             : 
     716             :     err = LLVMRunPasses(module, passes, NULL, options);
     717             : 
     718             :     if (err)
     719             :         elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
     720             : 
     721             :     LLVMDisposePassBuilderOptions(options);
     722             : #endif
     723        1314 : }
     724             : 
     725             : /*
     726             :  * Emit code for the currently pending module.
     727             :  */
     728             : static void
     729        1314 : llvm_compile_module(LLVMJitContext *context)
     730             : {
     731             :     LLVMJitHandle *handle;
     732             :     MemoryContext oldcontext;
     733             :     instr_time  starttime;
     734             :     instr_time  endtime;
     735             : #if LLVM_VERSION_MAJOR > 11
     736             :     LLVMOrcLLJITRef compile_orc;
     737             : #else
     738             :     LLVMOrcJITStackRef compile_orc;
     739             : #endif
     740             : 
     741        1314 :     if (context->base.flags & PGJIT_OPT3)
     742         980 :         compile_orc = llvm_opt3_orc;
     743             :     else
     744         334 :         compile_orc = llvm_opt0_orc;
     745             : 
     746             :     /* perform inlining */
     747        1314 :     if (context->base.flags & PGJIT_INLINE)
     748             :     {
     749         980 :         INSTR_TIME_SET_CURRENT(starttime);
     750         980 :         llvm_inline(context->module);
     751         980 :         INSTR_TIME_SET_CURRENT(endtime);
     752         980 :         INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
     753             :                               endtime, starttime);
     754             :     }
     755             : 
     756        1314 :     if (jit_dump_bitcode)
     757             :     {
     758             :         char       *filename;
     759             : 
     760           0 :         filename = psprintf("%d.%zu.bc",
     761             :                             MyProcPid,
     762             :                             context->module_generation);
     763           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     764           0 :         pfree(filename);
     765             :     }
     766             : 
     767             : 
     768             :     /* optimize according to the chosen optimization settings */
     769        1314 :     INSTR_TIME_SET_CURRENT(starttime);
     770        1314 :     llvm_optimize_module(context, context->module);
     771        1314 :     INSTR_TIME_SET_CURRENT(endtime);
     772        1314 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
     773             :                           endtime, starttime);
     774             : 
     775        1314 :     if (jit_dump_bitcode)
     776             :     {
     777             :         char       *filename;
     778             : 
     779           0 :         filename = psprintf("%d.%zu.optimized.bc",
     780             :                             MyProcPid,
     781             :                             context->module_generation);
     782           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     783           0 :         pfree(filename);
     784             :     }
     785             : 
     786             :     handle = (LLVMJitHandle *)
     787        1314 :         MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
     788             : 
     789             :     /*
     790             :      * Emit the code. Note that this can, depending on the optimization
     791             :      * settings, take noticeable resources as code emission executes low-level
     792             :      * instruction combining/selection passes etc. Without optimization a
     793             :      * faster instruction selection mechanism is used.
     794             :      */
     795        1314 :     INSTR_TIME_SET_CURRENT(starttime);
     796             : #if LLVM_VERSION_MAJOR > 11
     797             :     {
     798             :         LLVMOrcThreadSafeModuleRef ts_module;
     799             :         LLVMErrorRef error;
     800             :         LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
     801             : 
     802             :         ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
     803             : 
     804             :         handle->lljit = compile_orc;
     805             :         handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
     806             : 
     807             :         /*
     808             :          * NB: This doesn't actually emit code. That happens lazily the first
     809             :          * time a symbol defined in the module is requested. Due to that
     810             :          * llvm_get_function() also accounts for emission time.
     811             :          */
     812             : 
     813             :         context->module = NULL; /* will be owned by LLJIT */
     814             :         error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
     815             :                                                   handle->resource_tracker,
     816             :                                                   ts_module);
     817             : 
     818             :         if (error)
     819             :             elog(ERROR, "failed to JIT module: %s",
     820             :                  llvm_error_message(error));
     821             : 
     822             :         /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
     823             :     }
     824             : #else
     825             :     {
     826        1314 :         handle->stack = compile_orc;
     827        1314 :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
     828             :                                         llvm_resolve_symbol, NULL))
     829           0 :             elog(ERROR, "failed to JIT module");
     830             : 
     831             :         /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
     832             :     }
     833             : #endif
     834             : 
     835        1314 :     INSTR_TIME_SET_CURRENT(endtime);
     836        1314 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     837             :                           endtime, starttime);
     838             : 
     839        1314 :     context->module = NULL;
     840        1314 :     context->compiled = true;
     841             : 
     842             :     /* remember emitted code for cleanup and lookups */
     843        1314 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     844        1314 :     context->handles = lappend(context->handles, handle);
     845        1314 :     MemoryContextSwitchTo(oldcontext);
     846             : 
     847        1314 :     ereport(DEBUG1,
     848             :             (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
     849             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
     850             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
     851             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
     852             :              errhidestmt(true),
     853             :              errhidecontext(true)));
     854        1314 : }
     855             : 
     856             : /*
     857             :  * Per session initialization.
     858             :  */
     859             : static void
     860        1774 : llvm_session_initialize(void)
     861             : {
     862             :     MemoryContext oldcontext;
     863        1774 :     char       *error = NULL;
     864        1774 :     char       *cpu = NULL;
     865        1774 :     char       *features = NULL;
     866             :     LLVMTargetMachineRef opt0_tm;
     867             :     LLVMTargetMachineRef opt3_tm;
     868             : 
     869        1774 :     if (llvm_session_initialized)
     870         976 :         return;
     871             : 
     872         798 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     873             : 
     874         798 :     LLVMInitializeNativeTarget();
     875         798 :     LLVMInitializeNativeAsmPrinter();
     876         798 :     LLVMInitializeNativeAsmParser();
     877             : 
     878         798 :     if (llvm_context == NULL)
     879             :     {
     880         798 :         llvm_context = LLVMContextCreate();
     881             : 
     882         798 :         llvm_jit_context_in_use_count = 0;
     883         798 :         llvm_llvm_context_reuse_count = 0;
     884             :     }
     885             : 
     886             :     /*
     887             :      * When targeting LLVM 15, turn off opaque pointers for the context we
     888             :      * build our code in.  We don't need to do so for other contexts (e.g.
     889             :      * llvm_ts_context).  Once the IR is generated, it carries the necessary
     890             :      * information.
     891             :      *
     892             :      * For 16 and above, opaque pointers must be used, and we have special
     893             :      * code for that.
     894             :      */
     895             : #if LLVM_VERSION_MAJOR == 15
     896             :     LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
     897             : #endif
     898             : 
     899             :     /*
     900             :      * Synchronize types early, as that also includes inferring the target
     901             :      * triple.
     902             :      */
     903         798 :     llvm_create_types();
     904             : 
     905             :     /*
     906             :      * Extract target information from loaded module.
     907             :      */
     908         798 :     llvm_set_target();
     909             : 
     910         798 :     if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
     911             :     {
     912           0 :         elog(FATAL, "failed to query triple %s", error);
     913             :     }
     914             : 
     915             :     /*
     916             :      * We want the generated code to use all available features. Therefore
     917             :      * grab the host CPU string and detect features of the current CPU. The
     918             :      * latter is needed because some CPU architectures default to enabling
     919             :      * features not all CPUs have (weird, huh).
     920             :      */
     921         798 :     cpu = LLVMGetHostCPUName();
     922         798 :     features = LLVMGetHostCPUFeatures();
     923         798 :     elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
     924             :          cpu, features);
     925             : 
     926             :     opt0_tm =
     927         798 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     928             :                                 LLVMCodeGenLevelNone,
     929             :                                 LLVMRelocDefault,
     930             :                                 LLVMCodeModelJITDefault);
     931             :     opt3_tm =
     932         798 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     933             :                                 LLVMCodeGenLevelAggressive,
     934             :                                 LLVMRelocDefault,
     935             :                                 LLVMCodeModelJITDefault);
     936             : 
     937         798 :     LLVMDisposeMessage(cpu);
     938         798 :     cpu = NULL;
     939         798 :     LLVMDisposeMessage(features);
     940         798 :     features = NULL;
     941             : 
     942             :     /* force symbols in main binary to be loaded */
     943         798 :     LLVMLoadLibraryPermanently(NULL);
     944             : 
     945             : #if LLVM_VERSION_MAJOR > 11
     946             :     {
     947             :         llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
     948             : 
     949             :         llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
     950             :         opt0_tm = 0;
     951             : 
     952             :         llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
     953             :         opt3_tm = 0;
     954             :     }
     955             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     956             :     {
     957         798 :         llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
     958         798 :         llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
     959             : 
     960             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
     961         798 :         if (jit_debugging_support)
     962             :         {
     963           0 :             LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
     964             : 
     965           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     966           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     967             :         }
     968             : #endif
     969             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
     970         798 :         if (jit_profiling_support)
     971             :         {
     972           0 :             LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
     973             : 
     974           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     975           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     976             :         }
     977             : #endif
     978             :     }
     979             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     980             : 
     981         798 :     on_proc_exit(llvm_shutdown, 0);
     982             : 
     983         798 :     llvm_session_initialized = true;
     984             : 
     985         798 :     MemoryContextSwitchTo(oldcontext);
     986             : }
     987             : 
     988             : static void
     989         798 : llvm_shutdown(int code, Datum arg)
     990             : {
     991             :     /*
     992             :      * If llvm_shutdown() is reached while in a fatal-on-oom section an error
     993             :      * has occurred in the middle of LLVM code. It is not safe to call back
     994             :      * into LLVM (which is why a FATAL error was thrown).
     995             :      *
     996             :      * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
     997             :      * profiling data won't be written out.
     998             :      */
     999         798 :     if (llvm_in_fatal_on_oom())
    1000             :     {
    1001             :         Assert(proc_exit_inprogress);
    1002           0 :         return;
    1003             :     }
    1004             : 
    1005         798 :     if (llvm_jit_context_in_use_count != 0)
    1006           0 :         elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
    1007             :              llvm_jit_context_in_use_count);
    1008             : 
    1009             : #if LLVM_VERSION_MAJOR > 11
    1010             :     {
    1011             :         if (llvm_opt3_orc)
    1012             :         {
    1013             :             LLVMOrcDisposeLLJIT(llvm_opt3_orc);
    1014             :             llvm_opt3_orc = NULL;
    1015             :         }
    1016             :         if (llvm_opt0_orc)
    1017             :         {
    1018             :             LLVMOrcDisposeLLJIT(llvm_opt0_orc);
    1019             :             llvm_opt0_orc = NULL;
    1020             :         }
    1021             :         if (llvm_ts_context)
    1022             :         {
    1023             :             LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
    1024             :             llvm_ts_context = NULL;
    1025             :         }
    1026             :     }
    1027             : #else                           /* LLVM_VERSION_MAJOR > 11 */
    1028             :     {
    1029             :         /* unregister profiling support, needs to be flushed to be useful */
    1030             : 
    1031         798 :         if (llvm_opt3_orc)
    1032             :         {
    1033             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
    1034             :             if (jit_profiling_support)
    1035             :                 LLVMOrcUnregisterPerf(llvm_opt3_orc);
    1036             : #endif
    1037         798 :             LLVMOrcDisposeInstance(llvm_opt3_orc);
    1038         798 :             llvm_opt3_orc = NULL;
    1039             :         }
    1040             : 
    1041         798 :         if (llvm_opt0_orc)
    1042             :         {
    1043             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
    1044             :             if (jit_profiling_support)
    1045             :                 LLVMOrcUnregisterPerf(llvm_opt0_orc);
    1046             : #endif
    1047         798 :             LLVMOrcDisposeInstance(llvm_opt0_orc);
    1048         798 :             llvm_opt0_orc = NULL;
    1049             :         }
    1050             :     }
    1051             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1052             : }
    1053             : 
    1054             : /* helper for llvm_create_types, returning a function's return type */
    1055             : static LLVMTypeRef
    1056         798 : load_return_type(LLVMModuleRef mod, const char *name)
    1057             : {
    1058             :     LLVMValueRef value;
    1059             :     LLVMTypeRef typ;
    1060             : 
    1061             :     /* this'll return a *pointer* to the function */
    1062         798 :     value = LLVMGetNamedFunction(mod, name);
    1063         798 :     if (!value)
    1064           0 :         elog(ERROR, "function %s is unknown", name);
    1065             : 
    1066         798 :     typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
    1067             : 
    1068         798 :     return typ;
    1069             : }
    1070             : 
    1071             : /*
    1072             :  * Load triple & layout from clang emitted file so we're guaranteed to be
    1073             :  * compatible.
    1074             :  */
    1075             : static void
    1076         798 : llvm_set_target(void)
    1077             : {
    1078         798 :     if (!llvm_types_module)
    1079           0 :         elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
    1080             : 
    1081         798 :     if (llvm_triple == NULL)
    1082         798 :         llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
    1083             : 
    1084         798 :     if (llvm_layout == NULL)
    1085         798 :         llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
    1086         798 : }
    1087             : 
    1088             : /*
    1089             :  * Load required information, types, function signatures from llvmjit_types.c
    1090             :  * and make them available in global variables.
    1091             :  *
    1092             :  * Those global variables are then used while emitting code.
    1093             :  */
    1094             : static void
    1095         798 : llvm_create_types(void)
    1096             : {
    1097             :     char        path[MAXPGPATH];
    1098             :     LLVMMemoryBufferRef buf;
    1099             :     char       *msg;
    1100             : 
    1101         798 :     snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
    1102             : 
    1103             :     /* open file */
    1104         798 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
    1105             :     {
    1106           0 :         elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
    1107             :              path, msg);
    1108             :     }
    1109             : 
    1110             :     /* eagerly load contents, going to need it all */
    1111         798 :     if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
    1112             :     {
    1113           0 :         elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
    1114             :     }
    1115         798 :     LLVMDisposeMemoryBuffer(buf);
    1116             : 
    1117         798 :     TypeSizeT = llvm_pg_var_type("TypeSizeT");
    1118         798 :     TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
    1119         798 :     TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
    1120         798 :     TypePGFunction = llvm_pg_var_type("TypePGFunction");
    1121         798 :     StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
    1122         798 :     StructExprContext = llvm_pg_var_type("StructExprContext");
    1123         798 :     StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
    1124         798 :     StructExprState = llvm_pg_var_type("StructExprState");
    1125         798 :     StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
    1126         798 :     StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
    1127         798 :     StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
    1128         798 :     StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
    1129         798 :     StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
    1130         798 :     StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
    1131         798 :     StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
    1132         798 :     StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
    1133         798 :     StructAggState = llvm_pg_var_type("StructAggState");
    1134         798 :     StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
    1135         798 :     StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
    1136         798 :     StructPlanState = llvm_pg_var_type("StructPlanState");
    1137         798 :     StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
    1138             : 
    1139         798 :     AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
    1140         798 :     ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
    1141         798 :     ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
    1142         798 : }
    1143             : 
    1144             : /*
    1145             :  * Split a symbol into module / function parts.  If the function is in the
    1146             :  * main binary (or an external library) *modname will be NULL.
    1147             :  */
    1148             : void
    1149       31768 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
    1150             : {
    1151       31768 :     *modname = NULL;
    1152       31768 :     *funcname = NULL;
    1153             : 
    1154             :     /*
    1155             :      * Module function names are pgextern.$module.$funcname
    1156             :      */
    1157       31768 :     if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
    1158             :     {
    1159             :         /*
    1160             :          * Symbol names cannot contain a ., therefore we can split based on
    1161             :          * first and last occurrence of one.
    1162             :          */
    1163          58 :         *funcname = rindex(name, '.');
    1164          58 :         (*funcname)++;          /* jump over . */
    1165             : 
    1166         116 :         *modname = pnstrdup(name + strlen("pgextern."),
    1167          58 :                             *funcname - name - strlen("pgextern.") - 1);
    1168             :         Assert(funcname);
    1169             : 
    1170          58 :         *funcname = pstrdup(*funcname);
    1171             :     }
    1172             :     else
    1173             :     {
    1174       31710 :         *modname = NULL;
    1175       31710 :         *funcname = pstrdup(name);
    1176             :     }
    1177       31768 : }
    1178             : 
    1179             : /*
    1180             :  * Attempt to resolve symbol, so LLVM can emit a reference to it.
    1181             :  */
    1182             : static uint64_t
    1183        5212 : llvm_resolve_symbol(const char *symname, void *ctx)
    1184             : {
    1185             :     uintptr_t   addr;
    1186             :     char       *funcname;
    1187             :     char       *modname;
    1188             : 
    1189             :     /*
    1190             :      * macOS prefixes all object level symbols with an underscore. But neither
    1191             :      * dlsym() nor PG's inliner expect that. So undo.
    1192             :      */
    1193             : #if defined(__darwin__)
    1194             :     if (symname[0] != '_')
    1195             :         elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
    1196             :     symname++;
    1197             : #endif
    1198             : 
    1199        5212 :     llvm_split_symbol_name(symname, &modname, &funcname);
    1200             : 
    1201             :     /* functions that aren't resolved to names shouldn't ever get here */
    1202             :     Assert(funcname);
    1203             : 
    1204        5212 :     if (modname)
    1205          20 :         addr = (uintptr_t) load_external_function(modname, funcname,
    1206             :                                                   true, NULL);
    1207             :     else
    1208        5192 :         addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
    1209             : 
    1210        5212 :     pfree(funcname);
    1211        5212 :     if (modname)
    1212          20 :         pfree(modname);
    1213             : 
    1214             :     /* let LLVM will error out - should never happen */
    1215        5212 :     if (!addr)
    1216           0 :         elog(WARNING, "failed to resolve name %s", symname);
    1217             : 
    1218        5212 :     return (uint64_t) addr;
    1219             : }
    1220             : 
    1221             : #if LLVM_VERSION_MAJOR > 11
    1222             : 
    1223             : static LLVMErrorRef
    1224             : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
    1225             :                      LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
    1226             :                      LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
    1227             :                      LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
    1228             : {
    1229             : #if LLVM_VERSION_MAJOR > 14
    1230             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
    1231             : #else
    1232             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
    1233             : #endif
    1234             :     LLVMErrorRef error;
    1235             :     LLVMOrcMaterializationUnitRef mu;
    1236             : 
    1237             :     for (int i = 0; i < LookupSetSize; i++)
    1238             :     {
    1239             :         const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
    1240             : 
    1241             : #if LLVM_VERSION_MAJOR > 12
    1242             :         LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
    1243             : #endif
    1244             :         symbols[i].Name = LookupSet[i].Name;
    1245             :         symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
    1246             :         symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
    1247             :     }
    1248             : 
    1249             :     mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
    1250             :     error = LLVMOrcJITDylibDefine(JD, mu);
    1251             :     if (error != LLVMErrorSuccess)
    1252             :         LLVMOrcDisposeMaterializationUnit(mu);
    1253             : 
    1254             :     pfree(symbols);
    1255             : 
    1256             :     return error;
    1257             : }
    1258             : 
    1259             : /*
    1260             :  * We cannot throw errors through LLVM (without causing a FATAL at least), so
    1261             :  * just use WARNING here. That's OK anyway, as the error is also reported at
    1262             :  * the top level action (with less detail) and there might be multiple
    1263             :  * invocations of errors with details.
    1264             :  *
    1265             :  * This doesn't really happen during normal operation, but in cases like
    1266             :  * symbol resolution breakage. So just using elog(WARNING) is fine.
    1267             :  */
    1268             : static void
    1269             : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
    1270             : {
    1271             :     elog(WARNING, "error during JITing: %s",
    1272             :          llvm_error_message(error));
    1273             : }
    1274             : 
    1275             : /*
    1276             :  * Create our own object layer, so we can add event listeners.
    1277             :  */
    1278             : static LLVMOrcObjectLayerRef
    1279             : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
    1280             : {
    1281             :     LLVMOrcObjectLayerRef objlayer =
    1282             :         LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
    1283             : 
    1284             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
    1285             :     if (jit_debugging_support)
    1286             :     {
    1287             :         LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
    1288             : 
    1289             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1290             :     }
    1291             : #endif
    1292             : 
    1293             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
    1294             :     if (jit_profiling_support)
    1295             :     {
    1296             :         LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
    1297             : 
    1298             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1299             :     }
    1300             : #endif
    1301             : 
    1302             :     return objlayer;
    1303             : }
    1304             : 
    1305             : /*
    1306             :  * Create LLJIT instance, using the passed in target machine. Note that the
    1307             :  * target machine afterwards is owned by the LLJIT instance.
    1308             :  */
    1309             : static LLVMOrcLLJITRef
    1310             : llvm_create_jit_instance(LLVMTargetMachineRef tm)
    1311             : {
    1312             :     LLVMOrcLLJITRef lljit;
    1313             :     LLVMOrcJITTargetMachineBuilderRef tm_builder;
    1314             :     LLVMOrcLLJITBuilderRef lljit_builder;
    1315             :     LLVMErrorRef error;
    1316             :     LLVMOrcDefinitionGeneratorRef main_gen;
    1317             :     LLVMOrcDefinitionGeneratorRef ref_gen;
    1318             : 
    1319             :     lljit_builder = LLVMOrcCreateLLJITBuilder();
    1320             :     tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
    1321             :     LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
    1322             : 
    1323             :     LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
    1324             :                                                     llvm_create_object_layer,
    1325             :                                                     NULL);
    1326             : 
    1327             :     error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
    1328             :     if (error)
    1329             :         elog(ERROR, "failed to create lljit instance: %s",
    1330             :              llvm_error_message(error));
    1331             : 
    1332             :     LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
    1333             :                                             llvm_log_jit_error, NULL);
    1334             : 
    1335             :     /*
    1336             :      * Symbol resolution support for symbols in the postgres binary /
    1337             :      * libraries already loaded.
    1338             :      */
    1339             :     error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
    1340             :                                                                  LLVMOrcLLJITGetGlobalPrefix(lljit),
    1341             :                                                                  0, NULL);
    1342             :     if (error)
    1343             :         elog(ERROR, "failed to create generator: %s",
    1344             :              llvm_error_message(error));
    1345             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
    1346             : 
    1347             :     /*
    1348             :      * Symbol resolution support for "special" functions, e.g. a call into an
    1349             :      * SQL callable function.
    1350             :      */
    1351             : #if LLVM_VERSION_MAJOR > 14
    1352             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
    1353             : #else
    1354             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
    1355             : #endif
    1356             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
    1357             : 
    1358             :     return lljit;
    1359             : }
    1360             : 
    1361             : static char *
    1362             : llvm_error_message(LLVMErrorRef error)
    1363             : {
    1364             :     char       *orig = LLVMGetErrorMessage(error);
    1365             :     char       *msg = pstrdup(orig);
    1366             : 
    1367             :     LLVMDisposeErrorMessage(orig);
    1368             : 
    1369             :     return msg;
    1370             : }
    1371             : 
    1372             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1373             : 
    1374             : /*
    1375             :  * ResourceOwner callbacks
    1376             :  */
    1377             : static void
    1378          16 : ResOwnerReleaseJitContext(Datum res)
    1379             : {
    1380          16 :     JitContext *context = (JitContext *) DatumGetPointer(res);
    1381             : 
    1382          16 :     context->resowner = NULL;
    1383          16 :     jit_release_context(context);
    1384          16 : }

Generated by: LCOV version 1.14