LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit.c (source / functions) Hit Total Coverage
Test: PostgreSQL 17devel Lines: 293 330 88.8 %
Date: 2023-12-07 07:10:44 Functions: 26 26 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit.c
       4             :  *    Core part of the LLVM JIT provider.
       5             :  *
       6             :  * Copyright (c) 2016-2023, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/jit/llvm/llvmjit.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres.h"
      15             : 
      16             : #include <llvm-c/Analysis.h>
      17             : #include <llvm-c/BitReader.h>
      18             : #include <llvm-c/BitWriter.h>
      19             : #include <llvm-c/Core.h>
      20             : #include <llvm-c/ExecutionEngine.h>
      21             : #if LLVM_VERSION_MAJOR > 16
      22             : #include <llvm-c/Transforms/PassBuilder.h>
      23             : #endif
      24             : #if LLVM_VERSION_MAJOR > 11
      25             : #include <llvm-c/Orc.h>
      26             : #include <llvm-c/OrcEE.h>
      27             : #include <llvm-c/LLJIT.h>
      28             : #else
      29             : #include <llvm-c/OrcBindings.h>
      30             : #endif
      31             : #include <llvm-c/Support.h>
      32             : #include <llvm-c/Target.h>
      33             : #if LLVM_VERSION_MAJOR < 17
      34             : #include <llvm-c/Transforms/IPO.h>
      35             : #include <llvm-c/Transforms/PassManagerBuilder.h>
      36             : #include <llvm-c/Transforms/Scalar.h>
      37             : #if LLVM_VERSION_MAJOR > 6
      38             : #include <llvm-c/Transforms/Utils.h>
      39             : #endif
      40             : #endif
      41             : 
      42             : #include "jit/llvmjit.h"
      43             : #include "jit/llvmjit_emit.h"
      44             : #include "miscadmin.h"
      45             : #include "portability/instr_time.h"
      46             : #include "storage/ipc.h"
      47             : #include "utils/memutils.h"
      48             : #include "utils/resowner.h"
      49             : 
      50             : #define LLVMJIT_LLVM_CONTEXT_REUSE_MAX 100
      51             : 
      52             : /* Handle of a module emitted via ORC JIT */
      53             : typedef struct LLVMJitHandle
      54             : {
      55             : #if LLVM_VERSION_MAJOR > 11
      56             :     LLVMOrcLLJITRef lljit;
      57             :     LLVMOrcResourceTrackerRef resource_tracker;
      58             : #else
      59             :     LLVMOrcJITStackRef stack;
      60             :     LLVMOrcModuleHandle orc_handle;
      61             : #endif
      62             : } LLVMJitHandle;
      63             : 
      64             : 
      65             : /* types & functions commonly needed for JITing */
      66             : LLVMTypeRef TypeSizeT;
      67             : LLVMTypeRef TypeParamBool;
      68             : LLVMTypeRef TypeStorageBool;
      69             : LLVMTypeRef TypePGFunction;
      70             : LLVMTypeRef StructNullableDatum;
      71             : LLVMTypeRef StructHeapTupleData;
      72             : LLVMTypeRef StructMinimalTupleData;
      73             : LLVMTypeRef StructTupleDescData;
      74             : LLVMTypeRef StructTupleTableSlot;
      75             : LLVMTypeRef StructHeapTupleHeaderData;
      76             : LLVMTypeRef StructHeapTupleTableSlot;
      77             : LLVMTypeRef StructMinimalTupleTableSlot;
      78             : LLVMTypeRef StructMemoryContextData;
      79             : LLVMTypeRef StructFunctionCallInfoData;
      80             : LLVMTypeRef StructExprContext;
      81             : LLVMTypeRef StructExprEvalStep;
      82             : LLVMTypeRef StructExprState;
      83             : LLVMTypeRef StructAggState;
      84             : LLVMTypeRef StructAggStatePerGroupData;
      85             : LLVMTypeRef StructAggStatePerTransData;
      86             : LLVMTypeRef StructPlanState;
      87             : 
      88             : LLVMValueRef AttributeTemplate;
      89             : LLVMValueRef ExecEvalSubroutineTemplate;
      90             : LLVMValueRef ExecEvalBoolSubroutineTemplate;
      91             : 
      92             : static LLVMModuleRef llvm_types_module = NULL;
      93             : 
      94             : static bool llvm_session_initialized = false;
      95             : static size_t llvm_generation = 0;
      96             : 
      97             : /* number of LLVMJitContexts that currently are in use */
      98             : static size_t llvm_jit_context_in_use_count = 0;
      99             : 
     100             : /* how many times has the current LLVMContextRef been used */
     101             : static size_t llvm_llvm_context_reuse_count = 0;
     102             : static const char *llvm_triple = NULL;
     103             : static const char *llvm_layout = NULL;
     104             : static LLVMContextRef llvm_context;
     105             : 
     106             : 
     107             : static LLVMTargetRef llvm_targetref;
     108             : #if LLVM_VERSION_MAJOR > 11
     109             : static LLVMOrcThreadSafeContextRef llvm_ts_context;
     110             : static LLVMOrcLLJITRef llvm_opt0_orc;
     111             : static LLVMOrcLLJITRef llvm_opt3_orc;
     112             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     113             : static LLVMOrcJITStackRef llvm_opt0_orc;
     114             : static LLVMOrcJITStackRef llvm_opt3_orc;
     115             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     116             : 
     117             : 
     118             : static void llvm_release_context(JitContext *context);
     119             : static void llvm_session_initialize(void);
     120             : static void llvm_shutdown(int code, Datum arg);
     121             : static void llvm_compile_module(LLVMJitContext *context);
     122             : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
     123             : 
     124             : static void llvm_create_types(void);
     125             : static void llvm_set_target(void);
     126             : static void llvm_recreate_llvm_context(void);
     127             : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
     128             : 
     129             : #if LLVM_VERSION_MAJOR > 11
     130             : static LLVMOrcLLJITRef llvm_create_jit_instance(LLVMTargetMachineRef tm);
     131             : static char *llvm_error_message(LLVMErrorRef error);
     132             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     133             : 
     134             : /* ResourceOwner callbacks to hold JitContexts  */
     135             : static void ResOwnerReleaseJitContext(Datum res);
     136             : 
     137             : static const ResourceOwnerDesc jit_resowner_desc =
     138             : {
     139             :     .name = "LLVM JIT context",
     140             :     .release_phase = RESOURCE_RELEASE_BEFORE_LOCKS,
     141             :     .release_priority = RELEASE_PRIO_JIT_CONTEXTS,
     142             :     .ReleaseResource = ResOwnerReleaseJitContext,
     143             :     .DebugPrint = NULL          /* the default message is fine */
     144             : };
     145             : 
     146             : /* Convenience wrappers over ResourceOwnerRemember/Forget */
     147             : static inline void
     148        1672 : ResourceOwnerRememberJIT(ResourceOwner owner, LLVMJitContext *handle)
     149             : {
     150        1672 :     ResourceOwnerRemember(owner, PointerGetDatum(handle), &jit_resowner_desc);
     151        1672 : }
     152             : static inline void
     153        1656 : ResourceOwnerForgetJIT(ResourceOwner owner, LLVMJitContext *handle)
     154             : {
     155        1656 :     ResourceOwnerForget(owner, PointerGetDatum(handle), &jit_resowner_desc);
     156        1656 : }
     157             : 
     158        1960 : PG_MODULE_MAGIC;
     159             : 
     160             : 
     161             : /*
     162             :  * Initialize LLVM JIT provider.
     163             :  */
     164             : void
     165         770 : _PG_jit_provider_init(JitProviderCallbacks *cb)
     166             : {
     167         770 :     cb->reset_after_error = llvm_reset_after_error;
     168         770 :     cb->release_context = llvm_release_context;
     169         770 :     cb->compile_expr = llvm_compile_expr;
     170         770 : }
     171             : 
     172             : 
     173             : /*
     174             :  * Every now and then create a new LLVMContextRef. Unfortunately, during every
     175             :  * round of inlining, types may "leak" (they can still be found/used via the
     176             :  * context, but new types will be created the next time in inlining is
     177             :  * performed). To prevent that from slowly accumulating problematic amounts of
     178             :  * memory, recreate the LLVMContextRef we use. We don't want to do so too
     179             :  * often, as that implies some overhead (particularly re-loading the module
     180             :  * summaries / modules is fairly expensive). A future TODO would be to make
     181             :  * this more finegrained and only drop/recreate the LLVMContextRef when we know
     182             :  * there has been inlining. If we can get the size of the context from LLVM
     183             :  * then that might be a better way to determine when to drop/recreate rather
     184             :  * then the usagecount heuristic currently employed.
     185             :  */
     186             : static void
     187        1672 : llvm_recreate_llvm_context(void)
     188             : {
     189        1672 :     if (!llvm_context)
     190           0 :         elog(ERROR, "Trying to recreate a non-existing context");
     191             : 
     192             :     /*
     193             :      * We can only safely recreate the LLVM context if no other code is being
     194             :      * JITed, otherwise we'd release the types in use for that.
     195             :      */
     196        1672 :     if (llvm_jit_context_in_use_count > 0)
     197             :     {
     198           0 :         llvm_llvm_context_reuse_count++;
     199           0 :         return;
     200             :     }
     201             : 
     202        1672 :     if (llvm_llvm_context_reuse_count <= LLVMJIT_LLVM_CONTEXT_REUSE_MAX)
     203             :     {
     204        1672 :         llvm_llvm_context_reuse_count++;
     205        1672 :         return;
     206             :     }
     207             : 
     208             :     /*
     209             :      * Need to reset the modules that the inlining code caches before
     210             :      * disposing of the context. LLVM modules exist within a specific LLVM
     211             :      * context, therefore disposing of the context before resetting the cache
     212             :      * would lead to dangling pointers to modules.
     213             :      */
     214           0 :     llvm_inline_reset_caches();
     215             : 
     216           0 :     LLVMContextDispose(llvm_context);
     217           0 :     llvm_context = LLVMContextCreate();
     218           0 :     llvm_llvm_context_reuse_count = 0;
     219             : 
     220             :     /*
     221             :      * Re-build cached type information, so code generation code can rely on
     222             :      * that information to be present (also prevents the variables to be
     223             :      * dangling references).
     224             :      */
     225           0 :     llvm_create_types();
     226             : }
     227             : 
     228             : 
     229             : /*
     230             :  * Create a context for JITing work.
     231             :  *
     232             :  * The context, including subsidiary resources, will be cleaned up either when
     233             :  * the context is explicitly released, or when the lifetime of
     234             :  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
     235             :  */
     236             : LLVMJitContext *
     237        1672 : llvm_create_context(int jitFlags)
     238             : {
     239             :     LLVMJitContext *context;
     240             : 
     241        1672 :     llvm_assert_in_fatal_section();
     242             : 
     243        1672 :     llvm_session_initialize();
     244             : 
     245        1672 :     llvm_recreate_llvm_context();
     246             : 
     247        1672 :     ResourceOwnerEnlarge(CurrentResourceOwner);
     248             : 
     249        1672 :     context = MemoryContextAllocZero(TopMemoryContext,
     250             :                                      sizeof(LLVMJitContext));
     251        1672 :     context->base.flags = jitFlags;
     252             : 
     253             :     /* ensure cleanup */
     254        1672 :     context->base.resowner = CurrentResourceOwner;
     255        1672 :     ResourceOwnerRememberJIT(CurrentResourceOwner, context);
     256             : 
     257        1672 :     llvm_jit_context_in_use_count++;
     258             : 
     259        1672 :     return context;
     260             : }
     261             : 
     262             : /*
     263             :  * Release resources required by one llvm context.
     264             :  */
     265             : static void
     266        1672 : llvm_release_context(JitContext *context)
     267             : {
     268        1672 :     LLVMJitContext *llvm_jit_context = (LLVMJitContext *) context;
     269             :     ListCell   *lc;
     270             : 
     271             :     /*
     272             :      * Consider as cleaned up even if we skip doing so below, that way we can
     273             :      * verify the tracking is correct (see llvm_shutdown()).
     274             :      */
     275        1672 :     llvm_jit_context_in_use_count--;
     276             : 
     277             :     /*
     278             :      * When this backend is exiting, don't clean up LLVM. As an error might
     279             :      * have occurred from within LLVM, we do not want to risk reentering. All
     280             :      * resource cleanup is going to happen through process exit.
     281             :      */
     282        1672 :     if (proc_exit_inprogress)
     283           0 :         return;
     284             : 
     285        1672 :     llvm_enter_fatal_on_oom();
     286             : 
     287        1672 :     if (llvm_jit_context->module)
     288             :     {
     289         486 :         LLVMDisposeModule(llvm_jit_context->module);
     290         486 :         llvm_jit_context->module = NULL;
     291             :     }
     292             : 
     293        2980 :     foreach(lc, llvm_jit_context->handles)
     294             :     {
     295        1308 :         LLVMJitHandle *jit_handle = (LLVMJitHandle *) lfirst(lc);
     296             : 
     297             : #if LLVM_VERSION_MAJOR > 11
     298             :         {
     299             :             LLVMOrcExecutionSessionRef ee;
     300             :             LLVMOrcSymbolStringPoolRef sp;
     301             : 
     302             :             LLVMOrcResourceTrackerRemove(jit_handle->resource_tracker);
     303             :             LLVMOrcReleaseResourceTracker(jit_handle->resource_tracker);
     304             : 
     305             :             /*
     306             :              * Without triggering cleanup of the string pool, we'd leak
     307             :              * memory. It'd be sufficient to do this far less often, but in
     308             :              * experiments the required time was small enough to just always
     309             :              * do it.
     310             :              */
     311             :             ee = LLVMOrcLLJITGetExecutionSession(jit_handle->lljit);
     312             :             sp = LLVMOrcExecutionSessionGetSymbolStringPool(ee);
     313             :             LLVMOrcSymbolStringPoolClearDeadEntries(sp);
     314             :         }
     315             : #else                           /* LLVM_VERSION_MAJOR > 11 */
     316             :         {
     317        1308 :             LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
     318             :         }
     319             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
     320             : 
     321        1308 :         pfree(jit_handle);
     322             :     }
     323        1672 :     list_free(llvm_jit_context->handles);
     324        1672 :     llvm_jit_context->handles = NIL;
     325             : 
     326        1672 :     llvm_leave_fatal_on_oom();
     327             : 
     328        1672 :     if (context->resowner)
     329        1656 :         ResourceOwnerForgetJIT(context->resowner, llvm_jit_context);
     330             : }
     331             : 
     332             : /*
     333             :  * Return module which may be modified, e.g. by creating new functions.
     334             :  */
     335             : LLVMModuleRef
     336       16194 : llvm_mutable_module(LLVMJitContext *context)
     337             : {
     338       16194 :     llvm_assert_in_fatal_section();
     339             : 
     340             :     /*
     341             :      * If there's no in-progress module, create a new one.
     342             :      */
     343       16194 :     if (!context->module)
     344             :     {
     345        1794 :         context->compiled = false;
     346        1794 :         context->module_generation = llvm_generation++;
     347        1794 :         context->module = LLVMModuleCreateWithNameInContext("pg", llvm_context);
     348        1794 :         LLVMSetTarget(context->module, llvm_triple);
     349        1794 :         LLVMSetDataLayout(context->module, llvm_layout);
     350             :     }
     351             : 
     352       16194 :     return context->module;
     353             : }
     354             : 
     355             : /*
     356             :  * Expand function name to be non-conflicting. This should be used by code
     357             :  * generating code, when adding new externally visible function definitions to
     358             :  * a Module.
     359             :  */
     360             : char *
     361       16194 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
     362             : {
     363             :     Assert(context->module != NULL);
     364             : 
     365       16194 :     context->base.instr.created_functions++;
     366             : 
     367             :     /*
     368             :      * Previously we used dots to separate, but turns out some tools, e.g.
     369             :      * GDB, don't like that and truncate name.
     370             :      */
     371       32388 :     return psprintf("%s_%zu_%d",
     372             :                     basename,
     373             :                     context->module_generation,
     374       16194 :                     context->counter++);
     375             : }
     376             : 
     377             : /*
     378             :  * Return pointer to function funcname, which has to exist. If there's pending
     379             :  * code to be optimized and emitted, do so first.
     380             :  */
     381             : void *
     382        6002 : llvm_get_function(LLVMJitContext *context, const char *funcname)
     383             : {
     384             : #if LLVM_VERSION_MAJOR > 11 || \
     385             :     defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
     386             :     ListCell   *lc;
     387             : #endif
     388             : 
     389        6002 :     llvm_assert_in_fatal_section();
     390             : 
     391             :     /*
     392             :      * If there is a pending / not emitted module, compile and emit now.
     393             :      * Otherwise we might not find the [correct] function.
     394             :      */
     395        6002 :     if (!context->compiled)
     396             :     {
     397        1308 :         llvm_compile_module(context);
     398             :     }
     399             : 
     400             :     /*
     401             :      * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
     402             :      * to mangle here.
     403             :      */
     404             : 
     405             : #if LLVM_VERSION_MAJOR > 11
     406             :     foreach(lc, context->handles)
     407             :     {
     408             :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     409             :         instr_time  starttime;
     410             :         instr_time  endtime;
     411             :         LLVMErrorRef error;
     412             :         LLVMOrcJITTargetAddress addr;
     413             : 
     414             :         INSTR_TIME_SET_CURRENT(starttime);
     415             : 
     416             :         addr = 0;
     417             :         error = LLVMOrcLLJITLookup(handle->lljit, &addr, funcname);
     418             :         if (error)
     419             :             elog(ERROR, "failed to look up symbol \"%s\": %s",
     420             :                  funcname, llvm_error_message(error));
     421             : 
     422             :         /*
     423             :          * LLJIT only actually emits code the first time a symbol is
     424             :          * referenced. Thus add lookup time to emission time. That's counting
     425             :          * a bit more than with older LLVM versions, but unlikely to ever
     426             :          * matter.
     427             :          */
     428             :         INSTR_TIME_SET_CURRENT(endtime);
     429             :         INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     430             :                               endtime, starttime);
     431             : 
     432             :         if (addr)
     433             :             return (void *) (uintptr_t) addr;
     434             :     }
     435             : #elif defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
     436        6136 :     foreach(lc, context->handles)
     437             :     {
     438             :         LLVMOrcTargetAddress addr;
     439        6136 :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     440             : 
     441        6136 :         addr = 0;
     442        6136 :         if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
     443           0 :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     444        6136 :         if (addr)
     445        6002 :             return (void *) (uintptr_t) addr;
     446             :     }
     447             : #elif LLVM_VERSION_MAJOR < 5
     448             :     {
     449             :         LLVMOrcTargetAddress addr;
     450             : 
     451             :         if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
     452             :             return (void *) (uintptr_t) addr;
     453             :         if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
     454             :             return (void *) (uintptr_t) addr;
     455             :     }
     456             : #else
     457             :     {
     458             :         LLVMOrcTargetAddress addr;
     459             : 
     460             :         if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
     461             :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     462             :         if (addr)
     463             :             return (void *) (uintptr_t) addr;
     464             :         if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
     465             :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     466             :         if (addr)
     467             :             return (void *) (uintptr_t) addr;
     468             :     }
     469             : #endif
     470             : 
     471           0 :     elog(ERROR, "failed to JIT: %s", funcname);
     472             : 
     473             :     return NULL;
     474             : }
     475             : 
     476             : /*
     477             :  * Return type of a variable in llvmjit_types.c. This is useful to keep types
     478             :  * in sync between plain C and JIT related code.
     479             :  */
     480             : LLVMTypeRef
     481       15518 : llvm_pg_var_type(const char *varname)
     482             : {
     483             :     LLVMValueRef v_srcvar;
     484             :     LLVMTypeRef typ;
     485             : 
     486             :     /* this'll return a *pointer* to the global */
     487       15518 :     v_srcvar = LLVMGetNamedGlobal(llvm_types_module, varname);
     488       15518 :     if (!v_srcvar)
     489           0 :         elog(ERROR, "variable %s not in llvmjit_types.c", varname);
     490             : 
     491       15518 :     typ = LLVMGlobalGetValueType(v_srcvar);
     492             : 
     493       15518 :     return typ;
     494             : }
     495             : 
     496             : /*
     497             :  * Return function type of a variable in llvmjit_types.c. This is useful to
     498             :  * keep function types in sync between C and JITed code.
     499             :  */
     500             : LLVMTypeRef
     501       14638 : llvm_pg_var_func_type(const char *varname)
     502             : {
     503             :     LLVMValueRef v_srcvar;
     504             :     LLVMTypeRef typ;
     505             : 
     506       14638 :     v_srcvar = LLVMGetNamedFunction(llvm_types_module, varname);
     507       14638 :     if (!v_srcvar)
     508           0 :         elog(ERROR, "function %s not in llvmjit_types.c", varname);
     509             : 
     510       14638 :     typ = LLVMGetFunctionType(v_srcvar);
     511             : 
     512       14638 :     return typ;
     513             : }
     514             : 
     515             : /*
     516             :  * Return declaration for a function referenced in llvmjit_types.c, adding it
     517             :  * to the module if necessary.
     518             :  *
     519             :  * This is used to make functions discovered via llvm_create_types() known to
     520             :  * the module that's currently being worked on.
     521             :  */
     522             : LLVMValueRef
     523       13244 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
     524             : {
     525             :     LLVMValueRef v_srcfn;
     526             :     LLVMValueRef v_fn;
     527             : 
     528             :     /* don't repeatedly add function */
     529       13244 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     530       13244 :     if (v_fn)
     531        9500 :         return v_fn;
     532             : 
     533        3744 :     v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
     534             : 
     535        3744 :     if (!v_srcfn)
     536           0 :         elog(ERROR, "function %s not in llvmjit_types.c", funcname);
     537             : 
     538        3744 :     v_fn = LLVMAddFunction(mod,
     539             :                            funcname,
     540             :                            LLVMGetFunctionType(v_srcfn));
     541        3744 :     llvm_copy_attributes(v_srcfn, v_fn);
     542             : 
     543        3744 :     return v_fn;
     544             : }
     545             : 
     546             : /*
     547             :  * Copy attributes from one function to another, for a specific index (an
     548             :  * index can reference return value, function and parameter attributes).
     549             :  */
     550             : static void
     551       65360 : llvm_copy_attributes_at_index(LLVMValueRef v_from, LLVMValueRef v_to, uint32 index)
     552             : {
     553             :     int         num_attributes;
     554             :     LLVMAttributeRef *attrs;
     555             : 
     556       65360 :     num_attributes = LLVMGetAttributeCountAtIndexPG(v_from, index);
     557             : 
     558             :     /*
     559             :      * Not just for efficiency: LLVM <= 3.9 crashes when
     560             :      * LLVMGetAttributesAtIndex() is called for an index with 0 attributes.
     561             :      */
     562       65360 :     if (num_attributes == 0)
     563       29166 :         return;
     564             : 
     565       36194 :     attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
     566       36194 :     LLVMGetAttributesAtIndex(v_from, index, attrs);
     567             : 
     568      425002 :     for (int attno = 0; attno < num_attributes; attno++)
     569      388808 :         LLVMAddAttributeAtIndex(v_to, index, attrs[attno]);
     570             : 
     571       36194 :     pfree(attrs);
     572             : }
     573             : 
     574             : /*
     575             :  * Copy all attributes from one function to another. I.e. function, return and
     576             :  * parameters will be copied.
     577             :  */
     578             : void
     579       19938 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
     580             : {
     581             :     uint32      param_count;
     582             : 
     583             :     /* copy function attributes */
     584       19938 :     llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeFunctionIndex);
     585             : 
     586             :     /* and the return value attributes */
     587       19938 :     llvm_copy_attributes_at_index(v_from, v_to, LLVMAttributeReturnIndex);
     588             : 
     589             :     /* and each function parameter's attribute */
     590       19938 :     param_count = LLVMCountParams(v_from);
     591             : 
     592       45422 :     for (int paramidx = 1; paramidx <= param_count; paramidx++)
     593       25484 :         llvm_copy_attributes_at_index(v_from, v_to, paramidx);
     594       19938 : }
     595             : 
     596             : /*
     597             :  * Return a callable LLVMValueRef for fcinfo.
     598             :  */
     599             : LLVMValueRef
     600        5476 : llvm_function_reference(LLVMJitContext *context,
     601             :                         LLVMBuilderRef builder,
     602             :                         LLVMModuleRef mod,
     603             :                         FunctionCallInfo fcinfo)
     604             : {
     605             :     char       *modname;
     606             :     char       *basename;
     607             :     char       *funcname;
     608             : 
     609             :     LLVMValueRef v_fn;
     610             : 
     611        5476 :     fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
     612             : 
     613        5476 :     if (modname != NULL && basename != NULL)
     614             :     {
     615             :         /* external function in loadable library */
     616          38 :         funcname = psprintf("pgextern.%s.%s", modname, basename);
     617             :     }
     618        5438 :     else if (basename != NULL)
     619             :     {
     620             :         /* internal function */
     621        5434 :         funcname = pstrdup(basename);
     622             :     }
     623             :     else
     624             :     {
     625             :         /*
     626             :          * Function we don't know to handle, return pointer. We do so by
     627             :          * creating a global constant containing a pointer to the function.
     628             :          * Makes IR more readable.
     629             :          */
     630             :         LLVMValueRef v_fn_addr;
     631             : 
     632           4 :         funcname = psprintf("pgoidextern.%u",
     633           4 :                             fcinfo->flinfo->fn_oid);
     634           4 :         v_fn = LLVMGetNamedGlobal(mod, funcname);
     635           4 :         if (v_fn != 0)
     636           0 :             return l_load(builder, TypePGFunction, v_fn, "");
     637             : 
     638           4 :         v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
     639             : 
     640           4 :         v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
     641           4 :         LLVMSetInitializer(v_fn, v_fn_addr);
     642           4 :         LLVMSetGlobalConstant(v_fn, true);
     643           4 :         LLVMSetLinkage(v_fn, LLVMPrivateLinkage);
     644           4 :         LLVMSetUnnamedAddr(v_fn, true);
     645             : 
     646           4 :         return l_load(builder, TypePGFunction, v_fn, "");
     647             :     }
     648             : 
     649             :     /* check if function already has been added */
     650        5472 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     651        5472 :     if (v_fn != 0)
     652        2318 :         return v_fn;
     653             : 
     654        3154 :     v_fn = LLVMAddFunction(mod, funcname, LLVMGetFunctionType(AttributeTemplate));
     655             : 
     656        3154 :     return v_fn;
     657             : }
     658             : 
     659             : /*
     660             :  * Optimize code in module using the flags set in context.
     661             :  */
     662             : static void
     663        1308 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
     664             : {
     665             : #if LLVM_VERSION_MAJOR < 17
     666             :     LLVMPassManagerBuilderRef llvm_pmb;
     667             :     LLVMPassManagerRef llvm_mpm;
     668             :     LLVMPassManagerRef llvm_fpm;
     669             :     LLVMValueRef func;
     670             :     int         compile_optlevel;
     671             : 
     672        1308 :     if (context->base.flags & PGJIT_OPT3)
     673         978 :         compile_optlevel = 3;
     674             :     else
     675         330 :         compile_optlevel = 0;
     676             : 
     677             :     /*
     678             :      * Have to create a new pass manager builder every pass through, as the
     679             :      * inliner has some per-builder state. Otherwise one ends up only inlining
     680             :      * a function the first time though.
     681             :      */
     682        1308 :     llvm_pmb = LLVMPassManagerBuilderCreate();
     683        1308 :     LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
     684        1308 :     llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
     685             : 
     686        1308 :     if (context->base.flags & PGJIT_OPT3)
     687             :     {
     688             :         /* TODO: Unscientifically determined threshold */
     689         978 :         LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
     690             :     }
     691             :     else
     692             :     {
     693             :         /* we rely on mem2reg heavily, so emit even in the O0 case */
     694         330 :         LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
     695             :     }
     696             : 
     697        1308 :     LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
     698             : 
     699             :     /*
     700             :      * Do function level optimization. This could be moved to the point where
     701             :      * functions are emitted, to reduce memory usage a bit.
     702             :      */
     703        1308 :     LLVMInitializeFunctionPassManager(llvm_fpm);
     704       31494 :     for (func = LLVMGetFirstFunction(context->module);
     705             :          func != NULL;
     706       30186 :          func = LLVMGetNextFunction(func))
     707       30186 :         LLVMRunFunctionPassManager(llvm_fpm, func);
     708        1308 :     LLVMFinalizeFunctionPassManager(llvm_fpm);
     709        1308 :     LLVMDisposePassManager(llvm_fpm);
     710             : 
     711             :     /*
     712             :      * Perform module level optimization. We do so even in the non-optimized
     713             :      * case, so always-inline functions etc get inlined. It's cheap enough.
     714             :      */
     715        1308 :     llvm_mpm = LLVMCreatePassManager();
     716        1308 :     LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
     717             :                                                     llvm_mpm);
     718             :     /* always use always-inliner pass */
     719        1308 :     if (!(context->base.flags & PGJIT_OPT3))
     720         330 :         LLVMAddAlwaysInlinerPass(llvm_mpm);
     721             :     /* if doing inlining, but no expensive optimization, add inlining pass */
     722        1308 :     if (context->base.flags & PGJIT_INLINE
     723         978 :         && !(context->base.flags & PGJIT_OPT3))
     724           0 :         LLVMAddFunctionInliningPass(llvm_mpm);
     725        1308 :     LLVMRunPassManager(llvm_mpm, context->module);
     726        1308 :     LLVMDisposePassManager(llvm_mpm);
     727             : 
     728        1308 :     LLVMPassManagerBuilderDispose(llvm_pmb);
     729             : #else
     730             :     LLVMPassBuilderOptionsRef options;
     731             :     LLVMErrorRef err;
     732             :     const char *passes;
     733             : 
     734             :     if (context->base.flags & PGJIT_OPT3)
     735             :         passes = "default<O3>";
     736             :     else
     737             :         passes = "default<O0>,mem2reg";
     738             : 
     739             :     options = LLVMCreatePassBuilderOptions();
     740             : 
     741             : #ifdef LLVM_PASS_DEBUG
     742             :     LLVMPassBuilderOptionsSetDebugLogging(options, 1);
     743             : #endif
     744             : 
     745             :     LLVMPassBuilderOptionsSetInlinerThreshold(options, 512);
     746             : 
     747             :     err = LLVMRunPasses(module, passes, NULL, options);
     748             : 
     749             :     if (err)
     750             :         elog(ERROR, "failed to JIT module: %s", llvm_error_message(err));
     751             : 
     752             :     LLVMDisposePassBuilderOptions(options);
     753             : #endif
     754        1308 : }
     755             : 
     756             : /*
     757             :  * Emit code for the currently pending module.
     758             :  */
     759             : static void
     760        1308 : llvm_compile_module(LLVMJitContext *context)
     761             : {
     762             :     LLVMJitHandle *handle;
     763             :     MemoryContext oldcontext;
     764             :     instr_time  starttime;
     765             :     instr_time  endtime;
     766             : #if LLVM_VERSION_MAJOR > 11
     767             :     LLVMOrcLLJITRef compile_orc;
     768             : #else
     769             :     LLVMOrcJITStackRef compile_orc;
     770             : #endif
     771             : 
     772        1308 :     if (context->base.flags & PGJIT_OPT3)
     773         978 :         compile_orc = llvm_opt3_orc;
     774             :     else
     775         330 :         compile_orc = llvm_opt0_orc;
     776             : 
     777             :     /* perform inlining */
     778        1308 :     if (context->base.flags & PGJIT_INLINE)
     779             :     {
     780         978 :         INSTR_TIME_SET_CURRENT(starttime);
     781         978 :         llvm_inline(context->module);
     782         978 :         INSTR_TIME_SET_CURRENT(endtime);
     783         978 :         INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
     784             :                               endtime, starttime);
     785             :     }
     786             : 
     787        1308 :     if (jit_dump_bitcode)
     788             :     {
     789             :         char       *filename;
     790             : 
     791           0 :         filename = psprintf("%d.%zu.bc",
     792             :                             MyProcPid,
     793             :                             context->module_generation);
     794           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     795           0 :         pfree(filename);
     796             :     }
     797             : 
     798             : 
     799             :     /* optimize according to the chosen optimization settings */
     800        1308 :     INSTR_TIME_SET_CURRENT(starttime);
     801        1308 :     llvm_optimize_module(context, context->module);
     802        1308 :     INSTR_TIME_SET_CURRENT(endtime);
     803        1308 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
     804             :                           endtime, starttime);
     805             : 
     806        1308 :     if (jit_dump_bitcode)
     807             :     {
     808             :         char       *filename;
     809             : 
     810           0 :         filename = psprintf("%d.%zu.optimized.bc",
     811             :                             MyProcPid,
     812             :                             context->module_generation);
     813           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     814           0 :         pfree(filename);
     815             :     }
     816             : 
     817             :     handle = (LLVMJitHandle *)
     818        1308 :         MemoryContextAlloc(TopMemoryContext, sizeof(LLVMJitHandle));
     819             : 
     820             :     /*
     821             :      * Emit the code. Note that this can, depending on the optimization
     822             :      * settings, take noticeable resources as code emission executes low-level
     823             :      * instruction combining/selection passes etc. Without optimization a
     824             :      * faster instruction selection mechanism is used.
     825             :      */
     826        1308 :     INSTR_TIME_SET_CURRENT(starttime);
     827             : #if LLVM_VERSION_MAJOR > 11
     828             :     {
     829             :         LLVMOrcThreadSafeModuleRef ts_module;
     830             :         LLVMErrorRef error;
     831             :         LLVMOrcJITDylibRef jd = LLVMOrcLLJITGetMainJITDylib(compile_orc);
     832             : 
     833             :         ts_module = LLVMOrcCreateNewThreadSafeModule(context->module, llvm_ts_context);
     834             : 
     835             :         handle->lljit = compile_orc;
     836             :         handle->resource_tracker = LLVMOrcJITDylibCreateResourceTracker(jd);
     837             : 
     838             :         /*
     839             :          * NB: This doesn't actually emit code. That happens lazily the first
     840             :          * time a symbol defined in the module is requested. Due to that
     841             :          * llvm_get_function() also accounts for emission time.
     842             :          */
     843             : 
     844             :         context->module = NULL; /* will be owned by LLJIT */
     845             :         error = LLVMOrcLLJITAddLLVMIRModuleWithRT(compile_orc,
     846             :                                                   handle->resource_tracker,
     847             :                                                   ts_module);
     848             : 
     849             :         if (error)
     850             :             elog(ERROR, "failed to JIT module: %s",
     851             :                  llvm_error_message(error));
     852             : 
     853             :         /* LLVMOrcLLJITAddLLVMIRModuleWithRT takes ownership of the module */
     854             :     }
     855             : #elif LLVM_VERSION_MAJOR > 6
     856             :     {
     857        1308 :         handle->stack = compile_orc;
     858        1308 :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, context->module,
     859             :                                         llvm_resolve_symbol, NULL))
     860           0 :             elog(ERROR, "failed to JIT module");
     861             : 
     862             :         /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
     863             :     }
     864             : #elif LLVM_VERSION_MAJOR > 4
     865             :     {
     866             :         LLVMSharedModuleRef smod;
     867             : 
     868             :         smod = LLVMOrcMakeSharedModule(context->module);
     869             :         handle->stack = compile_orc;
     870             :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &handle->orc_handle, smod,
     871             :                                         llvm_resolve_symbol, NULL))
     872             :             elog(ERROR, "failed to JIT module");
     873             : 
     874             :         LLVMOrcDisposeSharedModuleRef(smod);
     875             :     }
     876             : #else                           /* LLVM 4.0 and 3.9 */
     877             :     {
     878             :         handle->stack = compile_orc;
     879             :         handle->orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
     880             :                                                          llvm_resolve_symbol, NULL);
     881             : 
     882             :         LLVMDisposeModule(context->module);
     883             :     }
     884             : #endif
     885             : 
     886        1308 :     INSTR_TIME_SET_CURRENT(endtime);
     887        1308 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     888             :                           endtime, starttime);
     889             : 
     890        1308 :     context->module = NULL;
     891        1308 :     context->compiled = true;
     892             : 
     893             :     /* remember emitted code for cleanup and lookups */
     894        1308 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     895        1308 :     context->handles = lappend(context->handles, handle);
     896        1308 :     MemoryContextSwitchTo(oldcontext);
     897             : 
     898        1308 :     ereport(DEBUG1,
     899             :             (errmsg_internal("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
     900             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
     901             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
     902             :                              INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
     903             :              errhidestmt(true),
     904             :              errhidecontext(true)));
     905        1308 : }
     906             : 
     907             : /*
     908             :  * Per session initialization.
     909             :  */
     910             : static void
     911        1672 : llvm_session_initialize(void)
     912             : {
     913             :     MemoryContext oldcontext;
     914        1672 :     char       *error = NULL;
     915        1672 :     char       *cpu = NULL;
     916        1672 :     char       *features = NULL;
     917             :     LLVMTargetMachineRef opt0_tm;
     918             :     LLVMTargetMachineRef opt3_tm;
     919             : 
     920        1672 :     if (llvm_session_initialized)
     921         902 :         return;
     922             : 
     923         770 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     924             : 
     925         770 :     LLVMInitializeNativeTarget();
     926         770 :     LLVMInitializeNativeAsmPrinter();
     927         770 :     LLVMInitializeNativeAsmParser();
     928             : 
     929         770 :     if (llvm_context == NULL)
     930             :     {
     931         770 :         llvm_context = LLVMContextCreate();
     932             : 
     933         770 :         llvm_jit_context_in_use_count = 0;
     934         770 :         llvm_llvm_context_reuse_count = 0;
     935             :     }
     936             : 
     937             :     /*
     938             :      * When targeting LLVM 15, turn off opaque pointers for the context we
     939             :      * build our code in.  We don't need to do so for other contexts (e.g.
     940             :      * llvm_ts_context).  Once the IR is generated, it carries the necessary
     941             :      * information.
     942             :      *
     943             :      * For 16 and above, opaque pointers must be used, and we have special
     944             :      * code for that.
     945             :      */
     946             : #if LLVM_VERSION_MAJOR == 15
     947             :     LLVMContextSetOpaquePointers(LLVMGetGlobalContext(), false);
     948             : #endif
     949             : 
     950             :     /*
     951             :      * Synchronize types early, as that also includes inferring the target
     952             :      * triple.
     953             :      */
     954         770 :     llvm_create_types();
     955             : 
     956             :     /*
     957             :      * Extract target information from loaded module.
     958             :      */
     959         770 :     llvm_set_target();
     960             : 
     961         770 :     if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
     962             :     {
     963           0 :         elog(FATAL, "failed to query triple %s", error);
     964             :     }
     965             : 
     966             :     /*
     967             :      * We want the generated code to use all available features. Therefore
     968             :      * grab the host CPU string and detect features of the current CPU. The
     969             :      * latter is needed because some CPU architectures default to enabling
     970             :      * features not all CPUs have (weird, huh).
     971             :      */
     972         770 :     cpu = LLVMGetHostCPUName();
     973         770 :     features = LLVMGetHostCPUFeatures();
     974         770 :     elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
     975             :          cpu, features);
     976             : 
     977             :     opt0_tm =
     978         770 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     979             :                                 LLVMCodeGenLevelNone,
     980             :                                 LLVMRelocDefault,
     981             :                                 LLVMCodeModelJITDefault);
     982             :     opt3_tm =
     983         770 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     984             :                                 LLVMCodeGenLevelAggressive,
     985             :                                 LLVMRelocDefault,
     986             :                                 LLVMCodeModelJITDefault);
     987             : 
     988         770 :     LLVMDisposeMessage(cpu);
     989         770 :     cpu = NULL;
     990         770 :     LLVMDisposeMessage(features);
     991         770 :     features = NULL;
     992             : 
     993             :     /* force symbols in main binary to be loaded */
     994         770 :     LLVMLoadLibraryPermanently(NULL);
     995             : 
     996             : #if LLVM_VERSION_MAJOR > 11
     997             :     {
     998             :         llvm_ts_context = LLVMOrcCreateNewThreadSafeContext();
     999             : 
    1000             :         llvm_opt0_orc = llvm_create_jit_instance(opt0_tm);
    1001             :         opt0_tm = 0;
    1002             : 
    1003             :         llvm_opt3_orc = llvm_create_jit_instance(opt3_tm);
    1004             :         opt3_tm = 0;
    1005             :     }
    1006             : #else                           /* LLVM_VERSION_MAJOR > 11 */
    1007             :     {
    1008         770 :         llvm_opt0_orc = LLVMOrcCreateInstance(opt0_tm);
    1009         770 :         llvm_opt3_orc = LLVMOrcCreateInstance(opt3_tm);
    1010             : 
    1011             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
    1012         770 :         if (jit_debugging_support)
    1013             :         {
    1014           0 :             LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
    1015             : 
    1016           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
    1017           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
    1018             :         }
    1019             : #endif
    1020             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
    1021         770 :         if (jit_profiling_support)
    1022             :         {
    1023           0 :             LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
    1024             : 
    1025           0 :             LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
    1026           0 :             LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
    1027             :         }
    1028             : #endif
    1029             :     }
    1030             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1031             : 
    1032         770 :     on_proc_exit(llvm_shutdown, 0);
    1033             : 
    1034         770 :     llvm_session_initialized = true;
    1035             : 
    1036         770 :     MemoryContextSwitchTo(oldcontext);
    1037             : }
    1038             : 
    1039             : static void
    1040         770 : llvm_shutdown(int code, Datum arg)
    1041             : {
    1042             :     /*
    1043             :      * If llvm_shutdown() is reached while in a fatal-on-oom section an error
    1044             :      * has occurred in the middle of LLVM code. It is not safe to call back
    1045             :      * into LLVM (which is why a FATAL error was thrown).
    1046             :      *
    1047             :      * We do need to shutdown LLVM in other shutdown cases, otherwise e.g.
    1048             :      * profiling data won't be written out.
    1049             :      */
    1050         770 :     if (llvm_in_fatal_on_oom())
    1051             :     {
    1052             :         Assert(proc_exit_inprogress);
    1053           0 :         return;
    1054             :     }
    1055             : 
    1056         770 :     if (llvm_jit_context_in_use_count != 0)
    1057           0 :         elog(PANIC, "LLVMJitContext in use count not 0 at exit (is %zu)",
    1058             :              llvm_jit_context_in_use_count);
    1059             : 
    1060             : #if LLVM_VERSION_MAJOR > 11
    1061             :     {
    1062             :         if (llvm_opt3_orc)
    1063             :         {
    1064             :             LLVMOrcDisposeLLJIT(llvm_opt3_orc);
    1065             :             llvm_opt3_orc = NULL;
    1066             :         }
    1067             :         if (llvm_opt0_orc)
    1068             :         {
    1069             :             LLVMOrcDisposeLLJIT(llvm_opt0_orc);
    1070             :             llvm_opt0_orc = NULL;
    1071             :         }
    1072             :         if (llvm_ts_context)
    1073             :         {
    1074             :             LLVMOrcDisposeThreadSafeContext(llvm_ts_context);
    1075             :             llvm_ts_context = NULL;
    1076             :         }
    1077             :     }
    1078             : #else                           /* LLVM_VERSION_MAJOR > 11 */
    1079             :     {
    1080             :         /* unregister profiling support, needs to be flushed to be useful */
    1081             : 
    1082         770 :         if (llvm_opt3_orc)
    1083             :         {
    1084             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
    1085             :             if (jit_profiling_support)
    1086             :                 LLVMOrcUnregisterPerf(llvm_opt3_orc);
    1087             : #endif
    1088         770 :             LLVMOrcDisposeInstance(llvm_opt3_orc);
    1089         770 :             llvm_opt3_orc = NULL;
    1090             :         }
    1091             : 
    1092         770 :         if (llvm_opt0_orc)
    1093             :         {
    1094             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
    1095             :             if (jit_profiling_support)
    1096             :                 LLVMOrcUnregisterPerf(llvm_opt0_orc);
    1097             : #endif
    1098         770 :             LLVMOrcDisposeInstance(llvm_opt0_orc);
    1099         770 :             llvm_opt0_orc = NULL;
    1100             :         }
    1101             :     }
    1102             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1103             : }
    1104             : 
    1105             : /* helper for llvm_create_types, returning a function's return type */
    1106             : static LLVMTypeRef
    1107         770 : load_return_type(LLVMModuleRef mod, const char *name)
    1108             : {
    1109             :     LLVMValueRef value;
    1110             :     LLVMTypeRef typ;
    1111             : 
    1112             :     /* this'll return a *pointer* to the function */
    1113         770 :     value = LLVMGetNamedFunction(mod, name);
    1114         770 :     if (!value)
    1115           0 :         elog(ERROR, "function %s is unknown", name);
    1116             : 
    1117         770 :     typ = LLVMGetFunctionReturnType(value); /* in llvmjit_wrap.cpp */
    1118             : 
    1119         770 :     return typ;
    1120             : }
    1121             : 
    1122             : /*
    1123             :  * Load triple & layout from clang emitted file so we're guaranteed to be
    1124             :  * compatible.
    1125             :  */
    1126             : static void
    1127         770 : llvm_set_target(void)
    1128             : {
    1129         770 :     if (!llvm_types_module)
    1130           0 :         elog(ERROR, "failed to extract target information, llvmjit_types.c not loaded");
    1131             : 
    1132         770 :     if (llvm_triple == NULL)
    1133         770 :         llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
    1134             : 
    1135         770 :     if (llvm_layout == NULL)
    1136         770 :         llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
    1137         770 : }
    1138             : 
    1139             : /*
    1140             :  * Load required information, types, function signatures from llvmjit_types.c
    1141             :  * and make them available in global variables.
    1142             :  *
    1143             :  * Those global variables are then used while emitting code.
    1144             :  */
    1145             : static void
    1146         770 : llvm_create_types(void)
    1147             : {
    1148             :     char        path[MAXPGPATH];
    1149             :     LLVMMemoryBufferRef buf;
    1150             :     char       *msg;
    1151             : 
    1152         770 :     snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
    1153             : 
    1154             :     /* open file */
    1155         770 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
    1156             :     {
    1157           0 :         elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
    1158             :              path, msg);
    1159             :     }
    1160             : 
    1161             :     /* eagerly load contents, going to need it all */
    1162         770 :     if (LLVMParseBitcodeInContext2(llvm_context, buf, &llvm_types_module))
    1163             :     {
    1164           0 :         elog(ERROR, "LLVMParseBitcodeInContext2 of %s failed", path);
    1165             :     }
    1166         770 :     LLVMDisposeMemoryBuffer(buf);
    1167             : 
    1168         770 :     TypeSizeT = llvm_pg_var_type("TypeSizeT");
    1169         770 :     TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
    1170         770 :     TypeStorageBool = llvm_pg_var_type("TypeStorageBool");
    1171         770 :     TypePGFunction = llvm_pg_var_type("TypePGFunction");
    1172         770 :     StructNullableDatum = llvm_pg_var_type("StructNullableDatum");
    1173         770 :     StructExprContext = llvm_pg_var_type("StructExprContext");
    1174         770 :     StructExprEvalStep = llvm_pg_var_type("StructExprEvalStep");
    1175         770 :     StructExprState = llvm_pg_var_type("StructExprState");
    1176         770 :     StructFunctionCallInfoData = llvm_pg_var_type("StructFunctionCallInfoData");
    1177         770 :     StructMemoryContextData = llvm_pg_var_type("StructMemoryContextData");
    1178         770 :     StructTupleTableSlot = llvm_pg_var_type("StructTupleTableSlot");
    1179         770 :     StructHeapTupleTableSlot = llvm_pg_var_type("StructHeapTupleTableSlot");
    1180         770 :     StructMinimalTupleTableSlot = llvm_pg_var_type("StructMinimalTupleTableSlot");
    1181         770 :     StructHeapTupleData = llvm_pg_var_type("StructHeapTupleData");
    1182         770 :     StructHeapTupleHeaderData = llvm_pg_var_type("StructHeapTupleHeaderData");
    1183         770 :     StructTupleDescData = llvm_pg_var_type("StructTupleDescData");
    1184         770 :     StructAggState = llvm_pg_var_type("StructAggState");
    1185         770 :     StructAggStatePerGroupData = llvm_pg_var_type("StructAggStatePerGroupData");
    1186         770 :     StructAggStatePerTransData = llvm_pg_var_type("StructAggStatePerTransData");
    1187         770 :     StructPlanState = llvm_pg_var_type("StructPlanState");
    1188         770 :     StructMinimalTupleData = llvm_pg_var_type("StructMinimalTupleData");
    1189             : 
    1190         770 :     AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
    1191         770 :     ExecEvalSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalSubroutineTemplate");
    1192         770 :     ExecEvalBoolSubroutineTemplate = LLVMGetNamedFunction(llvm_types_module, "ExecEvalBoolSubroutineTemplate");
    1193         770 : }
    1194             : 
    1195             : /*
    1196             :  * Split a symbol into module / function parts.  If the function is in the
    1197             :  * main binary (or an external library) *modname will be NULL.
    1198             :  */
    1199             : void
    1200       33254 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
    1201             : {
    1202       33254 :     *modname = NULL;
    1203       33254 :     *funcname = NULL;
    1204             : 
    1205             :     /*
    1206             :      * Module function names are pgextern.$module.$funcname
    1207             :      */
    1208       33254 :     if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
    1209             :     {
    1210             :         /*
    1211             :          * Symbol names cannot contain a ., therefore we can split based on
    1212             :          * first and last occurrence of one.
    1213             :          */
    1214         110 :         *funcname = rindex(name, '.');
    1215         110 :         (*funcname)++;          /* jump over . */
    1216             : 
    1217         220 :         *modname = pnstrdup(name + strlen("pgextern."),
    1218         110 :                             *funcname - name - strlen("pgextern.") - 1);
    1219             :         Assert(funcname);
    1220             : 
    1221         110 :         *funcname = pstrdup(*funcname);
    1222             :     }
    1223             :     else
    1224             :     {
    1225       33144 :         *modname = NULL;
    1226       33144 :         *funcname = pstrdup(name);
    1227             :     }
    1228       33254 : }
    1229             : 
    1230             : /*
    1231             :  * Attempt to resolve symbol, so LLVM can emit a reference to it.
    1232             :  */
    1233             : static uint64_t
    1234        5690 : llvm_resolve_symbol(const char *symname, void *ctx)
    1235             : {
    1236             :     uintptr_t   addr;
    1237             :     char       *funcname;
    1238             :     char       *modname;
    1239             : 
    1240             :     /*
    1241             :      * macOS prefixes all object level symbols with an underscore. But neither
    1242             :      * dlsym() nor PG's inliner expect that. So undo.
    1243             :      */
    1244             : #if defined(__darwin__)
    1245             :     if (symname[0] != '_')
    1246             :         elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
    1247             :     symname++;
    1248             : #endif
    1249             : 
    1250        5690 :     llvm_split_symbol_name(symname, &modname, &funcname);
    1251             : 
    1252             :     /* functions that aren't resolved to names shouldn't ever get here */
    1253             :     Assert(funcname);
    1254             : 
    1255        5690 :     if (modname)
    1256          38 :         addr = (uintptr_t) load_external_function(modname, funcname,
    1257             :                                                   true, NULL);
    1258             :     else
    1259        5652 :         addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
    1260             : 
    1261        5690 :     pfree(funcname);
    1262        5690 :     if (modname)
    1263          38 :         pfree(modname);
    1264             : 
    1265             :     /* let LLVM will error out - should never happen */
    1266        5690 :     if (!addr)
    1267           0 :         elog(WARNING, "failed to resolve name %s", symname);
    1268             : 
    1269        5690 :     return (uint64_t) addr;
    1270             : }
    1271             : 
    1272             : #if LLVM_VERSION_MAJOR > 11
    1273             : 
    1274             : static LLVMErrorRef
    1275             : llvm_resolve_symbols(LLVMOrcDefinitionGeneratorRef GeneratorObj, void *Ctx,
    1276             :                      LLVMOrcLookupStateRef *LookupState, LLVMOrcLookupKind Kind,
    1277             :                      LLVMOrcJITDylibRef JD, LLVMOrcJITDylibLookupFlags JDLookupFlags,
    1278             :                      LLVMOrcCLookupSet LookupSet, size_t LookupSetSize)
    1279             : {
    1280             : #if LLVM_VERSION_MAJOR > 14
    1281             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMOrcCSymbolMapPair) * LookupSetSize);
    1282             : #else
    1283             :     LLVMOrcCSymbolMapPairs symbols = palloc0(sizeof(LLVMJITCSymbolMapPair) * LookupSetSize);
    1284             : #endif
    1285             :     LLVMErrorRef error;
    1286             :     LLVMOrcMaterializationUnitRef mu;
    1287             : 
    1288             :     for (int i = 0; i < LookupSetSize; i++)
    1289             :     {
    1290             :         const char *name = LLVMOrcSymbolStringPoolEntryStr(LookupSet[i].Name);
    1291             : 
    1292             : #if LLVM_VERSION_MAJOR > 12
    1293             :         LLVMOrcRetainSymbolStringPoolEntry(LookupSet[i].Name);
    1294             : #endif
    1295             :         symbols[i].Name = LookupSet[i].Name;
    1296             :         symbols[i].Sym.Address = llvm_resolve_symbol(name, NULL);
    1297             :         symbols[i].Sym.Flags.GenericFlags = LLVMJITSymbolGenericFlagsExported;
    1298             :     }
    1299             : 
    1300             :     mu = LLVMOrcAbsoluteSymbols(symbols, LookupSetSize);
    1301             :     error = LLVMOrcJITDylibDefine(JD, mu);
    1302             :     if (error != LLVMErrorSuccess)
    1303             :         LLVMOrcDisposeMaterializationUnit(mu);
    1304             : 
    1305             :     pfree(symbols);
    1306             : 
    1307             :     return error;
    1308             : }
    1309             : 
    1310             : /*
    1311             :  * We cannot throw errors through LLVM (without causing a FATAL at least), so
    1312             :  * just use WARNING here. That's OK anyway, as the error is also reported at
    1313             :  * the top level action (with less detail) and there might be multiple
    1314             :  * invocations of errors with details.
    1315             :  *
    1316             :  * This doesn't really happen during normal operation, but in cases like
    1317             :  * symbol resolution breakage. So just using elog(WARNING) is fine.
    1318             :  */
    1319             : static void
    1320             : llvm_log_jit_error(void *ctx, LLVMErrorRef error)
    1321             : {
    1322             :     elog(WARNING, "error during JITing: %s",
    1323             :          llvm_error_message(error));
    1324             : }
    1325             : 
    1326             : /*
    1327             :  * Create our own object layer, so we can add event listeners.
    1328             :  */
    1329             : static LLVMOrcObjectLayerRef
    1330             : llvm_create_object_layer(void *Ctx, LLVMOrcExecutionSessionRef ES, const char *Triple)
    1331             : {
    1332             :     LLVMOrcObjectLayerRef objlayer =
    1333             :         LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager(ES);
    1334             : 
    1335             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
    1336             :     if (jit_debugging_support)
    1337             :     {
    1338             :         LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
    1339             : 
    1340             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1341             :     }
    1342             : #endif
    1343             : 
    1344             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
    1345             :     if (jit_profiling_support)
    1346             :     {
    1347             :         LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
    1348             : 
    1349             :         LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener(objlayer, l);
    1350             :     }
    1351             : #endif
    1352             : 
    1353             :     return objlayer;
    1354             : }
    1355             : 
    1356             : /*
    1357             :  * Create LLJIT instance, using the passed in target machine. Note that the
    1358             :  * target machine afterwards is owned by the LLJIT instance.
    1359             :  */
    1360             : static LLVMOrcLLJITRef
    1361             : llvm_create_jit_instance(LLVMTargetMachineRef tm)
    1362             : {
    1363             :     LLVMOrcLLJITRef lljit;
    1364             :     LLVMOrcJITTargetMachineBuilderRef tm_builder;
    1365             :     LLVMOrcLLJITBuilderRef lljit_builder;
    1366             :     LLVMErrorRef error;
    1367             :     LLVMOrcDefinitionGeneratorRef main_gen;
    1368             :     LLVMOrcDefinitionGeneratorRef ref_gen;
    1369             : 
    1370             :     lljit_builder = LLVMOrcCreateLLJITBuilder();
    1371             :     tm_builder = LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(tm);
    1372             :     LLVMOrcLLJITBuilderSetJITTargetMachineBuilder(lljit_builder, tm_builder);
    1373             : 
    1374             :     LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator(lljit_builder,
    1375             :                                                     llvm_create_object_layer,
    1376             :                                                     NULL);
    1377             : 
    1378             :     error = LLVMOrcCreateLLJIT(&lljit, lljit_builder);
    1379             :     if (error)
    1380             :         elog(ERROR, "failed to create lljit instance: %s",
    1381             :              llvm_error_message(error));
    1382             : 
    1383             :     LLVMOrcExecutionSessionSetErrorReporter(LLVMOrcLLJITGetExecutionSession(lljit),
    1384             :                                             llvm_log_jit_error, NULL);
    1385             : 
    1386             :     /*
    1387             :      * Symbol resolution support for symbols in the postgres binary /
    1388             :      * libraries already loaded.
    1389             :      */
    1390             :     error = LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess(&main_gen,
    1391             :                                                                  LLVMOrcLLJITGetGlobalPrefix(lljit),
    1392             :                                                                  0, NULL);
    1393             :     if (error)
    1394             :         elog(ERROR, "failed to create generator: %s",
    1395             :              llvm_error_message(error));
    1396             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), main_gen);
    1397             : 
    1398             :     /*
    1399             :      * Symbol resolution support for "special" functions, e.g. a call into an
    1400             :      * SQL callable function.
    1401             :      */
    1402             : #if LLVM_VERSION_MAJOR > 14
    1403             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL, NULL);
    1404             : #else
    1405             :     ref_gen = LLVMOrcCreateCustomCAPIDefinitionGenerator(llvm_resolve_symbols, NULL);
    1406             : #endif
    1407             :     LLVMOrcJITDylibAddGenerator(LLVMOrcLLJITGetMainJITDylib(lljit), ref_gen);
    1408             : 
    1409             :     return lljit;
    1410             : }
    1411             : 
    1412             : static char *
    1413             : llvm_error_message(LLVMErrorRef error)
    1414             : {
    1415             :     char       *orig = LLVMGetErrorMessage(error);
    1416             :     char       *msg = pstrdup(orig);
    1417             : 
    1418             :     LLVMDisposeErrorMessage(orig);
    1419             : 
    1420             :     return msg;
    1421             : }
    1422             : 
    1423             : #endif                          /* LLVM_VERSION_MAJOR > 11 */
    1424             : 
    1425             : /*
    1426             :  * ResourceOwner callbacks
    1427             :  */
    1428             : static void
    1429          16 : ResOwnerReleaseJitContext(Datum res)
    1430             : {
    1431          16 :     JitContext *context = (JitContext *) DatumGetPointer(res);
    1432             : 
    1433          16 :     context->resowner = NULL;
    1434          16 :     jit_release_context(context);
    1435          16 : }

Generated by: LCOV version 1.14