LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit.c (source / functions) Hit Total Coverage
Test: PostgreSQL 13beta1 Lines: 238 266 89.5 %
Date: 2020-05-25 06:06:29 Functions: 19 19 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit.c
       4             :  *    Core part of the LLVM JIT provider.
       5             :  *
       6             :  * Copyright (c) 2016-2020, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/jit/llvm/llvmjit.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres.h"
      15             : 
      16             : #include <llvm-c/Analysis.h>
      17             : #include <llvm-c/BitReader.h>
      18             : #include <llvm-c/BitWriter.h>
      19             : #include <llvm-c/Core.h>
      20             : #include <llvm-c/ExecutionEngine.h>
      21             : #include <llvm-c/OrcBindings.h>
      22             : #include <llvm-c/Support.h>
      23             : #include <llvm-c/Target.h>
      24             : #include <llvm-c/Transforms/IPO.h>
      25             : #include <llvm-c/Transforms/PassManagerBuilder.h>
      26             : #include <llvm-c/Transforms/Scalar.h>
      27             : #if LLVM_VERSION_MAJOR > 6
      28             : #include <llvm-c/Transforms/Utils.h>
      29             : #endif
      30             : 
      31             : #include "jit/llvmjit.h"
      32             : #include "jit/llvmjit_emit.h"
      33             : #include "miscadmin.h"
      34             : #include "portability/instr_time.h"
      35             : #include "storage/ipc.h"
      36             : #include "utils/memutils.h"
      37             : #include "utils/resowner_private.h"
      38             : 
      39             : /* Handle of a module emitted via ORC JIT */
      40             : typedef struct LLVMJitHandle
      41             : {
      42             :     LLVMOrcJITStackRef stack;
      43             :     LLVMOrcModuleHandle orc_handle;
      44             : } LLVMJitHandle;
      45             : 
      46             : 
      47             : /* types & functions commonly needed for JITing */
      48             : LLVMTypeRef TypeSizeT;
      49             : LLVMTypeRef TypeParamBool;
      50             : LLVMTypeRef TypeStorageBool;
      51             : LLVMTypeRef TypePGFunction;
      52             : LLVMTypeRef StructNullableDatum;
      53             : LLVMTypeRef StructHeapTupleFieldsField3;
      54             : LLVMTypeRef StructHeapTupleFields;
      55             : LLVMTypeRef StructHeapTupleHeaderData;
      56             : LLVMTypeRef StructHeapTupleDataChoice;
      57             : LLVMTypeRef StructHeapTupleData;
      58             : LLVMTypeRef StructMinimalTupleData;
      59             : LLVMTypeRef StructItemPointerData;
      60             : LLVMTypeRef StructBlockId;
      61             : LLVMTypeRef StructFormPgAttribute;
      62             : LLVMTypeRef StructTupleConstr;
      63             : LLVMTypeRef StructTupleDescData;
      64             : LLVMTypeRef StructTupleTableSlot;
      65             : LLVMTypeRef StructHeapTupleTableSlot;
      66             : LLVMTypeRef StructMinimalTupleTableSlot;
      67             : LLVMTypeRef StructMemoryContextData;
      68             : LLVMTypeRef StructPGFinfoRecord;
      69             : LLVMTypeRef StructFmgrInfo;
      70             : LLVMTypeRef StructFunctionCallInfoData;
      71             : LLVMTypeRef StructExprContext;
      72             : LLVMTypeRef StructExprEvalStep;
      73             : LLVMTypeRef StructExprState;
      74             : LLVMTypeRef StructAggState;
      75             : LLVMTypeRef StructAggStatePerGroupData;
      76             : LLVMTypeRef StructAggStatePerTransData;
      77             : 
      78             : LLVMValueRef AttributeTemplate;
      79             : 
      80             : LLVMModuleRef llvm_types_module = NULL;
      81             : 
      82             : static bool llvm_session_initialized = false;
      83             : static size_t llvm_generation = 0;
      84             : static const char *llvm_triple = NULL;
      85             : static const char *llvm_layout = NULL;
      86             : 
      87             : 
      88             : static LLVMTargetMachineRef llvm_opt0_targetmachine;
      89             : static LLVMTargetMachineRef llvm_opt3_targetmachine;
      90             : 
      91             : static LLVMTargetRef llvm_targetref;
      92             : static LLVMOrcJITStackRef llvm_opt0_orc;
      93             : static LLVMOrcJITStackRef llvm_opt3_orc;
      94             : 
      95             : 
      96             : static void llvm_release_context(JitContext *context);
      97             : static void llvm_session_initialize(void);
      98             : static void llvm_shutdown(int code, Datum arg);
      99             : static void llvm_compile_module(LLVMJitContext *context);
     100             : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
     101             : 
     102             : static void llvm_create_types(void);
     103             : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
     104             : 
     105             : 
     106        1188 : PG_MODULE_MAGIC;
     107             : 
     108             : 
     109             : /*
     110             :  * Initialize LLVM JIT provider.
     111             :  */
     112             : void
     113         500 : _PG_jit_provider_init(JitProviderCallbacks *cb)
     114             : {
     115         500 :     cb->reset_after_error = llvm_reset_after_error;
     116         500 :     cb->release_context = llvm_release_context;
     117         500 :     cb->compile_expr = llvm_compile_expr;
     118         500 : }
     119             : 
     120             : /*
     121             :  * Create a context for JITing work.
     122             :  *
     123             :  * The context, including subsidiary resources, will be cleaned up either when
     124             :  * the context is explicitly released, or when the lifetime of
     125             :  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
     126             :  */
     127             : LLVMJitContext *
     128         938 : llvm_create_context(int jitFlags)
     129             : {
     130             :     LLVMJitContext *context;
     131             : 
     132         938 :     llvm_assert_in_fatal_section();
     133             : 
     134         938 :     llvm_session_initialize();
     135             : 
     136         938 :     ResourceOwnerEnlargeJIT(CurrentResourceOwner);
     137             : 
     138         938 :     context = MemoryContextAllocZero(TopMemoryContext,
     139             :                                      sizeof(LLVMJitContext));
     140         938 :     context->base.flags = jitFlags;
     141             : 
     142             :     /* ensure cleanup */
     143         938 :     context->base.resowner = CurrentResourceOwner;
     144         938 :     ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
     145             : 
     146         938 :     return context;
     147             : }
     148             : 
     149             : /*
     150             :  * Release resources required by one llvm context.
     151             :  */
     152             : static void
     153         938 : llvm_release_context(JitContext *context)
     154             : {
     155         938 :     LLVMJitContext *llvm_context = (LLVMJitContext *) context;
     156             : 
     157         938 :     llvm_enter_fatal_on_oom();
     158             : 
     159             :     /*
     160             :      * When this backend is exiting, don't clean up LLVM. As an error might
     161             :      * have occurred from within LLVM, we do not want to risk reentering. All
     162             :      * resource cleanup is going to happen through process exit.
     163             :      */
     164         938 :     if (!proc_exit_inprogress)
     165             :     {
     166         938 :         if (llvm_context->module)
     167             :         {
     168         238 :             LLVMDisposeModule(llvm_context->module);
     169         238 :             llvm_context->module = NULL;
     170             :         }
     171             : 
     172        1678 :         while (llvm_context->handles != NIL)
     173             :         {
     174             :             LLVMJitHandle *jit_handle;
     175             : 
     176         740 :             jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
     177         740 :             llvm_context->handles = list_delete_first(llvm_context->handles);
     178             : 
     179         740 :             LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
     180         740 :             pfree(jit_handle);
     181             :         }
     182             :     }
     183         938 : }
     184             : 
     185             : /*
     186             :  * Return module which may be modified, e.g. by creating new functions.
     187             :  */
     188             : LLVMModuleRef
     189        8238 : llvm_mutable_module(LLVMJitContext *context)
     190             : {
     191        8238 :     llvm_assert_in_fatal_section();
     192             : 
     193             :     /*
     194             :      * If there's no in-progress module, create a new one.
     195             :      */
     196        8238 :     if (!context->module)
     197             :     {
     198         978 :         context->compiled = false;
     199         978 :         context->module_generation = llvm_generation++;
     200         978 :         context->module = LLVMModuleCreateWithName("pg");
     201         978 :         LLVMSetTarget(context->module, llvm_triple);
     202         978 :         LLVMSetDataLayout(context->module, llvm_layout);
     203             :     }
     204             : 
     205        8238 :     return context->module;
     206             : }
     207             : 
     208             : /*
     209             :  * Expand function name to be non-conflicting. This should be used by code
     210             :  * generating code, when adding new externally visible function definitions to
     211             :  * a Module.
     212             :  */
     213             : char *
     214        8238 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
     215             : {
     216             :     Assert(context->module != NULL);
     217             : 
     218        8238 :     context->base.instr.created_functions++;
     219             : 
     220             :     /*
     221             :      * Previously we used dots to separate, but turns out some tools, e.g.
     222             :      * GDB, don't like that and truncate name.
     223             :      */
     224       16476 :     return psprintf("%s_%zu_%d",
     225             :                     basename,
     226             :                     context->module_generation,
     227        8238 :                     context->counter++);
     228             : }
     229             : 
     230             : /*
     231             :  * Return pointer to function funcname, which has to exist. If there's pending
     232             :  * code to be optimized and emitted, do so first.
     233             :  */
     234             : void *
     235        3108 : llvm_get_function(LLVMJitContext *context, const char *funcname)
     236             : {
     237        3108 :     LLVMOrcTargetAddress addr = 0;
     238             : #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
     239             :     ListCell   *lc;
     240             : #endif
     241             : 
     242        3108 :     llvm_assert_in_fatal_section();
     243             : 
     244             :     /*
     245             :      * If there is a pending / not emitted module, compile and emit now.
     246             :      * Otherwise we might not find the [correct] function.
     247             :      */
     248        3108 :     if (!context->compiled)
     249             :     {
     250         740 :         llvm_compile_module(context);
     251             :     }
     252             : 
     253             :     /*
     254             :      * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
     255             :      * to mangle here.
     256             :      */
     257             : 
     258             : #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
     259             :     foreach(lc, context->handles)
     260             :     {
     261             :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     262             : 
     263             :         addr = 0;
     264             :         if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
     265             :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     266             :         if (addr)
     267             :             return (void *) (uintptr_t) addr;
     268             :     }
     269             : 
     270             : #else
     271             : 
     272             : #if LLVM_VERSION_MAJOR < 5
     273             :     if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
     274             :         return (void *) (uintptr_t) addr;
     275             :     if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
     276             :         return (void *) (uintptr_t) addr;
     277             : #else
     278        3108 :     if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
     279           0 :         elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     280        3108 :     if (addr)
     281         572 :         return (void *) (uintptr_t) addr;
     282        2536 :     if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
     283           0 :         elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     284        2536 :     if (addr)
     285        2536 :         return (void *) (uintptr_t) addr;
     286             : #endif                          /* LLVM_VERSION_MAJOR */
     287             : 
     288             : #endif                          /* HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN */
     289             : 
     290           0 :     elog(ERROR, "failed to JIT: %s", funcname);
     291             : 
     292             :     return NULL;
     293             : }
     294             : 
     295             : /*
     296             :  * Return declaration for a function referenced in llvmjit_types.c, adding it
     297             :  * to the module if necessary.
     298             :  *
     299             :  * This is used to make functions discovered via llvm_create_types() known to
     300             :  * the module that's currently being worked on.
     301             :  */
     302             : LLVMValueRef
     303        6882 : llvm_pg_func(LLVMModuleRef mod, const char *funcname)
     304             : {
     305             :     LLVMValueRef v_srcfn;
     306             :     LLVMValueRef v_fn;
     307             : 
     308             :     /* don't repeatedly add function */
     309        6882 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     310        6882 :     if (v_fn)
     311        5126 :         return v_fn;
     312             : 
     313        1756 :     v_srcfn = LLVMGetNamedFunction(llvm_types_module, funcname);
     314             : 
     315        1756 :     if (!v_srcfn)
     316           0 :         elog(ERROR, "function %s not in llvmjit_types.c", funcname);
     317             : 
     318        1756 :     v_fn = LLVMAddFunction(mod,
     319             :                            funcname,
     320             :                            LLVMGetElementType(LLVMTypeOf(v_srcfn)));
     321        1756 :     llvm_copy_attributes(v_srcfn, v_fn);
     322             : 
     323        1756 :     return v_fn;
     324             : }
     325             : 
     326             : /*
     327             :  * Copy attributes from one function to another.
     328             :  */
     329             : void
     330        9994 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
     331             : {
     332             :     int         num_attributes;
     333             :     int         attno;
     334             :     LLVMAttributeRef *attrs;
     335             : 
     336        9994 :     num_attributes =
     337        9994 :         LLVMGetAttributeCountAtIndex(v_from, LLVMAttributeFunctionIndex);
     338             : 
     339        9994 :     attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
     340        9994 :     LLVMGetAttributesAtIndex(v_from, LLVMAttributeFunctionIndex, attrs);
     341             : 
     342      172868 :     for (attno = 0; attno < num_attributes; attno++)
     343             :     {
     344      162874 :         LLVMAddAttributeAtIndex(v_to, LLVMAttributeFunctionIndex,
     345      162874 :                                 attrs[attno]);
     346             :     }
     347        9994 : }
     348             : 
     349             : /*
     350             :  * Return a callable LLVMValueRef for fcinfo.
     351             :  */
     352             : LLVMValueRef
     353        2948 : llvm_function_reference(LLVMJitContext *context,
     354             :                         LLVMBuilderRef builder,
     355             :                         LLVMModuleRef mod,
     356             :                         FunctionCallInfo fcinfo)
     357             : {
     358             :     char       *modname;
     359             :     char       *basename;
     360             :     char       *funcname;
     361             : 
     362             :     LLVMValueRef v_fn;
     363             : 
     364        2948 :     fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
     365             : 
     366        2948 :     if (modname != NULL && basename != NULL)
     367             :     {
     368             :         /* external function in loadable library */
     369          28 :         funcname = psprintf("pgextern.%s.%s", modname, basename);
     370             :     }
     371        2920 :     else if (basename != NULL)
     372             :     {
     373             :         /* internal function */
     374        2920 :         funcname = psprintf("%s", basename);
     375             :     }
     376             :     else
     377             :     {
     378             :         /*
     379             :          * Function we don't know to handle, return pointer. We do so by
     380             :          * creating a global constant containing a pointer to the function.
     381             :          * Makes IR more readable.
     382             :          */
     383             :         LLVMValueRef v_fn_addr;
     384             : 
     385           0 :         funcname = psprintf("pgoidextern.%u",
     386           0 :                             fcinfo->flinfo->fn_oid);
     387           0 :         v_fn = LLVMGetNamedGlobal(mod, funcname);
     388           0 :         if (v_fn != 0)
     389           0 :             return LLVMBuildLoad(builder, v_fn, "");
     390             : 
     391           0 :         v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
     392             : 
     393           0 :         v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
     394           0 :         LLVMSetInitializer(v_fn, v_fn_addr);
     395           0 :         LLVMSetGlobalConstant(v_fn, true);
     396             : 
     397           0 :         return LLVMBuildLoad(builder, v_fn, "");
     398             :     }
     399             : 
     400             :     /* check if function already has been added */
     401        2948 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     402        2948 :     if (v_fn != 0)
     403        1208 :         return v_fn;
     404             : 
     405        1740 :     v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
     406             : 
     407        1740 :     return v_fn;
     408             : }
     409             : 
     410             : /*
     411             :  * Optimize code in module using the flags set in context.
     412             :  */
     413             : static void
     414         740 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
     415             : {
     416             :     LLVMPassManagerBuilderRef llvm_pmb;
     417             :     LLVMPassManagerRef llvm_mpm;
     418             :     LLVMPassManagerRef llvm_fpm;
     419             :     LLVMValueRef func;
     420             :     int         compile_optlevel;
     421             : 
     422         740 :     if (context->base.flags & PGJIT_OPT3)
     423         606 :         compile_optlevel = 3;
     424             :     else
     425         134 :         compile_optlevel = 0;
     426             : 
     427             :     /*
     428             :      * Have to create a new pass manager builder every pass through, as the
     429             :      * inliner has some per-builder state. Otherwise one ends up only inlining
     430             :      * a function the first time though.
     431             :      */
     432         740 :     llvm_pmb = LLVMPassManagerBuilderCreate();
     433         740 :     LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
     434         740 :     llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
     435             : 
     436         740 :     if (context->base.flags & PGJIT_OPT3)
     437             :     {
     438             :         /* TODO: Unscientifically determined threshold */
     439         606 :         LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
     440             :     }
     441             :     else
     442             :     {
     443             :         /* we rely on mem2reg heavily, so emit even in the O0 case */
     444         134 :         LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
     445             :     }
     446             : 
     447         740 :     LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
     448             : 
     449             :     /*
     450             :      * Do function level optimization. This could be moved to the point where
     451             :      * functions are emitted, to reduce memory usage a bit.
     452             :      */
     453         740 :     LLVMInitializeFunctionPassManager(llvm_fpm);
     454       14176 :     for (func = LLVMGetFirstFunction(context->module);
     455             :          func != NULL;
     456       13436 :          func = LLVMGetNextFunction(func))
     457       13436 :         LLVMRunFunctionPassManager(llvm_fpm, func);
     458         740 :     LLVMFinalizeFunctionPassManager(llvm_fpm);
     459         740 :     LLVMDisposePassManager(llvm_fpm);
     460             : 
     461             :     /*
     462             :      * Perform module level optimization. We do so even in the non-optimized
     463             :      * case, so always-inline functions etc get inlined. It's cheap enough.
     464             :      */
     465         740 :     llvm_mpm = LLVMCreatePassManager();
     466         740 :     LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
     467             :                                                     llvm_mpm);
     468             :     /* always use always-inliner pass */
     469         740 :     if (!(context->base.flags & PGJIT_OPT3))
     470         134 :         LLVMAddAlwaysInlinerPass(llvm_mpm);
     471             :     /* if doing inlining, but no expensive optimization, add inlining pass */
     472         740 :     if (context->base.flags & PGJIT_INLINE
     473         606 :         && !(context->base.flags & PGJIT_OPT3))
     474           0 :         LLVMAddFunctionInliningPass(llvm_mpm);
     475         740 :     LLVMRunPassManager(llvm_mpm, context->module);
     476         740 :     LLVMDisposePassManager(llvm_mpm);
     477             : 
     478         740 :     LLVMPassManagerBuilderDispose(llvm_pmb);
     479         740 : }
     480             : 
     481             : /*
     482             :  * Emit code for the currently pending module.
     483             :  */
     484             : static void
     485         740 : llvm_compile_module(LLVMJitContext *context)
     486             : {
     487             :     LLVMOrcModuleHandle orc_handle;
     488             :     MemoryContext oldcontext;
     489             :     static LLVMOrcJITStackRef compile_orc;
     490             :     instr_time  starttime;
     491             :     instr_time  endtime;
     492             : 
     493         740 :     if (context->base.flags & PGJIT_OPT3)
     494         606 :         compile_orc = llvm_opt3_orc;
     495             :     else
     496         134 :         compile_orc = llvm_opt0_orc;
     497             : 
     498             :     /* perform inlining */
     499         740 :     if (context->base.flags & PGJIT_INLINE)
     500             :     {
     501         606 :         INSTR_TIME_SET_CURRENT(starttime);
     502         606 :         llvm_inline(context->module);
     503         606 :         INSTR_TIME_SET_CURRENT(endtime);
     504         648 :         INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
     505             :                               endtime, starttime);
     506             :     }
     507             : 
     508         740 :     if (jit_dump_bitcode)
     509             :     {
     510             :         char       *filename;
     511             : 
     512           0 :         filename = psprintf("%u.%zu.bc",
     513             :                             MyProcPid,
     514             :                             context->module_generation);
     515           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     516           0 :         pfree(filename);
     517             :     }
     518             : 
     519             : 
     520             :     /* optimize according to the chosen optimization settings */
     521         740 :     INSTR_TIME_SET_CURRENT(starttime);
     522         740 :     llvm_optimize_module(context, context->module);
     523         740 :     INSTR_TIME_SET_CURRENT(endtime);
     524         784 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
     525             :                           endtime, starttime);
     526             : 
     527         740 :     if (jit_dump_bitcode)
     528             :     {
     529             :         char       *filename;
     530             : 
     531           0 :         filename = psprintf("%u.%zu.optimized.bc",
     532             :                             MyProcPid,
     533             :                             context->module_generation);
     534           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     535           0 :         pfree(filename);
     536             :     }
     537             : 
     538             :     /*
     539             :      * Emit the code. Note that this can, depending on the optimization
     540             :      * settings, take noticeable resources as code emission executes low-level
     541             :      * instruction combining/selection passes etc. Without optimization a
     542             :      * faster instruction selection mechanism is used.
     543             :      */
     544         740 :     INSTR_TIME_SET_CURRENT(starttime);
     545             : #if LLVM_VERSION_MAJOR > 6
     546             :     {
     547             :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, context->module,
     548             :                                         llvm_resolve_symbol, NULL))
     549             :         {
     550             :             elog(ERROR, "failed to JIT module");
     551             :         }
     552             : 
     553             :         /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
     554             :     }
     555             : #elif LLVM_VERSION_MAJOR > 4
     556             :     {
     557             :         LLVMSharedModuleRef smod;
     558             : 
     559         740 :         smod = LLVMOrcMakeSharedModule(context->module);
     560         740 :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod,
     561             :                                         llvm_resolve_symbol, NULL))
     562             :         {
     563           0 :             elog(ERROR, "failed to JIT module");
     564             :         }
     565         740 :         LLVMOrcDisposeSharedModuleRef(smod);
     566             :     }
     567             : #else                           /* LLVM 4.0 and 3.9 */
     568             :     {
     569             :         orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
     570             :                                                  llvm_resolve_symbol, NULL);
     571             :         LLVMDisposeModule(context->module);
     572             :     }
     573             : #endif
     574         740 :     INSTR_TIME_SET_CURRENT(endtime);
     575         760 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     576             :                           endtime, starttime);
     577             : 
     578         740 :     context->module = NULL;
     579         740 :     context->compiled = true;
     580             : 
     581             :     /* remember emitted code for cleanup and lookups */
     582         740 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     583             :     {
     584             :         LLVMJitHandle *handle;
     585             : 
     586         740 :         handle = (LLVMJitHandle *) palloc(sizeof(LLVMJitHandle));
     587         740 :         handle->stack = compile_orc;
     588         740 :         handle->orc_handle = orc_handle;
     589             : 
     590         740 :         context->handles = lappend(context->handles, handle);
     591             :     }
     592         740 :     MemoryContextSwitchTo(oldcontext);
     593             : 
     594         740 :     ereport(DEBUG1,
     595             :             (errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
     596             :                     INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
     597             :                     INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
     598             :                     INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
     599             :              errhidestmt(true),
     600             :              errhidecontext(true)));
     601         740 : }
     602             : 
     603             : /*
     604             :  * Per session initialization.
     605             :  */
     606             : static void
     607         938 : llvm_session_initialize(void)
     608             : {
     609             :     MemoryContext oldcontext;
     610         938 :     char       *error = NULL;
     611         938 :     char       *cpu = NULL;
     612         938 :     char       *features = NULL;
     613             : 
     614         938 :     if (llvm_session_initialized)
     615         438 :         return;
     616             : 
     617         500 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     618             : 
     619         500 :     LLVMInitializeNativeTarget();
     620         500 :     LLVMInitializeNativeAsmPrinter();
     621         500 :     LLVMInitializeNativeAsmParser();
     622             : 
     623             :     /*
     624             :      * Synchronize types early, as that also includes inferring the target
     625             :      * triple.
     626             :      */
     627         500 :     llvm_create_types();
     628             : 
     629         500 :     if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
     630             :     {
     631           0 :         elog(FATAL, "failed to query triple %s\n", error);
     632             :     }
     633             : 
     634             :     /*
     635             :      * We want the generated code to use all available features. Therefore
     636             :      * grab the host CPU string and detect features of the current CPU. The
     637             :      * latter is needed because some CPU architectures default to enabling
     638             :      * features not all CPUs have (weird, huh).
     639             :      */
     640         500 :     cpu = LLVMGetHostCPUName();
     641         500 :     features = LLVMGetHostCPUFeatures();
     642         500 :     elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
     643             :          cpu, features);
     644             : 
     645         500 :     llvm_opt0_targetmachine =
     646         500 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     647             :                                 LLVMCodeGenLevelNone,
     648             :                                 LLVMRelocDefault,
     649             :                                 LLVMCodeModelJITDefault);
     650         500 :     llvm_opt3_targetmachine =
     651         500 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     652             :                                 LLVMCodeGenLevelAggressive,
     653             :                                 LLVMRelocDefault,
     654             :                                 LLVMCodeModelJITDefault);
     655             : 
     656         500 :     LLVMDisposeMessage(cpu);
     657         500 :     cpu = NULL;
     658         500 :     LLVMDisposeMessage(features);
     659         500 :     features = NULL;
     660             : 
     661             :     /* force symbols in main binary to be loaded */
     662         500 :     LLVMLoadLibraryPermanently(NULL);
     663             : 
     664         500 :     llvm_opt0_orc = LLVMOrcCreateInstance(llvm_opt0_targetmachine);
     665         500 :     llvm_opt3_orc = LLVMOrcCreateInstance(llvm_opt3_targetmachine);
     666             : 
     667             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
     668             :     if (jit_debugging_support)
     669             :     {
     670             :         LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
     671             : 
     672             :         LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     673             :         LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     674             :     }
     675             : #endif
     676             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
     677             :     if (jit_profiling_support)
     678             :     {
     679             :         LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
     680             : 
     681             :         LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     682             :         LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     683             :     }
     684             : #endif
     685             : 
     686         500 :     before_shmem_exit(llvm_shutdown, 0);
     687             : 
     688         500 :     llvm_session_initialized = true;
     689             : 
     690         500 :     MemoryContextSwitchTo(oldcontext);
     691             : }
     692             : 
     693             : static void
     694         500 : llvm_shutdown(int code, Datum arg)
     695             : {
     696             :     /* unregister profiling support, needs to be flushed to be useful */
     697             : 
     698         500 :     if (llvm_opt3_orc)
     699             :     {
     700             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
     701             :         if (jit_profiling_support)
     702             :             LLVMOrcUnregisterPerf(llvm_opt3_orc);
     703             : #endif
     704         500 :         LLVMOrcDisposeInstance(llvm_opt3_orc);
     705         500 :         llvm_opt3_orc = NULL;
     706             :     }
     707             : 
     708         500 :     if (llvm_opt0_orc)
     709             :     {
     710             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
     711             :         if (jit_profiling_support)
     712             :             LLVMOrcUnregisterPerf(llvm_opt0_orc);
     713             : #endif
     714         500 :         LLVMOrcDisposeInstance(llvm_opt0_orc);
     715         500 :         llvm_opt0_orc = NULL;
     716             :     }
     717         500 : }
     718             : 
     719             : /* helper for llvm_create_types, returning a global var's type */
     720             : static LLVMTypeRef
     721        8500 : load_type(LLVMModuleRef mod, const char *name)
     722             : {
     723             :     LLVMValueRef value;
     724             :     LLVMTypeRef typ;
     725             : 
     726             :     /* this'll return a *pointer* to the global */
     727        8500 :     value = LLVMGetNamedGlobal(mod, name);
     728        8500 :     if (!value)
     729           0 :         elog(ERROR, "type %s is unknown", name);
     730             : 
     731             :     /* therefore look at the contained type and return that */
     732        8500 :     typ = LLVMTypeOf(value);
     733             :     Assert(typ != NULL);
     734        8500 :     typ = LLVMGetElementType(typ);
     735             :     Assert(typ != NULL);
     736        8500 :     return typ;
     737             : }
     738             : 
     739             : /* helper for llvm_create_types, returning a function's return type */
     740             : static LLVMTypeRef
     741         500 : load_return_type(LLVMModuleRef mod, const char *name)
     742             : {
     743             :     LLVMValueRef value;
     744             :     LLVMTypeRef typ;
     745             : 
     746             :     /* this'll return a *pointer* to the function */
     747         500 :     value = LLVMGetNamedFunction(mod, name);
     748         500 :     if (!value)
     749           0 :         elog(ERROR, "function %s is unknown", name);
     750             : 
     751             :     /* get type of function pointer */
     752         500 :     typ = LLVMTypeOf(value);
     753             :     Assert(typ != NULL);
     754             :     /* dereference pointer */
     755         500 :     typ = LLVMGetElementType(typ);
     756             :     Assert(typ != NULL);
     757             :     /* and look at return type */
     758         500 :     typ = LLVMGetReturnType(typ);
     759             :     Assert(typ != NULL);
     760             : 
     761         500 :     return typ;
     762             : }
     763             : 
     764             : /*
     765             :  * Load required information, types, function signatures from llvmjit_types.c
     766             :  * and make them available in global variables.
     767             :  *
     768             :  * Those global variables are then used while emitting code.
     769             :  */
     770             : static void
     771         500 : llvm_create_types(void)
     772             : {
     773             :     char        path[MAXPGPATH];
     774             :     LLVMMemoryBufferRef buf;
     775             :     char       *msg;
     776             : 
     777         500 :     snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
     778             : 
     779             :     /* open file */
     780         500 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
     781             :     {
     782           0 :         elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
     783             :              path, msg);
     784             :     }
     785             : 
     786             :     /* eagerly load contents, going to need it all */
     787         500 :     if (LLVMParseBitcode2(buf, &llvm_types_module))
     788             :     {
     789           0 :         elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
     790             :     }
     791         500 :     LLVMDisposeMemoryBuffer(buf);
     792             : 
     793             :     /*
     794             :      * Load triple & layout from clang emitted file so we're guaranteed to be
     795             :      * compatible.
     796             :      */
     797         500 :     llvm_triple = pstrdup(LLVMGetTarget(llvm_types_module));
     798         500 :     llvm_layout = pstrdup(LLVMGetDataLayoutStr(llvm_types_module));
     799             : 
     800         500 :     TypeSizeT = load_type(llvm_types_module, "TypeSizeT");
     801         500 :     TypeParamBool = load_return_type(llvm_types_module, "FunctionReturningBool");
     802         500 :     TypeStorageBool = load_type(llvm_types_module, "TypeStorageBool");
     803         500 :     TypePGFunction = load_type(llvm_types_module, "TypePGFunction");
     804         500 :     StructNullableDatum = load_type(llvm_types_module, "StructNullableDatum");
     805         500 :     StructExprContext = load_type(llvm_types_module, "StructExprContext");
     806         500 :     StructExprEvalStep = load_type(llvm_types_module, "StructExprEvalStep");
     807         500 :     StructExprState = load_type(llvm_types_module, "StructExprState");
     808         500 :     StructFunctionCallInfoData = load_type(llvm_types_module, "StructFunctionCallInfoData");
     809         500 :     StructMemoryContextData = load_type(llvm_types_module, "StructMemoryContextData");
     810         500 :     StructTupleTableSlot = load_type(llvm_types_module, "StructTupleTableSlot");
     811         500 :     StructHeapTupleTableSlot = load_type(llvm_types_module, "StructHeapTupleTableSlot");
     812         500 :     StructMinimalTupleTableSlot = load_type(llvm_types_module, "StructMinimalTupleTableSlot");
     813         500 :     StructHeapTupleData = load_type(llvm_types_module, "StructHeapTupleData");
     814         500 :     StructTupleDescData = load_type(llvm_types_module, "StructTupleDescData");
     815         500 :     StructAggState = load_type(llvm_types_module, "StructAggState");
     816         500 :     StructAggStatePerGroupData = load_type(llvm_types_module, "StructAggStatePerGroupData");
     817         500 :     StructAggStatePerTransData = load_type(llvm_types_module, "StructAggStatePerTransData");
     818             : 
     819         500 :     AttributeTemplate = LLVMGetNamedFunction(llvm_types_module, "AttributeTemplate");
     820         500 : }
     821             : 
     822             : /*
     823             :  * Split a symbol into module / function parts.  If the function is in the
     824             :  * main binary (or an external library) *modname will be NULL.
     825             :  */
     826             : void
     827       11268 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
     828             : {
     829       11268 :     *modname = NULL;
     830       11268 :     *funcname = NULL;
     831             : 
     832             :     /*
     833             :      * Module function names are pgextern.$module.$funcname
     834             :      */
     835       11268 :     if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
     836             :     {
     837             :         /*
     838             :          * Symbol names cannot contain a ., therefore we can split based on
     839             :          * first and last occurrence of one.
     840             :          */
     841          80 :         *funcname = rindex(name, '.');
     842          80 :         (*funcname)++;          /* jump over . */
     843             : 
     844         160 :         *modname = pnstrdup(name + strlen("pgextern."),
     845          80 :                             *funcname - name - strlen("pgextern.") - 1);
     846             :         Assert(funcname);
     847             : 
     848          80 :         *funcname = pstrdup(*funcname);
     849             :     }
     850             :     else
     851             :     {
     852       11188 :         *modname = NULL;
     853       11188 :         *funcname = pstrdup(name);
     854             :     }
     855       11268 : }
     856             : 
     857             : /*
     858             :  * Attempt to resolve symbol, so LLVM can emit a reference to it.
     859             :  */
     860             : static uint64_t
     861        2702 : llvm_resolve_symbol(const char *symname, void *ctx)
     862             : {
     863             :     uintptr_t   addr;
     864             :     char       *funcname;
     865             :     char       *modname;
     866             : 
     867             :     /*
     868             :      * macOS prefixes all object level symbols with an underscore. But neither
     869             :      * dlsym() nor PG's inliner expect that. So undo.
     870             :      */
     871             : #if defined(__darwin__)
     872             :     if (symname[0] != '_')
     873             :         elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
     874             :     symname++;
     875             : #endif
     876             : 
     877        2702 :     llvm_split_symbol_name(symname, &modname, &funcname);
     878             : 
     879             :     /* functions that aren't resolved to names shouldn't ever get here */
     880             :     Assert(funcname);
     881             : 
     882        2702 :     if (modname)
     883          28 :         addr = (uintptr_t) load_external_function(modname, funcname,
     884             :                                                   true, NULL);
     885             :     else
     886        2674 :         addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
     887             : 
     888        2702 :     pfree(funcname);
     889        2702 :     if (modname)
     890          28 :         pfree(modname);
     891             : 
     892             :     /* let LLVM will error out - should never happen */
     893        2702 :     if (!addr)
     894           0 :         elog(WARNING, "failed to resolve name %s", symname);
     895             : 
     896        2702 :     return (uint64_t) addr;
     897             : }

Generated by: LCOV version 1.13