LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit.c (source / functions) Hit Total Coverage
Test: PostgreSQL 13devel Lines: 246 273 90.1 %
Date: 2019-11-21 14:06:36 Functions: 19 19 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit.c
       4             :  *    Core part of the LLVM JIT provider.
       5             :  *
       6             :  * Copyright (c) 2016-2019, PostgreSQL Global Development Group
       7             :  *
       8             :  * IDENTIFICATION
       9             :  *    src/backend/jit/llvm/llvmjit.c
      10             :  *
      11             :  *-------------------------------------------------------------------------
      12             :  */
      13             : 
      14             : #include "postgres.h"
      15             : 
      16             : #include <llvm-c/Analysis.h>
      17             : #include <llvm-c/BitReader.h>
      18             : #include <llvm-c/BitWriter.h>
      19             : #include <llvm-c/Core.h>
      20             : #include <llvm-c/ExecutionEngine.h>
      21             : #include <llvm-c/OrcBindings.h>
      22             : #include <llvm-c/Support.h>
      23             : #include <llvm-c/Target.h>
      24             : #include <llvm-c/Transforms/IPO.h>
      25             : #include <llvm-c/Transforms/PassManagerBuilder.h>
      26             : #include <llvm-c/Transforms/Scalar.h>
      27             : #if LLVM_VERSION_MAJOR > 6
      28             : #include <llvm-c/Transforms/Utils.h>
      29             : #endif
      30             : 
      31             : #include "jit/llvmjit.h"
      32             : #include "jit/llvmjit_emit.h"
      33             : #include "miscadmin.h"
      34             : #include "portability/instr_time.h"
      35             : #include "storage/ipc.h"
      36             : #include "utils/memutils.h"
      37             : #include "utils/resowner_private.h"
      38             : 
      39             : /* Handle of a module emitted via ORC JIT */
      40             : typedef struct LLVMJitHandle
      41             : {
      42             :     LLVMOrcJITStackRef stack;
      43             :     LLVMOrcModuleHandle orc_handle;
      44             : } LLVMJitHandle;
      45             : 
      46             : 
      47             : /* types & functions commonly needed for JITing */
      48             : LLVMTypeRef TypeSizeT;
      49             : LLVMTypeRef TypeParamBool;
      50             : LLVMTypeRef TypeStorageBool;
      51             : LLVMTypeRef TypePGFunction;
      52             : LLVMTypeRef StructNullableDatum;
      53             : LLVMTypeRef StructHeapTupleFieldsField3;
      54             : LLVMTypeRef StructHeapTupleFields;
      55             : LLVMTypeRef StructHeapTupleHeaderData;
      56             : LLVMTypeRef StructHeapTupleDataChoice;
      57             : LLVMTypeRef StructHeapTupleData;
      58             : LLVMTypeRef StructMinimalTupleData;
      59             : LLVMTypeRef StructItemPointerData;
      60             : LLVMTypeRef StructBlockId;
      61             : LLVMTypeRef StructFormPgAttribute;
      62             : LLVMTypeRef StructTupleConstr;
      63             : LLVMTypeRef StructTupleDescData;
      64             : LLVMTypeRef StructTupleTableSlot;
      65             : LLVMTypeRef StructHeapTupleTableSlot;
      66             : LLVMTypeRef StructMinimalTupleTableSlot;
      67             : LLVMTypeRef StructMemoryContextData;
      68             : LLVMTypeRef StructPGFinfoRecord;
      69             : LLVMTypeRef StructFmgrInfo;
      70             : LLVMTypeRef StructFunctionCallInfoData;
      71             : LLVMTypeRef StructExprContext;
      72             : LLVMTypeRef StructExprEvalStep;
      73             : LLVMTypeRef StructExprState;
      74             : LLVMTypeRef StructAggState;
      75             : LLVMTypeRef StructAggStatePerGroupData;
      76             : LLVMTypeRef StructAggStatePerTransData;
      77             : 
      78             : LLVMValueRef AttributeTemplate;
      79             : LLVMValueRef FuncStrlen;
      80             : LLVMValueRef FuncVarsizeAny;
      81             : LLVMValueRef FuncSlotGetsomeattrsInt;
      82             : LLVMValueRef FuncSlotGetmissingattrs;
      83             : LLVMValueRef FuncMakeExpandedObjectReadOnlyInternal;
      84             : LLVMValueRef FuncExecEvalSubscriptingRef;
      85             : LLVMValueRef FuncExecEvalSysVar;
      86             : LLVMValueRef FuncExecAggTransReparent;
      87             : LLVMValueRef FuncExecAggInitGroup;
      88             : 
      89             : 
      90             : static bool llvm_session_initialized = false;
      91             : static size_t llvm_generation = 0;
      92             : static const char *llvm_triple = NULL;
      93             : static const char *llvm_layout = NULL;
      94             : 
      95             : 
      96             : static LLVMTargetMachineRef llvm_opt0_targetmachine;
      97             : static LLVMTargetMachineRef llvm_opt3_targetmachine;
      98             : 
      99             : static LLVMTargetRef llvm_targetref;
     100             : static LLVMOrcJITStackRef llvm_opt0_orc;
     101             : static LLVMOrcJITStackRef llvm_opt3_orc;
     102             : 
     103             : 
     104             : static void llvm_release_context(JitContext *context);
     105             : static void llvm_session_initialize(void);
     106             : static void llvm_shutdown(int code, Datum arg);
     107             : static void llvm_compile_module(LLVMJitContext *context);
     108             : static void llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module);
     109             : 
     110             : static void llvm_create_types(void);
     111             : static uint64_t llvm_resolve_symbol(const char *name, void *ctx);
     112             : 
     113             : 
     114        1162 : PG_MODULE_MAGIC;
     115             : 
     116             : 
     117             : /*
     118             :  * Initialize LLVM JIT provider.
     119             :  */
     120             : void
     121         486 : _PG_jit_provider_init(JitProviderCallbacks *cb)
     122             : {
     123         486 :     cb->reset_after_error = llvm_reset_after_error;
     124         486 :     cb->release_context = llvm_release_context;
     125         486 :     cb->compile_expr = llvm_compile_expr;
     126         486 : }
     127             : 
     128             : /*
     129             :  * Create a context for JITing work.
     130             :  *
     131             :  * The context, including subsidiary resources, will be cleaned up either when
     132             :  * the context is explicitly released, or when the lifetime of
     133             :  * CurrentResourceOwner ends (usually the end of the current [sub]xact).
     134             :  */
     135             : LLVMJitContext *
     136         864 : llvm_create_context(int jitFlags)
     137             : {
     138             :     LLVMJitContext *context;
     139             : 
     140         864 :     llvm_assert_in_fatal_section();
     141             : 
     142         864 :     llvm_session_initialize();
     143             : 
     144         864 :     ResourceOwnerEnlargeJIT(CurrentResourceOwner);
     145             : 
     146         864 :     context = MemoryContextAllocZero(TopMemoryContext,
     147             :                                      sizeof(LLVMJitContext));
     148         864 :     context->base.flags = jitFlags;
     149             : 
     150             :     /* ensure cleanup */
     151         864 :     context->base.resowner = CurrentResourceOwner;
     152         864 :     ResourceOwnerRememberJIT(CurrentResourceOwner, PointerGetDatum(context));
     153             : 
     154         864 :     return context;
     155             : }
     156             : 
     157             : /*
     158             :  * Release resources required by one llvm context.
     159             :  */
     160             : static void
     161         864 : llvm_release_context(JitContext *context)
     162             : {
     163         864 :     LLVMJitContext *llvm_context = (LLVMJitContext *) context;
     164             : 
     165         864 :     llvm_enter_fatal_on_oom();
     166             : 
     167             :     /*
     168             :      * When this backend is exiting, don't clean up LLVM. As an error might
     169             :      * have occurred from within LLVM, we do not want to risk reentering. All
     170             :      * resource cleanup is going to happen through process exit.
     171             :      */
     172         864 :     if (!proc_exit_inprogress)
     173             :     {
     174         864 :         if (llvm_context->module)
     175             :         {
     176         212 :             LLVMDisposeModule(llvm_context->module);
     177         212 :             llvm_context->module = NULL;
     178             :         }
     179             : 
     180        2380 :         while (llvm_context->handles != NIL)
     181             :         {
     182             :             LLVMJitHandle *jit_handle;
     183             : 
     184         652 :             jit_handle = (LLVMJitHandle *) linitial(llvm_context->handles);
     185         652 :             llvm_context->handles = list_delete_first(llvm_context->handles);
     186             : 
     187         652 :             LLVMOrcRemoveModule(jit_handle->stack, jit_handle->orc_handle);
     188         652 :             pfree(jit_handle);
     189             :         }
     190             :     }
     191         864 : }
     192             : 
     193             : /*
     194             :  * Return module which may be modified, e.g. by creating new functions.
     195             :  */
     196             : LLVMModuleRef
     197        7194 : llvm_mutable_module(LLVMJitContext *context)
     198             : {
     199        7194 :     llvm_assert_in_fatal_section();
     200             : 
     201             :     /*
     202             :      * If there's no in-progress module, create a new one.
     203             :      */
     204        7194 :     if (!context->module)
     205             :     {
     206         864 :         context->compiled = false;
     207         864 :         context->module_generation = llvm_generation++;
     208         864 :         context->module = LLVMModuleCreateWithName("pg");
     209         864 :         LLVMSetTarget(context->module, llvm_triple);
     210         864 :         LLVMSetDataLayout(context->module, llvm_layout);
     211             :     }
     212             : 
     213        7194 :     return context->module;
     214             : }
     215             : 
     216             : /*
     217             :  * Expand function name to be non-conflicting. This should be used by code
     218             :  * generating code, when adding new externally visible function definitions to
     219             :  * a Module.
     220             :  */
     221             : char *
     222        7194 : llvm_expand_funcname(struct LLVMJitContext *context, const char *basename)
     223             : {
     224             :     Assert(context->module != NULL);
     225             : 
     226        7194 :     context->base.instr.created_functions++;
     227             : 
     228             :     /*
     229             :      * Previously we used dots to separate, but turns out some tools, e.g.
     230             :      * GDB, don't like that and truncate name.
     231             :      */
     232        7194 :     return psprintf("%s_%zu_%d",
     233             :                     basename,
     234             :                     context->module_generation,
     235        7194 :                     context->counter++);
     236             : }
     237             : 
     238             : /*
     239             :  * Return pointer to function funcname, which has to exist. If there's pending
     240             :  * code to be optimized and emitted, do so first.
     241             :  */
     242             : void *
     243        2748 : llvm_get_function(LLVMJitContext *context, const char *funcname)
     244             : {
     245        2748 :     LLVMOrcTargetAddress addr = 0;
     246             : #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
     247             :     ListCell   *lc;
     248             : #endif
     249             : 
     250        2748 :     llvm_assert_in_fatal_section();
     251             : 
     252             :     /*
     253             :      * If there is a pending / not emitted module, compile and emit now.
     254             :      * Otherwise we might not find the [correct] function.
     255             :      */
     256        2748 :     if (!context->compiled)
     257             :     {
     258         652 :         llvm_compile_module(context);
     259             :     }
     260             : 
     261             :     /*
     262             :      * ORC's symbol table is of *unmangled* symbols. Therefore we don't need
     263             :      * to mangle here.
     264             :      */
     265             : 
     266             : #if defined(HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN) && HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN
     267             :     foreach(lc, context->handles)
     268             :     {
     269             :         LLVMJitHandle *handle = (LLVMJitHandle *) lfirst(lc);
     270             : 
     271             :         addr = 0;
     272             :         if (LLVMOrcGetSymbolAddressIn(handle->stack, &addr, handle->orc_handle, funcname))
     273             :             elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     274             :         if (addr)
     275             :             return (void *) (uintptr_t) addr;
     276             :     }
     277             : 
     278             : #else
     279             : 
     280             : #if LLVM_VERSION_MAJOR < 5
     281             :     if ((addr = LLVMOrcGetSymbolAddress(llvm_opt0_orc, funcname)))
     282             :         return (void *) (uintptr_t) addr;
     283             :     if ((addr = LLVMOrcGetSymbolAddress(llvm_opt3_orc, funcname)))
     284             :         return (void *) (uintptr_t) addr;
     285             : #else
     286        2748 :     if (LLVMOrcGetSymbolAddress(llvm_opt0_orc, &addr, funcname))
     287           0 :         elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     288        2748 :     if (addr)
     289         264 :         return (void *) (uintptr_t) addr;
     290        2484 :     if (LLVMOrcGetSymbolAddress(llvm_opt3_orc, &addr, funcname))
     291           0 :         elog(ERROR, "failed to look up symbol \"%s\"", funcname);
     292        2484 :     if (addr)
     293        2484 :         return (void *) (uintptr_t) addr;
     294             : #endif                          /* LLVM_VERSION_MAJOR */
     295             : 
     296             : #endif                          /* HAVE_DECL_LLVMORCGETSYMBOLADDRESSIN */
     297             : 
     298           0 :     elog(ERROR, "failed to JIT: %s", funcname);
     299             : 
     300             :     return NULL;
     301             : }
     302             : 
     303             : /*
     304             :  * Return declaration for passed function, adding it to the module if
     305             :  * necessary.
     306             :  *
     307             :  * This is used to make functions imported by llvm_create_types() known to the
     308             :  * module that's currently being worked on.
     309             :  */
     310             : LLVMValueRef
     311        4880 : llvm_get_decl(LLVMModuleRef mod, LLVMValueRef v_src)
     312             : {
     313             :     LLVMValueRef v_fn;
     314             : 
     315             :     /* don't repeatedly add function */
     316        4880 :     v_fn = LLVMGetNamedFunction(mod, LLVMGetValueName(v_src));
     317        4880 :     if (v_fn)
     318        3576 :         return v_fn;
     319             : 
     320        1304 :     v_fn = LLVMAddFunction(mod,
     321             :                            LLVMGetValueName(v_src),
     322             :                            LLVMGetElementType(LLVMTypeOf(v_src)));
     323        1304 :     llvm_copy_attributes(v_src, v_fn);
     324             : 
     325        1304 :     return v_fn;
     326             : }
     327             : 
     328             : /*
     329             :  * Copy attributes from one function to another.
     330             :  */
     331             : void
     332        8498 : llvm_copy_attributes(LLVMValueRef v_from, LLVMValueRef v_to)
     333             : {
     334             :     int         num_attributes;
     335             :     int         attno;
     336             :     LLVMAttributeRef *attrs;
     337             : 
     338        8498 :     num_attributes =
     339        8498 :         LLVMGetAttributeCountAtIndex(v_from, LLVMAttributeFunctionIndex);
     340             : 
     341        8498 :     attrs = palloc(sizeof(LLVMAttributeRef) * num_attributes);
     342        8498 :     LLVMGetAttributesAtIndex(v_from, LLVMAttributeFunctionIndex, attrs);
     343             : 
     344      147748 :     for (attno = 0; attno < num_attributes; attno++)
     345             :     {
     346      139250 :         LLVMAddAttributeAtIndex(v_to, LLVMAttributeFunctionIndex,
     347      139250 :                                 attrs[attno]);
     348             :     }
     349        8498 : }
     350             : 
     351             : /*
     352             :  * Return a callable LLVMValueRef for fcinfo.
     353             :  */
     354             : LLVMValueRef
     355        1616 : llvm_function_reference(LLVMJitContext *context,
     356             :                         LLVMBuilderRef builder,
     357             :                         LLVMModuleRef mod,
     358             :                         FunctionCallInfo fcinfo)
     359             : {
     360             :     char       *modname;
     361             :     char       *basename;
     362             :     char       *funcname;
     363             : 
     364             :     LLVMValueRef v_fn;
     365             : 
     366        1616 :     fmgr_symbol(fcinfo->flinfo->fn_oid, &modname, &basename);
     367             : 
     368        1616 :     if (modname != NULL && basename != NULL)
     369             :     {
     370             :         /* external function in loadable library */
     371          26 :         funcname = psprintf("pgextern.%s.%s", modname, basename);
     372             :     }
     373        1590 :     else if (basename != NULL)
     374             :     {
     375             :         /* internal function */
     376        1590 :         funcname = psprintf("%s", basename);
     377             :     }
     378             :     else
     379             :     {
     380             :         /*
     381             :          * Function we don't know to handle, return pointer. We do so by
     382             :          * creating a global constant containing a pointer to the function.
     383             :          * Makes IR more readable.
     384             :          */
     385             :         LLVMValueRef v_fn_addr;
     386             : 
     387           0 :         funcname = psprintf("pgoidextern.%u",
     388           0 :                             fcinfo->flinfo->fn_oid);
     389           0 :         v_fn = LLVMGetNamedGlobal(mod, funcname);
     390           0 :         if (v_fn != 0)
     391           0 :             return LLVMBuildLoad(builder, v_fn, "");
     392             : 
     393           0 :         v_fn_addr = l_ptr_const(fcinfo->flinfo->fn_addr, TypePGFunction);
     394             : 
     395           0 :         v_fn = LLVMAddGlobal(mod, TypePGFunction, funcname);
     396           0 :         LLVMSetInitializer(v_fn, v_fn_addr);
     397           0 :         LLVMSetGlobalConstant(v_fn, true);
     398             : 
     399           0 :         return LLVMBuildLoad(builder, v_fn, "");
     400             :     }
     401             : 
     402             :     /* check if function already has been added */
     403        1616 :     v_fn = LLVMGetNamedFunction(mod, funcname);
     404        1616 :     if (v_fn != 0)
     405         348 :         return v_fn;
     406             : 
     407        1268 :     v_fn = LLVMAddFunction(mod, funcname, LLVMGetElementType(TypePGFunction));
     408             : 
     409        1268 :     return v_fn;
     410             : }
     411             : 
     412             : /*
     413             :  * Optimize code in module using the flags set in context.
     414             :  */
     415             : static void
     416         652 : llvm_optimize_module(LLVMJitContext *context, LLVMModuleRef module)
     417             : {
     418             :     LLVMPassManagerBuilderRef llvm_pmb;
     419             :     LLVMPassManagerRef llvm_mpm;
     420             :     LLVMPassManagerRef llvm_fpm;
     421             :     LLVMValueRef func;
     422             :     int         compile_optlevel;
     423             : 
     424         652 :     if (context->base.flags & PGJIT_OPT3)
     425         586 :         compile_optlevel = 3;
     426             :     else
     427          66 :         compile_optlevel = 0;
     428             : 
     429             :     /*
     430             :      * Have to create a new pass manager builder every pass through, as the
     431             :      * inliner has some per-builder state. Otherwise one ends up only inlining
     432             :      * a function the first time though.
     433             :      */
     434         652 :     llvm_pmb = LLVMPassManagerBuilderCreate();
     435         652 :     LLVMPassManagerBuilderSetOptLevel(llvm_pmb, compile_optlevel);
     436         652 :     llvm_fpm = LLVMCreateFunctionPassManagerForModule(module);
     437             : 
     438         652 :     if (context->base.flags & PGJIT_OPT3)
     439             :     {
     440             :         /* TODO: Unscientifically determined threshold */
     441         586 :         LLVMPassManagerBuilderUseInlinerWithThreshold(llvm_pmb, 512);
     442             :     }
     443             :     else
     444             :     {
     445             :         /* we rely on mem2reg heavily, so emit even in the O0 case */
     446          66 :         LLVMAddPromoteMemoryToRegisterPass(llvm_fpm);
     447             :     }
     448             : 
     449         652 :     LLVMPassManagerBuilderPopulateFunctionPassManager(llvm_pmb, llvm_fpm);
     450             : 
     451             :     /*
     452             :      * Do function level optimization. This could be moved to the point where
     453             :      * functions are emitted, to reduce memory usage a bit.
     454             :      */
     455         652 :     LLVMInitializeFunctionPassManager(llvm_fpm);
     456       13334 :     for (func = LLVMGetFirstFunction(context->module);
     457             :          func != NULL;
     458       12030 :          func = LLVMGetNextFunction(func))
     459       12030 :         LLVMRunFunctionPassManager(llvm_fpm, func);
     460         652 :     LLVMFinalizeFunctionPassManager(llvm_fpm);
     461         652 :     LLVMDisposePassManager(llvm_fpm);
     462             : 
     463             :     /*
     464             :      * Perform module level optimization. We do so even in the non-optimized
     465             :      * case, so always-inline functions etc get inlined. It's cheap enough.
     466             :      */
     467         652 :     llvm_mpm = LLVMCreatePassManager();
     468         652 :     LLVMPassManagerBuilderPopulateModulePassManager(llvm_pmb,
     469             :                                                     llvm_mpm);
     470             :     /* always use always-inliner pass */
     471         652 :     if (!(context->base.flags & PGJIT_OPT3))
     472          66 :         LLVMAddAlwaysInlinerPass(llvm_mpm);
     473             :     /* if doing inlining, but no expensive optimization, add inlining pass */
     474         652 :     if (context->base.flags & PGJIT_INLINE
     475         586 :         && !(context->base.flags & PGJIT_OPT3))
     476           0 :         LLVMAddFunctionInliningPass(llvm_mpm);
     477         652 :     LLVMRunPassManager(llvm_mpm, context->module);
     478         652 :     LLVMDisposePassManager(llvm_mpm);
     479             : 
     480         652 :     LLVMPassManagerBuilderDispose(llvm_pmb);
     481         652 : }
     482             : 
     483             : /*
     484             :  * Emit code for the currently pending module.
     485             :  */
     486             : static void
     487         652 : llvm_compile_module(LLVMJitContext *context)
     488             : {
     489             :     LLVMOrcModuleHandle orc_handle;
     490             :     MemoryContext oldcontext;
     491             :     static LLVMOrcJITStackRef compile_orc;
     492             :     instr_time  starttime;
     493             :     instr_time  endtime;
     494             : 
     495         652 :     if (context->base.flags & PGJIT_OPT3)
     496         586 :         compile_orc = llvm_opt3_orc;
     497             :     else
     498          66 :         compile_orc = llvm_opt0_orc;
     499             : 
     500             :     /* perform inlining */
     501         652 :     if (context->base.flags & PGJIT_INLINE)
     502             :     {
     503         586 :         INSTR_TIME_SET_CURRENT(starttime);
     504         586 :         llvm_inline(context->module);
     505         586 :         INSTR_TIME_SET_CURRENT(endtime);
     506         586 :         INSTR_TIME_ACCUM_DIFF(context->base.instr.inlining_counter,
     507             :                               endtime, starttime);
     508             :     }
     509             : 
     510         652 :     if (jit_dump_bitcode)
     511             :     {
     512             :         char       *filename;
     513             : 
     514           0 :         filename = psprintf("%u.%zu.bc",
     515             :                             MyProcPid,
     516             :                             context->module_generation);
     517           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     518           0 :         pfree(filename);
     519             :     }
     520             : 
     521             : 
     522             :     /* optimize according to the chosen optimization settings */
     523         652 :     INSTR_TIME_SET_CURRENT(starttime);
     524         652 :     llvm_optimize_module(context, context->module);
     525         652 :     INSTR_TIME_SET_CURRENT(endtime);
     526         652 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.optimization_counter,
     527             :                           endtime, starttime);
     528             : 
     529         652 :     if (jit_dump_bitcode)
     530             :     {
     531             :         char       *filename;
     532             : 
     533           0 :         filename = psprintf("%u.%zu.optimized.bc",
     534             :                             MyProcPid,
     535             :                             context->module_generation);
     536           0 :         LLVMWriteBitcodeToFile(context->module, filename);
     537           0 :         pfree(filename);
     538             :     }
     539             : 
     540             :     /*
     541             :      * Emit the code. Note that this can, depending on the optimization
     542             :      * settings, take noticeable resources as code emission executes low-level
     543             :      * instruction combining/selection passes etc. Without optimization a
     544             :      * faster instruction selection mechanism is used.
     545             :      */
     546         652 :     INSTR_TIME_SET_CURRENT(starttime);
     547             : #if LLVM_VERSION_MAJOR > 6
     548             :     {
     549             :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, context->module,
     550             :                                         llvm_resolve_symbol, NULL))
     551             :         {
     552             :             elog(ERROR, "failed to JIT module");
     553             :         }
     554             : 
     555             :         /* LLVMOrcAddEagerlyCompiledIR takes ownership of the module */
     556             :     }
     557             : #elif LLVM_VERSION_MAJOR > 4
     558             :     {
     559             :         LLVMSharedModuleRef smod;
     560             : 
     561         652 :         smod = LLVMOrcMakeSharedModule(context->module);
     562         652 :         if (LLVMOrcAddEagerlyCompiledIR(compile_orc, &orc_handle, smod,
     563             :                                         llvm_resolve_symbol, NULL))
     564             :         {
     565           0 :             elog(ERROR, "failed to JIT module");
     566             :         }
     567         652 :         LLVMOrcDisposeSharedModuleRef(smod);
     568             :     }
     569             : #else                           /* LLVM 4.0 and 3.9 */
     570             :     {
     571             :         orc_handle = LLVMOrcAddEagerlyCompiledIR(compile_orc, context->module,
     572             :                                                  llvm_resolve_symbol, NULL);
     573             :         LLVMDisposeModule(context->module);
     574             :     }
     575             : #endif
     576         652 :     INSTR_TIME_SET_CURRENT(endtime);
     577         652 :     INSTR_TIME_ACCUM_DIFF(context->base.instr.emission_counter,
     578             :                           endtime, starttime);
     579             : 
     580         652 :     context->module = NULL;
     581         652 :     context->compiled = true;
     582             : 
     583             :     /* remember emitted code for cleanup and lookups */
     584         652 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     585             :     {
     586             :         LLVMJitHandle *handle;
     587             : 
     588         652 :         handle = (LLVMJitHandle *) palloc(sizeof(LLVMJitHandle));
     589         652 :         handle->stack = compile_orc;
     590         652 :         handle->orc_handle = orc_handle;
     591             : 
     592         652 :         context->handles = lappend(context->handles, handle);
     593             :     }
     594         652 :     MemoryContextSwitchTo(oldcontext);
     595             : 
     596         652 :     ereport(DEBUG1,
     597             :             (errmsg("time to inline: %.3fs, opt: %.3fs, emit: %.3fs",
     598             :                     INSTR_TIME_GET_DOUBLE(context->base.instr.inlining_counter),
     599             :                     INSTR_TIME_GET_DOUBLE(context->base.instr.optimization_counter),
     600             :                     INSTR_TIME_GET_DOUBLE(context->base.instr.emission_counter)),
     601             :              errhidestmt(true),
     602             :              errhidecontext(true)));
     603         652 : }
     604             : 
     605             : /*
     606             :  * Per session initialization.
     607             :  */
     608             : static void
     609         864 : llvm_session_initialize(void)
     610             : {
     611             :     MemoryContext oldcontext;
     612         864 :     char       *error = NULL;
     613         864 :     char       *cpu = NULL;
     614         864 :     char       *features = NULL;
     615             : 
     616         864 :     if (llvm_session_initialized)
     617         378 :         return;
     618             : 
     619         486 :     oldcontext = MemoryContextSwitchTo(TopMemoryContext);
     620             : 
     621         486 :     LLVMInitializeNativeTarget();
     622         486 :     LLVMInitializeNativeAsmPrinter();
     623         486 :     LLVMInitializeNativeAsmParser();
     624             : 
     625             :     /*
     626             :      * Synchronize types early, as that also includes inferring the target
     627             :      * triple.
     628             :      */
     629         486 :     llvm_create_types();
     630             : 
     631         486 :     if (LLVMGetTargetFromTriple(llvm_triple, &llvm_targetref, &error) != 0)
     632             :     {
     633           0 :         elog(FATAL, "failed to query triple %s\n", error);
     634             :     }
     635             : 
     636             :     /*
     637             :      * We want the generated code to use all available features. Therefore
     638             :      * grab the host CPU string and detect features of the current CPU. The
     639             :      * latter is needed because some CPU architectures default to enabling
     640             :      * features not all CPUs have (weird, huh).
     641             :      */
     642         486 :     cpu = LLVMGetHostCPUName();
     643         486 :     features = LLVMGetHostCPUFeatures();
     644         486 :     elog(DEBUG2, "LLVMJIT detected CPU \"%s\", with features \"%s\"",
     645             :          cpu, features);
     646             : 
     647         486 :     llvm_opt0_targetmachine =
     648         486 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     649             :                                 LLVMCodeGenLevelNone,
     650             :                                 LLVMRelocDefault,
     651             :                                 LLVMCodeModelJITDefault);
     652         486 :     llvm_opt3_targetmachine =
     653         486 :         LLVMCreateTargetMachine(llvm_targetref, llvm_triple, cpu, features,
     654             :                                 LLVMCodeGenLevelAggressive,
     655             :                                 LLVMRelocDefault,
     656             :                                 LLVMCodeModelJITDefault);
     657             : 
     658         486 :     LLVMDisposeMessage(cpu);
     659         486 :     cpu = NULL;
     660         486 :     LLVMDisposeMessage(features);
     661         486 :     features = NULL;
     662             : 
     663             :     /* force symbols in main binary to be loaded */
     664         486 :     LLVMLoadLibraryPermanently(NULL);
     665             : 
     666         486 :     llvm_opt0_orc = LLVMOrcCreateInstance(llvm_opt0_targetmachine);
     667         486 :     llvm_opt3_orc = LLVMOrcCreateInstance(llvm_opt3_targetmachine);
     668             : 
     669             : #if defined(HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER) && HAVE_DECL_LLVMCREATEGDBREGISTRATIONLISTENER
     670             :     if (jit_debugging_support)
     671             :     {
     672             :         LLVMJITEventListenerRef l = LLVMCreateGDBRegistrationListener();
     673             : 
     674             :         LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     675             :         LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     676             :     }
     677             : #endif
     678             : #if defined(HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER) && HAVE_DECL_LLVMCREATEPERFJITEVENTLISTENER
     679             :     if (jit_profiling_support)
     680             :     {
     681             :         LLVMJITEventListenerRef l = LLVMCreatePerfJITEventListener();
     682             : 
     683             :         LLVMOrcRegisterJITEventListener(llvm_opt0_orc, l);
     684             :         LLVMOrcRegisterJITEventListener(llvm_opt3_orc, l);
     685             :     }
     686             : #endif
     687             : 
     688         486 :     before_shmem_exit(llvm_shutdown, 0);
     689             : 
     690         486 :     llvm_session_initialized = true;
     691             : 
     692         486 :     MemoryContextSwitchTo(oldcontext);
     693             : }
     694             : 
     695             : static void
     696         486 : llvm_shutdown(int code, Datum arg)
     697             : {
     698             :     /* unregister profiling support, needs to be flushed to be useful */
     699             : 
     700         486 :     if (llvm_opt3_orc)
     701             :     {
     702             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
     703             :         if (jit_profiling_support)
     704             :             LLVMOrcUnregisterPerf(llvm_opt3_orc);
     705             : #endif
     706         486 :         LLVMOrcDisposeInstance(llvm_opt3_orc);
     707         486 :         llvm_opt3_orc = NULL;
     708             :     }
     709             : 
     710         486 :     if (llvm_opt0_orc)
     711             :     {
     712             : #if defined(HAVE_DECL_LLVMORCREGISTERPERF) && HAVE_DECL_LLVMORCREGISTERPERF
     713             :         if (jit_profiling_support)
     714             :             LLVMOrcUnregisterPerf(llvm_opt0_orc);
     715             : #endif
     716         486 :         LLVMOrcDisposeInstance(llvm_opt0_orc);
     717         486 :         llvm_opt0_orc = NULL;
     718             :     }
     719         486 : }
     720             : 
     721             : /* helper for llvm_create_types, returning a global var's type */
     722             : static LLVMTypeRef
     723        8262 : load_type(LLVMModuleRef mod, const char *name)
     724             : {
     725             :     LLVMValueRef value;
     726             :     LLVMTypeRef typ;
     727             : 
     728             :     /* this'll return a *pointer* to the global */
     729        8262 :     value = LLVMGetNamedGlobal(mod, name);
     730        8262 :     if (!value)
     731           0 :         elog(ERROR, "type %s is unknown", name);
     732             : 
     733             :     /* therefore look at the contained type and return that */
     734        8262 :     typ = LLVMTypeOf(value);
     735             :     Assert(typ != NULL);
     736        8262 :     typ = LLVMGetElementType(typ);
     737             :     Assert(typ != NULL);
     738        8262 :     return typ;
     739             : }
     740             : 
     741             : /* helper for llvm_create_types, returning a function's return type */
     742             : static LLVMTypeRef
     743         486 : load_return_type(LLVMModuleRef mod, const char *name)
     744             : {
     745             :     LLVMValueRef value;
     746             :     LLVMTypeRef typ;
     747             : 
     748             :     /* this'll return a *pointer* to the function */
     749         486 :     value = LLVMGetNamedFunction(mod, name);
     750         486 :     if (!value)
     751           0 :         elog(ERROR, "function %s is unknown", name);
     752             : 
     753             :     /* get type of function pointer */
     754         486 :     typ = LLVMTypeOf(value);
     755             :     Assert(typ != NULL);
     756             :     /* dereference pointer */
     757         486 :     typ = LLVMGetElementType(typ);
     758             :     Assert(typ != NULL);
     759             :     /* and look at return type */
     760         486 :     typ = LLVMGetReturnType(typ);
     761             :     Assert(typ != NULL);
     762             : 
     763         486 :     return typ;
     764             : }
     765             : 
     766             : /*
     767             :  * Load required information, types, function signatures from llvmjit_types.c
     768             :  * and make them available in global variables.
     769             :  *
     770             :  * Those global variables are then used while emitting code.
     771             :  */
     772             : static void
     773         486 : llvm_create_types(void)
     774             : {
     775             :     char        path[MAXPGPATH];
     776             :     LLVMMemoryBufferRef buf;
     777             :     char       *msg;
     778         486 :     LLVMModuleRef mod = NULL;
     779             : 
     780         486 :     snprintf(path, MAXPGPATH, "%s/%s", pkglib_path, "llvmjit_types.bc");
     781             : 
     782             :     /* open file */
     783         486 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
     784             :     {
     785           0 :         elog(ERROR, "LLVMCreateMemoryBufferWithContentsOfFile(%s) failed: %s",
     786             :              path, msg);
     787             :     }
     788             : 
     789             :     /* eagerly load contents, going to need it all */
     790         486 :     if (LLVMParseBitcode2(buf, &mod))
     791             :     {
     792           0 :         elog(ERROR, "LLVMParseBitcode2 of %s failed", path);
     793             :     }
     794         486 :     LLVMDisposeMemoryBuffer(buf);
     795             : 
     796             :     /*
     797             :      * Load triple & layout from clang emitted file so we're guaranteed to be
     798             :      * compatible.
     799             :      */
     800         486 :     llvm_triple = pstrdup(LLVMGetTarget(mod));
     801         486 :     llvm_layout = pstrdup(LLVMGetDataLayoutStr(mod));
     802             : 
     803         486 :     TypeSizeT = load_type(mod, "TypeSizeT");
     804         486 :     TypeParamBool = load_return_type(mod, "FunctionReturningBool");
     805         486 :     TypeStorageBool = load_type(mod, "TypeStorageBool");
     806         486 :     TypePGFunction = load_type(mod, "TypePGFunction");
     807         486 :     StructNullableDatum = load_type(mod, "StructNullableDatum");
     808         486 :     StructExprContext = load_type(mod, "StructExprContext");
     809         486 :     StructExprEvalStep = load_type(mod, "StructExprEvalStep");
     810         486 :     StructExprState = load_type(mod, "StructExprState");
     811         486 :     StructFunctionCallInfoData = load_type(mod, "StructFunctionCallInfoData");
     812         486 :     StructMemoryContextData = load_type(mod, "StructMemoryContextData");
     813         486 :     StructTupleTableSlot = load_type(mod, "StructTupleTableSlot");
     814         486 :     StructHeapTupleTableSlot = load_type(mod, "StructHeapTupleTableSlot");
     815         486 :     StructMinimalTupleTableSlot = load_type(mod, "StructMinimalTupleTableSlot");
     816         486 :     StructHeapTupleData = load_type(mod, "StructHeapTupleData");
     817         486 :     StructTupleDescData = load_type(mod, "StructTupleDescData");
     818         486 :     StructAggState = load_type(mod, "StructAggState");
     819         486 :     StructAggStatePerGroupData = load_type(mod, "StructAggStatePerGroupData");
     820         486 :     StructAggStatePerTransData = load_type(mod, "StructAggStatePerTransData");
     821             : 
     822         486 :     AttributeTemplate = LLVMGetNamedFunction(mod, "AttributeTemplate");
     823         486 :     FuncStrlen = LLVMGetNamedFunction(mod, "strlen");
     824         486 :     FuncVarsizeAny = LLVMGetNamedFunction(mod, "varsize_any");
     825         486 :     FuncSlotGetsomeattrsInt = LLVMGetNamedFunction(mod, "slot_getsomeattrs_int");
     826         486 :     FuncSlotGetmissingattrs = LLVMGetNamedFunction(mod, "slot_getmissingattrs");
     827         486 :     FuncMakeExpandedObjectReadOnlyInternal = LLVMGetNamedFunction(mod, "MakeExpandedObjectReadOnlyInternal");
     828         486 :     FuncExecEvalSubscriptingRef = LLVMGetNamedFunction(mod, "ExecEvalSubscriptingRef");
     829         486 :     FuncExecEvalSysVar = LLVMGetNamedFunction(mod, "ExecEvalSysVar");
     830         486 :     FuncExecAggTransReparent = LLVMGetNamedFunction(mod, "ExecAggTransReparent");
     831         486 :     FuncExecAggInitGroup = LLVMGetNamedFunction(mod, "ExecAggInitGroup");
     832             : 
     833             :     /*
     834             :      * Leave the module alive, otherwise references to function would be
     835             :      * dangling.
     836             :      */
     837             : 
     838         486 :     return;
     839             : }
     840             : 
     841             : /*
     842             :  * Split a symbol into module / function parts.  If the function is in the
     843             :  * main binary (or an external library) *modname will be NULL.
     844             :  */
     845             : void
     846       10278 : llvm_split_symbol_name(const char *name, char **modname, char **funcname)
     847             : {
     848       10278 :     *modname = NULL;
     849       10278 :     *funcname = NULL;
     850             : 
     851             :     /*
     852             :      * Module function names are pgextern.$module.$funcname
     853             :      */
     854       10278 :     if (strncmp(name, "pgextern.", strlen("pgextern.")) == 0)
     855             :     {
     856             :         /*
     857             :          * Symbol names cannot contain a ., therefore we can split based on
     858             :          * first and last occurrence of one.
     859             :          */
     860          74 :         *funcname = rindex(name, '.');
     861          74 :         (*funcname)++;          /* jump over . */
     862             : 
     863          74 :         *modname = pnstrdup(name + strlen("pgextern."),
     864          74 :                             *funcname - name - strlen("pgextern.") - 1);
     865             :         Assert(funcname);
     866             : 
     867          74 :         *funcname = pstrdup(*funcname);
     868             :     }
     869             :     else
     870             :     {
     871       10204 :         *modname = NULL;
     872       10204 :         *funcname = pstrdup(name);
     873             :     }
     874       10278 : }
     875             : 
     876             : /*
     877             :  * Attempt to resolve symbol, so LLVM can emit a reference to it.
     878             :  */
     879             : static uint64_t
     880        2104 : llvm_resolve_symbol(const char *symname, void *ctx)
     881             : {
     882             :     uintptr_t   addr;
     883             :     char       *funcname;
     884             :     char       *modname;
     885             : 
     886             :     /*
     887             :      * macOS prefixes all object level symbols with an underscore. But neither
     888             :      * dlsym() nor PG's inliner expect that. So undo.
     889             :      */
     890             : #if defined(__darwin__)
     891             :     if (symname[0] != '_')
     892             :         elog(ERROR, "expected prefixed symbol name, but got \"%s\"", symname);
     893             :     symname++;
     894             : #endif
     895             : 
     896        2104 :     llvm_split_symbol_name(symname, &modname, &funcname);
     897             : 
     898             :     /* functions that aren't resolved to names shouldn't ever get here */
     899             :     Assert(funcname);
     900             : 
     901        2104 :     if (modname)
     902          26 :         addr = (uintptr_t) load_external_function(modname, funcname,
     903             :                                                   true, NULL);
     904             :     else
     905        2078 :         addr = (uintptr_t) LLVMSearchForAddressOfSymbol(symname);
     906             : 
     907        2104 :     pfree(funcname);
     908        2104 :     if (modname)
     909          26 :         pfree(modname);
     910             : 
     911             :     /* let LLVM will error out - should never happen */
     912        2104 :     if (!addr)
     913           0 :         elog(WARNING, "failed to resolve name %s", symname);
     914             : 
     915        2104 :     return (uint64_t) addr;
     916             : }

Generated by: LCOV version 1.13