LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit_inline.cpp (source / functions) Hit Total Coverage
Test: PostgreSQL 19devel Lines: 222 264 84.1 %
Date: 2025-10-02 06:18:21 Functions: 11 13 84.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit_inline.cpp
       4             :  *    Cross module inlining suitable for postgres' JIT
       5             :  *
       6             :  * The inliner iterates over external functions referenced from the passed
       7             :  * module and attempts to inline those.  It does so by utilizing pre-built
       8             :  * indexes over both postgres core code and extension modules.  When a match
       9             :  * for an external function is found - not guaranteed! - the index will then
      10             :  * be used to judge their instruction count / inline worthiness. After doing
      11             :  * so for all external functions, all the referenced functions (and
      12             :  * prerequisites) will be imported.
      13             :  *
      14             :  * Copyright (c) 2016-2025, PostgreSQL Global Development Group
      15             :  *
      16             :  * IDENTIFICATION
      17             :  *    src/backend/lib/llvmjit/llvmjit_inline.cpp
      18             :  *
      19             :  *-------------------------------------------------------------------------
      20             :  */
      21             : 
      22             : extern "C"
      23             : {
      24             : #include "postgres.h"
      25             : }
      26             : 
      27             : #include "jit/llvmjit.h"
      28             : 
      29             : extern "C"
      30             : {
      31             : #include <fcntl.h>
      32             : #include <sys/mman.h>
      33             : #include <sys/stat.h>
      34             : #include <sys/types.h>
      35             : #include <unistd.h>
      36             : 
      37             : #include "common/string.h"
      38             : #include "miscadmin.h"
      39             : #include "storage/fd.h"
      40             : }
      41             : 
      42             : #include <llvm-c/Core.h>
      43             : #include <llvm-c/BitReader.h>
      44             : 
      45             : /* Avoid macro clash with LLVM's C++ headers */
      46             : #undef Min
      47             : 
      48             : #include <llvm/ADT/SetVector.h>
      49             : #include <llvm/ADT/StringSet.h>
      50             : #include <llvm/ADT/StringMap.h>
      51             : #include <llvm/Analysis/ModuleSummaryAnalysis.h>
      52             : #include <llvm/Bitcode/BitcodeReader.h>
      53             : #include <llvm/IR/Attributes.h>
      54             : #include <llvm/IR/DebugInfo.h>
      55             : #include <llvm/IR/IntrinsicInst.h>
      56             : #include <llvm/IR/IRBuilder.h>
      57             : #include <llvm/IR/ModuleSummaryIndex.h>
      58             : #include <llvm/Linker/IRMover.h>
      59             : #include <llvm/Support/ManagedStatic.h>
      60             : #include <llvm/Support/MemoryBuffer.h>
      61             : 
      62             : 
      63             : /*
      64             :  * Type used to represent modules InlineWorkListItem's subject is searched for
      65             :  * in.
      66             :  */
      67             : typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
      68             : 
      69             : /*
      70             :  * Item in queue of to-be-checked symbols and corresponding queue.
      71             :  */
      72             : typedef struct InlineWorkListItem
      73             : {
      74             :     llvm::StringRef symbolName;
      75             :     llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
      76             : } InlineWorkListItem;
      77             : typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
      78             : 
      79             : /*
      80             :  * Information about symbols processed during inlining. Used to prevent
      81             :  * repeated searches and provide additional information.
      82             :  */
      83             : typedef struct FunctionInlineState
      84             : {
      85             :     int costLimit;
      86             :     bool processed;
      87             :     bool inlined;
      88             :     bool allowReconsidering;
      89             : } FunctionInlineState;
      90             : typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
      91             : 
      92             : /*
      93             :  * Map of modules that should be inlined, with a list of the to-be inlined
      94             :  * symbols.
      95             :  */
      96             : typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
      97             : 
      98             : 
      99             : const float inline_cost_decay_factor = 0.5;
     100             : const int inline_initial_cost = 150;
     101             : 
     102             : /*
     103             :  * These are managed statics so LLVM knows to deallocate them during an
     104             :  * LLVMShutdown(), rather than after (which'd cause crashes).
     105             :  */
     106             : typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
     107             : llvm::ManagedStatic<ModuleCache> module_cache;
     108             : typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
     109             : llvm::ManagedStatic<SummaryCache> summary_cache;
     110             : 
     111             : 
     112             : static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod);
     113             : static void llvm_execute_inline_plan(llvm::Module *mod,
     114             :                                      ImportMapTy *globalsToInline);
     115             : 
     116             : static llvm::Module* load_module_cached(LLVMContextRef c, llvm::StringRef modPath);
     117             : static std::unique_ptr<llvm::Module> load_module(LLVMContextRef c, llvm::StringRef Identifier);
     118             : static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
     119             : 
     120             : 
     121             : static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     122             :                                                    llvm::Function *F,
     123             :                                                    llvm::StringRef Name);
     124             : 
     125             : static bool function_inlinable(llvm::Function &F,
     126             :                                int threshold,
     127             :                                FunctionInlineStates &functionState,
     128             :                                InlineWorkList &worklist,
     129             :                                InlineSearchPath &searchpath,
     130             :                                llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     131             :                                int &running_instcount,
     132             :                                llvm::StringSet<> &importVars);
     133             : static void function_references(llvm::Function &F,
     134             :                                 int &running_instcount,
     135             :                                 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     136             :                                 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
     137             : 
     138             : static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
     139             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     140             : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
     141             : 
     142             : /* verbose debugging for inliner development */
     143             : /* #define INLINE_DEBUG */
     144             : #ifdef INLINE_DEBUG
     145             : #define ilog        elog
     146             : #else
     147             : #define ilog(...)   (void) 0
     148             : #endif
     149             : 
     150             : /*
     151             :  * Reset inlining related state. This needs to be called before the currently
     152             :  * used LLVMContextRef is disposed (and a new one create), otherwise we would
     153             :  * have dangling references to deleted modules.
     154             :  */
     155             : void
     156           0 : llvm_inline_reset_caches(void)
     157             : {
     158           0 :     module_cache->clear();
     159           0 :     summary_cache->clear();
     160           0 : }
     161             : 
     162             : /*
     163             :  * Perform inlining of external function references in M based on a simple
     164             :  * cost based analysis.
     165             :  */
     166             : void
     167         542 : llvm_inline(LLVMModuleRef M)
     168             : {
     169         542 :     LLVMContextRef lc = LLVMGetModuleContext(M);
     170         542 :     llvm::Module *mod = llvm::unwrap(M);
     171             : 
     172         542 :     std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
     173         542 :     if (!globalsToInline)
     174           0 :         return;
     175         542 :     llvm_execute_inline_plan(mod, globalsToInline.get());
     176         542 : }
     177             : 
     178             : /*
     179             :  * Build information necessary for inlining external function references in
     180             :  * mod.
     181             :  */
     182             : static std::unique_ptr<ImportMapTy>
     183         542 : llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
     184             : {
     185         542 :     std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
     186         542 :     FunctionInlineStates functionStates;
     187         542 :     InlineWorkList worklist;
     188             : 
     189         542 :     InlineSearchPath defaultSearchPath;
     190             : 
     191             :     /* attempt to add module to search path */
     192         542 :     add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
     193             :     /* if postgres isn't available, no point continuing */
     194         542 :     if (defaultSearchPath.empty())
     195           0 :         return nullptr;
     196             : 
     197             :     /*
     198             :      * Start inlining with current references to external functions by putting
     199             :      * them on the inlining worklist. If, during inlining of those, new extern
     200             :      * functions need to be inlined, they'll also be put there, with a lower
     201             :      * priority.
     202             :      */
     203       10096 :     for (const llvm::Function &funcDecl : mod->functions())
     204             :     {
     205        9554 :         InlineWorkListItem item = {};
     206        9554 :         FunctionInlineState inlineState = {};
     207             : 
     208             :         /* already has a definition */
     209        9554 :         if (!funcDecl.isDeclaration())
     210        6580 :             continue;
     211             : 
     212             :         /* llvm provides implementation */
     213        2974 :         if (funcDecl.isIntrinsic())
     214         482 :             continue;
     215             : 
     216        2492 :         item.symbolName = funcDecl.getName();
     217        2492 :         item.searchpath = defaultSearchPath;
     218        2492 :         worklist.push_back(item);
     219        2492 :         inlineState.costLimit = inline_initial_cost;
     220        2492 :         inlineState.processed = false;
     221        2492 :         inlineState.inlined = false;
     222        2492 :         inlineState.allowReconsidering = false;
     223        2492 :         functionStates[funcDecl.getName()] = inlineState;
     224        9554 :     }
     225             : 
     226             :     /*
     227             :      * Iterate over pending worklist items, look them up in index, check
     228             :      * whether they should be inlined.
     229             :      */
     230       10910 :     while (!worklist.empty())
     231             :     {
     232       10368 :         InlineWorkListItem item = worklist.pop_back_val();
     233       10368 :         llvm::StringRef symbolName = item.symbolName;
     234             :         char *cmodname;
     235             :         char *cfuncname;
     236       10368 :         FunctionInlineState &inlineState = functionStates[symbolName];
     237             :         llvm::GlobalValue::GUID funcGUID;
     238             : 
     239       10368 :         llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
     240             : 
     241             : #if LLVM_VERSION_MAJOR >= 21
     242             :         funcGUID = llvm::GlobalValue::getGUIDAssumingExternalLinkage(cfuncname);
     243             : #else
     244       10368 :         funcGUID = llvm::GlobalValue::getGUID(cfuncname);
     245             : #endif
     246             : 
     247             :         /* already processed */
     248       10368 :         if (inlineState.processed)
     249           0 :             continue;
     250             : 
     251             : 
     252       10368 :         if (cmodname)
     253           0 :             add_module_to_inline_search_path(item.searchpath, cmodname);
     254             : 
     255             :         /*
     256             :          * Iterate over all known definitions of function, via the index. Then
     257             :          * look up module(s), check if function actually is defined (there
     258             :          * could be hash conflicts).
     259             :          */
     260       15630 :         for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
     261             :         {
     262             :             const llvm::FunctionSummary *fs;
     263        9668 :             llvm::StringRef modPath = gvs->modulePath();
     264             :             llvm::Module *defMod;
     265             :             llvm::Function *funcDef;
     266             : 
     267        9668 :             fs = llvm::cast<llvm::FunctionSummary>(gvs);
     268             : 
     269        9668 :             if (gvs->notEligibleToImport())
     270             :             {
     271             :                 ilog(DEBUG1, "ineligibile to import %s due to summary",
     272             :                      symbolName.data());
     273        4552 :                 continue;
     274             :             }
     275             : 
     276        9668 :             if ((int) fs->instCount() > inlineState.costLimit)
     277             :             {
     278             :                 ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
     279             :                      symbolName.data(), fs->instCount(), inlineState.costLimit);
     280        3944 :                 inlineState.allowReconsidering = true;
     281        3944 :                 continue;
     282             :             }
     283             : 
     284        5724 :             defMod = load_module_cached(lc, modPath);
     285        5724 :             if (defMod->materializeMetadata())
     286           0 :                 elog(FATAL, "failed to materialize metadata");
     287             : 
     288        5724 :             funcDef = defMod->getFunction(cfuncname);
     289             : 
     290             :             /*
     291             :              * This can happen e.g. in case of a hash collision of the
     292             :              * function's name.
     293             :              */
     294        5724 :             if (!funcDef)
     295           0 :                 continue;
     296             : 
     297        5724 :             if (funcDef->materialize())
     298           0 :                 elog(FATAL, "failed to materialize metadata");
     299             : 
     300             :             Assert(!funcDef->isDeclaration());
     301             :             Assert(funcDef->hasExternalLinkage());
     302             : 
     303        5724 :             llvm::StringSet<> importVars;
     304        5724 :             llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
     305        5724 :             int running_instcount = 0;
     306             : 
     307             :             /*
     308             :              * Check whether function, and objects it depends on, are
     309             :              * inlinable.
     310             :              */
     311        5724 :             if (function_inlinable(*funcDef,
     312             :                                    inlineState.costLimit,
     313             :                                    functionStates,
     314             :                                    worklist,
     315             :                                    item.searchpath,
     316             :                                    visitedFunctions,
     317             :                                    running_instcount,
     318             :                                    importVars))
     319             :             {
     320             :                 /*
     321             :                  * Check whether function and all its dependencies are too
     322             :                  * big. Dependencies already counted for other functions that
     323             :                  * will get inlined are not counted again. While this make
     324             :                  * things somewhat order dependent, I can't quite see a point
     325             :                  * in a different behaviour.
     326             :                  */
     327        5014 :                 if (running_instcount > inlineState.costLimit)
     328             :                 {
     329             :                     ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
     330             :                          symbolName.data(), running_instcount, inlineState.costLimit);
     331         608 :                     inlineState.allowReconsidering = true;
     332         608 :                     continue;
     333             :                 }
     334             : 
     335             :                 ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
     336             :                      symbolName.data(), running_instcount, fs->instCount());
     337             : 
     338             :                 /* import referenced function itself */
     339        4406 :                 importVars.insert(symbolName);
     340             : 
     341             :                 {
     342        4406 :                     llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
     343       10798 :                     for (auto& importVar : importVars)
     344        6392 :                         modGlobalsToInline.insert(importVar.first());
     345             :                     Assert(modGlobalsToInline.size() > 0);
     346             :                 }
     347             : 
     348             :                 /* mark function as inlined */
     349        4406 :                 inlineState.inlined = true;
     350             : 
     351             :                 /*
     352             :                  * Found definition to inline, don't look for further
     353             :                  * potential definitions.
     354             :                  */
     355        4406 :                 break;
     356             :             }
     357             :             else
     358             :             {
     359             :                 ilog(DEBUG1, "had to skip inlining %s",
     360             :                      symbolName.data());
     361             : 
     362             :                 /* It's possible there's another definition that's inlinable. */
     363             :             }
     364       21106 :         }
     365             : 
     366             :         /*
     367             :          * Signal that we're done with symbol, whether successful (inlined =
     368             :          * true above) or not.
     369             :          */
     370       10368 :         inlineState.processed = true;
     371       10368 :     }
     372             : 
     373         542 :     return globalsToInline;
     374         542 : }
     375             : 
     376             : /*
     377             :  * Perform the actual inlining of external functions (and their dependencies)
     378             :  * into mod.
     379             :  */
     380             : static void
     381         542 : llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
     382             : {
     383         542 :     llvm::IRMover Mover(*mod);
     384             : 
     385        3512 :     for (const auto& toInline : *globalsToInline)
     386             :     {
     387        2970 :         const llvm::StringRef& modPath = toInline.first();
     388        2970 :         const llvm::StringSet<>& modGlobalsToInline = toInline.second;
     389        2970 :         llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
     390             : 
     391             :         Assert(module_cache->count(modPath));
     392        2970 :         std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
     393        2970 :         module_cache->erase(modPath);
     394             : 
     395        2970 :         if (modGlobalsToInline.empty())
     396           0 :             continue;
     397             : 
     398        9010 :         for (auto &glob: modGlobalsToInline)
     399             :         {
     400        6040 :             llvm::StringRef SymbolName = glob.first();
     401             :             char *modname;
     402             :             char *funcname;
     403             : 
     404        6040 :             llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
     405             : 
     406        6040 :             llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
     407             : 
     408        6040 :             if (!valueToImport)
     409           0 :                 elog(FATAL, "didn't refind value %s to import", SymbolName.data());
     410             : 
     411             :             /*
     412             :              * For functions (global vars are only inlined if already static),
     413             :              * mark imported variables as being clones from other
     414             :              * functions. That a) avoids symbol conflicts b) allows the
     415             :              * optimizer to perform inlining.
     416             :             */
     417        6040 :             if (llvm::isa<llvm::Function>(valueToImport))
     418             :             {
     419        4496 :                 llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
     420             :                 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     421             : 
     422             :                 /*
     423             :                  * Per-function info isn't necessarily stripped yet, as the
     424             :                  * module is lazy-loaded when stripped above.
     425             :                  */
     426        4496 :                 llvm::stripDebugInfo(*F);
     427             : 
     428             :                 /*
     429             :                  * If the to-be-imported function is one referenced including
     430             :                  * its module name, create a tiny inline function that just
     431             :                  * forwards the call. One might think a GlobalAlias would do
     432             :                  * the trick, but a) IRMover doesn't override a declaration
     433             :                  * with an alias pointing to a definition (instead renaming
     434             :                  * it), b) Aliases can't be AvailableExternally.
     435             :                  */
     436        4496 :                 if (modname)
     437             :                 {
     438             :                     llvm::Function *AF;
     439             : 
     440           0 :                     AF = create_redirection_function(importMod, F, SymbolName);
     441             : 
     442           0 :                     GlobalsToImport.insert(AF);
     443           0 :                     llvm::stripDebugInfo(*AF);
     444             :                 }
     445             : 
     446        4496 :                 if (valueToImport->hasExternalLinkage())
     447             :                 {
     448        4406 :                     valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
     449             :                 }
     450             :             }
     451             : 
     452        6040 :             GlobalsToImport.insert(valueToImport);
     453             :             ilog(DEBUG1, "performing import of %s %s",
     454             :                  modPath.data(), SymbolName.data());
     455             : 
     456             :         }
     457             : 
     458        2970 :         if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
     459         928 :                        [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {},
     460             :                        /*IsPerformingImport=*/false))
     461           0 :             elog(FATAL, "function import failed with linker error");
     462        2970 :     }
     463         542 : }
     464             : 
     465             : /*
     466             :  * Return a module identified by modPath, caching it in memory.
     467             :  *
     468             :  * Note that such a module may *not* be modified without copying, otherwise
     469             :  * the cache state would get corrupted.
     470             :  */
     471             : static llvm::Module*
     472        5724 : load_module_cached(LLVMContextRef lc, llvm::StringRef modPath)
     473             : {
     474        5724 :     auto it = module_cache->find(modPath);
     475        5724 :     if (it == module_cache->end())
     476             :     {
     477        6128 :         it = module_cache->insert(
     478        6128 :             std::make_pair(modPath, load_module(lc, modPath))).first;
     479             :     }
     480             : 
     481        5724 :     return it->second.get();
     482             : }
     483             : 
     484             : static std::unique_ptr<llvm::Module>
     485        3064 : load_module(LLVMContextRef lc, llvm::StringRef Identifier)
     486             : {
     487             :     LLVMMemoryBufferRef buf;
     488             :     LLVMModuleRef mod;
     489             :     char path[MAXPGPATH];
     490             :     char *msg;
     491             : 
     492        3064 :     snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
     493             : 
     494        3064 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
     495           0 :         elog(FATAL, "failed to open bitcode file \"%s\": %s",
     496             :              path, msg);
     497        3064 :     if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
     498           0 :         elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
     499             : 
     500             :     /*
     501             :      * Currently there's no use in more detailed debug info for JITed
     502             :      * code. Until that changes, not much point in wasting memory and cycles
     503             :      * on processing debuginfo.
     504             :      */
     505        3064 :     llvm::StripDebugInfo(*llvm::unwrap(mod));
     506             : 
     507        6128 :     return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
     508             : }
     509             : 
     510             : /*
     511             :  * Compute list of referenced variables, functions and the instruction count
     512             :  * for a function.
     513             :  */
     514             : static void
     515        9302 : function_references(llvm::Function &F,
     516             :                     int &running_instcount,
     517             :                     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     518             :                     llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
     519             : {
     520        9302 :     llvm::SmallPtrSet<const llvm::User *, 32> Visited;
     521             : 
     522      213578 :     for (llvm::BasicBlock &BB : F)
     523             :     {
     524     1114842 :         for (llvm::Instruction &I : BB)
     525             :         {
     526      506352 :             if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
     527           0 :                 continue;
     528             : 
     529      506352 :             llvm::SmallVector<llvm::User *, 8> Worklist;
     530      506352 :             Worklist.push_back(&I);
     531             : 
     532      506352 :             running_instcount++;
     533             : 
     534     1743700 :             while (!Worklist.empty()) {
     535     1237348 :                 llvm::User *U = Worklist.pop_back_val();
     536             : 
     537             :                 /* visited before */
     538     1237348 :                 if (!Visited.insert(U).second)
     539      610730 :                     continue;
     540             : 
     541     1665972 :                 for (auto &OI : U->operands()) {
     542     1039354 :                     llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
     543     1039354 :                     if (!Operand)
     544      344556 :                         continue;
     545      823566 :                     if (llvm::isa<llvm::BlockAddress>(Operand))
     546           0 :                         continue;
     547      823566 :                     if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
     548       37764 :                         referencedVars.insert(GV);
     549       37764 :                         if (GV->hasInitializer())
     550       36198 :                             Worklist.push_back(GV->getInitializer());
     551       37764 :                         continue;
     552             :                     }
     553      785802 :                     if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
     554       91004 :                         referencedFunctions.insert(CF);
     555       91004 :                         continue;
     556             :                     }
     557      694798 :                     Worklist.push_back(Operand);
     558             :                 }
     559             :             }
     560      506352 :         }
     561             :     }
     562        9302 : }
     563             : 
     564             : /*
     565             :  * Check whether function F is inlinable and, if so, what globals need to be
     566             :  * imported.
     567             :  *
     568             :  * References to external functions from, potentially recursively, inlined
     569             :  * functions are added to the passed in worklist.
     570             :  */
     571             : static bool
     572        9350 : function_inlinable(llvm::Function &F,
     573             :                    int threshold,
     574             :                    FunctionInlineStates &functionStates,
     575             :                    InlineWorkList &worklist,
     576             :                    InlineSearchPath &searchpath,
     577             :                    llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     578             :                    int &running_instcount,
     579             :                    llvm::StringSet<> &importVars)
     580             : {
     581        9350 :     int subThreshold = threshold * inline_cost_decay_factor;
     582        9350 :     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
     583        9350 :     llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
     584             : 
     585             :     /* can't rely on what may be inlined */
     586        9350 :     if (F.isInterposable())
     587           0 :         return false;
     588             : 
     589             :     /*
     590             :      * Can't rely on function being present. Alternatively we could create a
     591             :      * static version of these functions?
     592             :      */
     593        9350 :     if (F.hasAvailableExternallyLinkage())
     594           0 :         return false;
     595             : 
     596             :     ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
     597             : 
     598        9350 :     if (F.materialize())
     599           0 :         elog(FATAL, "failed to materialize metadata");
     600             : 
     601        9350 :     if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
     602             :     {
     603             :         ilog(DEBUG1, "ineligibile to import %s due to noinline",
     604             :              F.getName().data());
     605          48 :         return false;
     606             :     }
     607             : 
     608        9302 :     function_references(F, running_instcount, referencedVars, referencedFunctions);
     609             : 
     610       34194 :     for (llvm::GlobalVariable* rv: referencedVars)
     611             :     {
     612       25554 :         if (rv->materialize())
     613           0 :             elog(FATAL, "failed to materialize metadata");
     614             : 
     615             :         /*
     616             :          * Don't inline functions that access thread local variables.  That
     617             :          * doesn't work on current LLVM releases (but might in future).
     618             :          */
     619       25554 :         if (rv->isThreadLocal())
     620             :         {
     621             :             ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
     622             :                  F.getName().data(), rv->getName().data());
     623         662 :             return false;
     624             :         }
     625             : 
     626             :         /*
     627             :          * Never want to inline externally visible vars, cheap enough to
     628             :          * reference.
     629             :          */
     630       25554 :         if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
     631         654 :             continue;
     632             : 
     633             :         /*
     634             :          * If variable is file-local, we need to inline it, to be able to
     635             :          * inline the function itself. Can't do that if the variable can be
     636             :          * modified, because they'd obviously get out of sync.
     637             :          *
     638             :          * XXX: Currently not a problem, but there'd be problems with
     639             :          * nontrivial initializers if they were allowed for postgres.
     640             :          */
     641       24900 :         if (!rv->isConstant())
     642             :         {
     643             :             ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
     644             :                  F.getName().data(), rv->getName().data());
     645         662 :             return false;
     646             :         }
     647             : 
     648             :         ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
     649             :              rv->getName().data(), (int)rv->getLinkage());
     650             : 
     651       24238 :         importVars.insert(rv->getName());
     652             :         /* small cost attributed to each cloned global */
     653       24238 :         running_instcount += 5;
     654             :     }
     655             : 
     656        8640 :     visitedFunctions.insert(&F);
     657             : 
     658             :     /*
     659             :      * Check referenced functions. Check whether used static ones are
     660             :      * inlinable, and remember external ones for inlining.
     661             :      */
     662       52634 :     for (llvm::Function* referencedFunction: referencedFunctions)
     663             :     {
     664       44122 :         llvm::StringSet<> recImportVars;
     665             : 
     666       44122 :         if (referencedFunction->materialize())
     667           0 :             elog(FATAL, "failed to materialize metadata");
     668             : 
     669       44122 :         if (referencedFunction->isIntrinsic())
     670        6064 :             continue;
     671             : 
     672             :         /* if already visited skip, otherwise remember */
     673       38058 :         if (!visitedFunctions.insert(referencedFunction).second)
     674       16110 :             continue;
     675             : 
     676             :         /*
     677             :          * We don't inline external functions directly here, instead we put
     678             :          * them on the worklist if appropriate and check them from
     679             :          * llvm_build_inline_plan().
     680             :          */
     681       21948 :         if (referencedFunction->hasExternalLinkage())
     682             :         {
     683       18322 :             llvm::StringRef funcName = referencedFunction->getName();
     684             : 
     685             :             /*
     686             :              * Don't bother checking for inlining if remaining cost budget is
     687             :              * very small.
     688             :              */
     689       18322 :             if (subThreshold < 5)
     690        4530 :                 continue;
     691             : 
     692       13792 :             auto it = functionStates.find(funcName);
     693       13792 :             if (it == functionStates.end())
     694             :             {
     695             :                 FunctionInlineState inlineState;
     696             : 
     697        7512 :                 inlineState.costLimit = subThreshold;
     698        7512 :                 inlineState.processed = false;
     699        7512 :                 inlineState.inlined = false;
     700        7512 :                 inlineState.allowReconsidering = false;
     701             : 
     702        7512 :                 functionStates[funcName] = inlineState;
     703        7512 :                 worklist.push_back({funcName, searchpath});
     704             : 
     705             :                 ilog(DEBUG1,
     706             :                      "considering extern function %s at %d for inlining",
     707             :                      funcName.data(), subThreshold);
     708             :             }
     709        6280 :             else if (!it->second.inlined &&
     710       10708 :                      (!it->second.processed || it->second.allowReconsidering) &&
     711       10708 :                      it->second.costLimit < subThreshold)
     712             :             {
     713             :                 /*
     714             :                  * Update inlining threshold if higher. Need to re-queue
     715             :                  * to be processed if already processed with lower
     716             :                  * threshold.
     717             :                  */
     718         370 :                 if (it->second.processed)
     719             :                 {
     720             :                     ilog(DEBUG1,
     721             :                          "reconsidering extern function %s at %d for inlining, increasing from %d",
     722             :                          funcName.data(), subThreshold, it->second.costLimit);
     723             : 
     724         364 :                     it->second.processed = false;
     725         364 :                     it->second.allowReconsidering = false;
     726         364 :                     worklist.push_back({funcName, searchpath});
     727             :                 }
     728         370 :                 it->second.costLimit = subThreshold;
     729             :             }
     730       13792 :             continue;
     731       13792 :         }
     732             : 
     733             :         /* can't rely on what may be inlined */
     734        3626 :         if (referencedFunction->isInterposable())
     735           0 :             return false;
     736             : 
     737        3626 :         if (!function_inlinable(*referencedFunction,
     738             :                                 subThreshold,
     739             :                                 functionStates,
     740             :                                 worklist,
     741             :                                 searchpath,
     742             :                                 visitedFunctions,
     743             :                                 running_instcount,
     744             :                                 recImportVars))
     745             :         {
     746             :             ilog(DEBUG1,
     747             :                  "cannot inline %s due to required function %s not being inlinable",
     748             :                  F.getName().data(), referencedFunction->getName().data());
     749         128 :             return false;
     750             :         }
     751             : 
     752             :         /* import referenced function itself */
     753        3498 :         importVars.insert(referencedFunction->getName());
     754             : 
     755             :         /* import referenced function and its dependents */
     756       75782 :         for (auto& recImportVar : recImportVars)
     757       72284 :             importVars.insert(recImportVar.first());
     758       44122 :     }
     759             : 
     760        8512 :     return true;
     761       17226 : }
     762             : 
     763             : /*
     764             :  * Attempt to load module summary located at path. Return empty pointer when
     765             :  * loading fails.
     766             :  */
     767             : static std::unique_ptr<llvm::ModuleSummaryIndex>
     768         382 : llvm_load_summary(llvm::StringRef path)
     769             : {
     770             :     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
     771         382 :         llvm::MemoryBuffer::getFile(path);
     772             : 
     773         382 :     if (std::error_code EC = MBOrErr.getError())
     774             :     {
     775             :         ilog(DEBUG1, "failed to open %s: %s", path.data(),
     776             :              EC.message().c_str());
     777             :     }
     778             :     else
     779             :     {
     780         382 :         llvm::MemoryBufferRef ref(*MBOrErr.get().get());
     781             : 
     782             :         llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
     783         382 :             llvm::getModuleSummaryIndex(ref);
     784         382 :         if (IndexOrErr)
     785         382 :             return std::move(IndexOrErr.get());
     786           0 :         elog(FATAL, "failed to load summary \"%s\": %s",
     787             :              path.data(),
     788             :              toString(IndexOrErr.takeError()).c_str());
     789         382 :     }
     790           0 :     return nullptr;
     791         382 : }
     792             : 
     793             : /*
     794             :  * Attempt to add modpath to the search path.
     795             :  */
     796             : static void
     797         542 : add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
     798             : {
     799             :     /* only extension in libdir are candidates for inlining for now */
     800             : #if LLVM_VERSION_MAJOR < 16
     801             : #define starts_with startswith
     802             : #endif
     803         542 :     if (!modpath.starts_with("$libdir/"))
     804           0 :         return;
     805             : 
     806             :     /* if there's no match, attempt to load */
     807         542 :     auto it = summary_cache->find(modpath);
     808         542 :     if (it == summary_cache->end())
     809             :     {
     810         382 :         std::string path(modpath);
     811         382 :         path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
     812         382 :         path += ".index.bc";
     813         382 :         (*summary_cache)[modpath] = llvm_load_summary(path);
     814         382 :         it = summary_cache->find(modpath);
     815         382 :     }
     816             : 
     817             :     Assert(it != summary_cache->end());
     818             : 
     819             :     /* if the entry isn't NULL, it's validly loaded */
     820         542 :     if (it->second)
     821         542 :         searchpath.push_back(it->second.get());
     822             : }
     823             : 
     824             : /*
     825             :  * Search for all references for functions hashing to guid in the search path,
     826             :  * and return them in search path order.
     827             :  */
     828             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     829       10368 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
     830             : {
     831       10368 :     llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
     832             : 
     833       20736 :     for (auto index : path)
     834             :     {
     835       10368 :         llvm::ValueInfo funcVI = index->getValueInfo(guid);
     836             : 
     837             :         /* if index doesn't know function, we don't have a body, continue */
     838       10368 :         if (funcVI)
     839       19336 :             for (auto &gv : funcVI.getSummaryList())
     840        9668 :                 matches.push_back(gv.get());
     841             :     }
     842             : 
     843       10368 :     return matches;
     844           0 : }
     845             : 
     846             : /*
     847             :  * Create inline wrapper with the name Name, redirecting the call to F.
     848             :  */
     849             : static llvm::Function*
     850           0 : create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     851             :                             llvm::Function *F,
     852             :                             llvm::StringRef Name)
     853             : {
     854             :     typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     855             : 
     856           0 :     llvm::LLVMContext &Context = F->getContext();
     857           0 :     llvm::IRBuilder<> Builder(Context);
     858             :     llvm::Function *AF;
     859             :     llvm::BasicBlock *BB;
     860             :     llvm::CallInst *fwdcall;
     861             : 
     862           0 :     AF = llvm::Function::Create(F->getFunctionType(),
     863             :                                 LinkageTypes::AvailableExternallyLinkage,
     864             :                                 Name, importMod.get());
     865           0 :     BB = llvm::BasicBlock::Create(Context, "entry", AF);
     866             : 
     867           0 :     Builder.SetInsertPoint(BB);
     868           0 :     fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
     869           0 :     fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
     870           0 :     Builder.CreateRet(fwdcall);
     871             : 
     872           0 :     return AF;
     873           0 : }

Generated by: LCOV version 1.16