LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit_inline.cpp (source / functions) Hit Total Coverage
Test: PostgreSQL 18devel Lines: 227 253 89.7 %
Date: 2024-07-18 17:11:54 Functions: 12 13 92.3 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit_inline.cpp
       4             :  *    Cross module inlining suitable for postgres' JIT
       5             :  *
       6             :  * The inliner iterates over external functions referenced from the passed
       7             :  * module and attempts to inline those.  It does so by utilizing pre-built
       8             :  * indexes over both postgres core code and extension modules.  When a match
       9             :  * for an external function is found - not guaranteed! - the index will then
      10             :  * be used to judge their instruction count / inline worthiness. After doing
      11             :  * so for all external functions, all the referenced functions (and
      12             :  * prerequisites) will be imported.
      13             :  *
      14             :  * Copyright (c) 2016-2024, PostgreSQL Global Development Group
      15             :  *
      16             :  * IDENTIFICATION
      17             :  *    src/backend/lib/llvmjit/llvmjit_inline.cpp
      18             :  *
      19             :  *-------------------------------------------------------------------------
      20             :  */
      21             : 
      22             : extern "C"
      23             : {
      24             : #include "postgres.h"
      25             : }
      26             : 
      27             : #include "jit/llvmjit.h"
      28             : 
      29             : extern "C"
      30             : {
      31             : #include <fcntl.h>
      32             : #include <sys/mman.h>
      33             : #include <sys/stat.h>
      34             : #include <sys/types.h>
      35             : #include <unistd.h>
      36             : 
      37             : #include "common/string.h"
      38             : #include "miscadmin.h"
      39             : #include "storage/fd.h"
      40             : }
      41             : 
      42             : #include <llvm-c/Core.h>
      43             : #include <llvm-c/BitReader.h>
      44             : 
      45             : /* Avoid macro clash with LLVM's C++ headers */
      46             : #undef Min
      47             : 
      48             : #include <llvm/ADT/SetVector.h>
      49             : #include <llvm/ADT/StringSet.h>
      50             : #include <llvm/ADT/StringMap.h>
      51             : #include <llvm/Analysis/ModuleSummaryAnalysis.h>
      52             : #include <llvm/Bitcode/BitcodeReader.h>
      53             : #include <llvm/IR/Attributes.h>
      54             : #include <llvm/IR/DebugInfo.h>
      55             : #include <llvm/IR/IntrinsicInst.h>
      56             : #include <llvm/IR/IRBuilder.h>
      57             : #include <llvm/IR/ModuleSummaryIndex.h>
      58             : #include <llvm/Linker/IRMover.h>
      59             : #include <llvm/Support/ManagedStatic.h>
      60             : #include <llvm/Support/MemoryBuffer.h>
      61             : 
      62             : 
      63             : /*
      64             :  * Type used to represent modules InlineWorkListItem's subject is searched for
      65             :  * in.
      66             :  */
      67             : typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
      68             : 
      69             : /*
      70             :  * Item in queue of to-be-checked symbols and corresponding queue.
      71             :  */
      72             : typedef struct InlineWorkListItem
      73             : {
      74             :     llvm::StringRef symbolName;
      75             :     llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
      76             : } InlineWorkListItem;
      77             : typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
      78             : 
      79             : /*
      80             :  * Information about symbols processed during inlining. Used to prevent
      81             :  * repeated searches and provide additional information.
      82             :  */
      83             : typedef struct FunctionInlineState
      84             : {
      85             :     int costLimit;
      86             :     bool processed;
      87             :     bool inlined;
      88             :     bool allowReconsidering;
      89             : } FunctionInlineState;
      90             : typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
      91             : 
      92             : /*
      93             :  * Map of modules that should be inlined, with a list of the to-be inlined
      94             :  * symbols.
      95             :  */
      96             : typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
      97             : 
      98             : 
      99             : const float inline_cost_decay_factor = 0.5;
     100             : const int inline_initial_cost = 150;
     101             : 
     102             : /*
     103             :  * These are managed statics so LLVM knows to deallocate them during an
     104             :  * LLVMShutdown(), rather than after (which'd cause crashes).
     105             :  */
     106             : typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
     107             : llvm::ManagedStatic<ModuleCache> module_cache;
     108             : typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
     109             : llvm::ManagedStatic<SummaryCache> summary_cache;
     110             : 
     111             : 
     112             : static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod);
     113             : static void llvm_execute_inline_plan(llvm::Module *mod,
     114             :                                      ImportMapTy *globalsToInline);
     115             : 
     116             : static llvm::Module* load_module_cached(LLVMContextRef c, llvm::StringRef modPath);
     117             : static std::unique_ptr<llvm::Module> load_module(LLVMContextRef c, llvm::StringRef Identifier);
     118             : static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
     119             : 
     120             : 
     121             : static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     122             :                                                    llvm::Function *F,
     123             :                                                    llvm::StringRef Name);
     124             : 
     125             : static bool function_inlinable(llvm::Function &F,
     126             :                                int threshold,
     127             :                                FunctionInlineStates &functionState,
     128             :                                InlineWorkList &worklist,
     129             :                                InlineSearchPath &searchpath,
     130             :                                llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     131             :                                int &running_instcount,
     132             :                                llvm::StringSet<> &importVars);
     133             : static void function_references(llvm::Function &F,
     134             :                                 int &running_instcount,
     135             :                                 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     136             :                                 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
     137             : 
     138             : static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
     139             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     140             : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
     141             : 
     142             : /* verbose debugging for inliner development */
     143             : /* #define INLINE_DEBUG */
     144             : #ifdef INLINE_DEBUG
     145             : #define ilog        elog
     146             : #else
     147             : #define ilog(...)   (void) 0
     148             : #endif
     149             : 
     150             : /*
     151             :  * Reset inlining related state. This needs to be called before the currently
     152             :  * used LLVMContextRef is disposed (and a new one create), otherwise we would
     153             :  * have dangling references to deleted modules.
     154             :  */
     155             : void
     156           0 : llvm_inline_reset_caches(void)
     157             : {
     158           0 :     module_cache->clear();
     159           0 :     summary_cache->clear();
     160           0 : }
     161             : 
     162             : /*
     163             :  * Perform inlining of external function references in M based on a simple
     164             :  * cost based analysis.
     165             :  */
     166             : void
     167         998 : llvm_inline(LLVMModuleRef M)
     168             : {
     169         998 :     LLVMContextRef lc = LLVMGetModuleContext(M);
     170         998 :     llvm::Module *mod = llvm::unwrap(M);
     171             : 
     172         998 :     std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
     173         998 :     if (!globalsToInline)
     174           0 :         return;
     175         998 :     llvm_execute_inline_plan(mod, globalsToInline.get());
     176             : }
     177             : 
     178             : /*
     179             :  * Build information necessary for inlining external function references in
     180             :  * mod.
     181             :  */
     182             : static std::unique_ptr<ImportMapTy>
     183         998 : llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
     184             : {
     185        1996 :     std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
     186        1996 :     FunctionInlineStates functionStates;
     187        1996 :     InlineWorkList worklist;
     188             : 
     189        1996 :     InlineSearchPath defaultSearchPath;
     190             : 
     191             :     /* attempt to add module to search path */
     192         998 :     add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
     193             :     /* if postgres isn't available, no point continuing */
     194         998 :     if (defaultSearchPath.empty())
     195           0 :         return nullptr;
     196             : 
     197             :     /*
     198             :      * Start inlining with current references to external functions by putting
     199             :      * them on the inlining worklist. If, during inlining of those, new extern
     200             :      * functions need to be inlined, they'll also be put there, with a lower
     201             :      * priority.
     202             :      */
     203       15048 :     for (const llvm::Function &funcDecl : mod->functions())
     204             :     {
     205       14050 :         InlineWorkListItem item = {};
     206       14050 :         FunctionInlineState inlineState = {};
     207             : 
     208             :         /* already has a definition */
     209       14050 :         if (!funcDecl.isDeclaration())
     210        9534 :             continue;
     211             : 
     212             :         /* llvm provides implementation */
     213        4516 :         if (funcDecl.isIntrinsic())
     214         814 :             continue;
     215             : 
     216        3702 :         item.symbolName = funcDecl.getName();
     217        3702 :         item.searchpath = defaultSearchPath;
     218        3702 :         worklist.push_back(item);
     219        3702 :         inlineState.costLimit = inline_initial_cost;
     220        3702 :         inlineState.processed = false;
     221        3702 :         inlineState.inlined = false;
     222        3702 :         inlineState.allowReconsidering = false;
     223        3702 :         functionStates[funcDecl.getName()] = inlineState;
     224             :     }
     225             : 
     226             :     /*
     227             :      * Iterate over pending worklist items, look them up in index, check
     228             :      * whether they should be inlined.
     229             :      */
     230       19156 :     while (!worklist.empty())
     231             :     {
     232       18158 :         InlineWorkListItem item = worklist.pop_back_val();
     233       18158 :         llvm::StringRef symbolName = item.symbolName;
     234             :         char *cmodname;
     235             :         char *cfuncname;
     236       18158 :         FunctionInlineState &inlineState = functionStates[symbolName];
     237             :         llvm::GlobalValue::GUID funcGUID;
     238             : 
     239       18158 :         llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
     240             : 
     241       18158 :         funcGUID = llvm::GlobalValue::getGUID(cfuncname);
     242             : 
     243             :         /* already processed */
     244       18158 :         if (inlineState.processed)
     245           0 :             continue;
     246             : 
     247             : 
     248       18158 :         if (cmodname)
     249          20 :             add_module_to_inline_search_path(item.searchpath, cmodname);
     250             : 
     251             :         /*
     252             :          * Iterate over all known definitions of function, via the index. Then
     253             :          * look up module(s), check if function actually is defined (there
     254             :          * could be hash conflicts).
     255             :          */
     256       35544 :         for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
     257             :         {
     258             :             const llvm::FunctionSummary *fs;
     259       17386 :             llvm::StringRef modPath = gvs->modulePath();
     260             :             llvm::Module *defMod;
     261             :             llvm::Function *funcDef;
     262             : 
     263       17386 :             fs = llvm::cast<llvm::FunctionSummary>(gvs);
     264             : 
     265       17386 :             if (gvs->notEligibleToImport())
     266             :             {
     267             :                 ilog(DEBUG1, "ineligibile to import %s due to summary",
     268             :                      symbolName.data());
     269        8554 :                 continue;
     270             :             }
     271             : 
     272       17386 :             if ((int) fs->instCount() > inlineState.costLimit)
     273             :             {
     274             :                 ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
     275             :                      symbolName.data(), fs->instCount(), inlineState.costLimit);
     276        7522 :                 inlineState.allowReconsidering = true;
     277        7522 :                 continue;
     278             :             }
     279             : 
     280        9864 :             defMod = load_module_cached(lc, modPath);
     281        9864 :             if (defMod->materializeMetadata())
     282           0 :                 elog(FATAL, "failed to materialize metadata");
     283             : 
     284        9864 :             funcDef = defMod->getFunction(cfuncname);
     285             : 
     286             :             /*
     287             :              * This can happen e.g. in case of a hash collision of the
     288             :              * function's name.
     289             :              */
     290        9864 :             if (!funcDef)
     291           0 :                 continue;
     292             : 
     293        9864 :             if (funcDef->materialize())
     294           0 :                 elog(FATAL, "failed to materialize metadata");
     295             : 
     296             :             Assert(!funcDef->isDeclaration());
     297             :             Assert(funcDef->hasExternalLinkage());
     298             : 
     299        9864 :             llvm::StringSet<> importVars;
     300        9864 :             llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
     301        9864 :             int running_instcount = 0;
     302             : 
     303             :             /*
     304             :              * Check whether function, and objects it depends on, are
     305             :              * inlinable.
     306             :              */
     307        9864 :             if (function_inlinable(*funcDef,
     308             :                                    inlineState.costLimit,
     309             :                                    functionStates,
     310             :                                    worklist,
     311             :                                    item.searchpath,
     312             :                                    visitedFunctions,
     313             :                                    running_instcount,
     314             :                                    importVars))
     315             :             {
     316             :                 /*
     317             :                  * Check whether function and all its dependencies are too
     318             :                  * big. Dependencies already counted for other functions that
     319             :                  * will get inlined are not counted again. While this make
     320             :                  * things somewhat order dependent, I can't quite see a point
     321             :                  * in a different behaviour.
     322             :                  */
     323        8226 :                 if (running_instcount > inlineState.costLimit)
     324             :                 {
     325             :                     ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
     326             :                          symbolName.data(), running_instcount, inlineState.costLimit);
     327        1032 :                     inlineState.allowReconsidering = true;
     328        1032 :                     continue;
     329             :                 }
     330             : 
     331             :                 ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
     332             :                      symbolName.data(), running_instcount, fs->instCount());
     333             : 
     334             :                 /* import referenced function itself */
     335        7194 :                 importVars.insert(symbolName);
     336             : 
     337             :                 {
     338        7194 :                     llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
     339       18618 :                     for (auto& importVar : importVars)
     340       11424 :                         modGlobalsToInline.insert(importVar.first());
     341             :                     Assert(modGlobalsToInline.size() > 0);
     342             :                 }
     343             : 
     344             :                 /* mark function as inlined */
     345        7194 :                 inlineState.inlined = true;
     346             : 
     347             :                 /*
     348             :                  * Found definition to inline, don't look for further
     349             :                  * potential definitions.
     350             :                  */
     351        7194 :                 break;
     352             :             }
     353             :             else
     354             :             {
     355             :                 ilog(DEBUG1, "had to skip inlining %s",
     356             :                      symbolName.data());
     357             : 
     358             :                 /* It's possible there's another definition that's inlinable. */
     359             :             }
     360             :         }
     361             : 
     362             :         /*
     363             :          * Signal that we're done with symbol, whether successful (inlined =
     364             :          * true above) or not.
     365             :          */
     366       18158 :         inlineState.processed = true;
     367             :     }
     368             : 
     369         998 :     return globalsToInline;
     370             : }
     371             : 
     372             : /*
     373             :  * Perform the actual inlining of external functions (and their dependencies)
     374             :  * into mod.
     375             :  */
     376             : static void
     377         998 : llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
     378             : {
     379        1996 :     llvm::IRMover Mover(*mod);
     380             : 
     381        5852 :     for (const auto& toInline : *globalsToInline)
     382             :     {
     383        4854 :         const llvm::StringRef& modPath = toInline.first();
     384        4854 :         const llvm::StringSet<>& modGlobalsToInline = toInline.second;
     385        4854 :         llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
     386             : 
     387             :         Assert(module_cache->count(modPath));
     388        4854 :         std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
     389        4854 :         module_cache->erase(modPath);
     390             : 
     391        4854 :         if (modGlobalsToInline.empty())
     392           0 :             continue;
     393             : 
     394       15610 :         for (auto &glob: modGlobalsToInline)
     395             :         {
     396       10756 :             llvm::StringRef SymbolName = glob.first();
     397             :             char *modname;
     398             :             char *funcname;
     399             : 
     400       10756 :             llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
     401             : 
     402       10756 :             llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
     403             : 
     404       10756 :             if (!valueToImport)
     405           0 :                 elog(FATAL, "didn't refind value %s to import", SymbolName.data());
     406             : 
     407             :             /*
     408             :              * For functions (global vars are only inlined if already static),
     409             :              * mark imported variables as being clones from other
     410             :              * functions. That a) avoids symbol conflicts b) allows the
     411             :              * optimizer to perform inlining.
     412             :             */
     413       10756 :             if (llvm::isa<llvm::Function>(valueToImport))
     414             :             {
     415        7258 :                 llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
     416             :                 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     417             : 
     418             :                 /*
     419             :                  * Per-function info isn't necessarily stripped yet, as the
     420             :                  * module is lazy-loaded when stripped above.
     421             :                  */
     422        7258 :                 llvm::stripDebugInfo(*F);
     423             : 
     424             :                 /*
     425             :                  * If the to-be-imported function is one referenced including
     426             :                  * its module name, create a tiny inline function that just
     427             :                  * forwards the call. One might think a GlobalAlias would do
     428             :                  * the trick, but a) IRMover doesn't override a declaration
     429             :                  * with an alias pointing to a definition (instead renaming
     430             :                  * it), b) Aliases can't be AvailableExternally.
     431             :                  */
     432        7258 :                 if (modname)
     433             :                 {
     434             :                     llvm::Function *AF;
     435             : 
     436          18 :                     AF = create_redirection_function(importMod, F, SymbolName);
     437             : 
     438          18 :                     GlobalsToImport.insert(AF);
     439          18 :                     llvm::stripDebugInfo(*AF);
     440             :                 }
     441             : 
     442        7258 :                 if (valueToImport->hasExternalLinkage())
     443             :                 {
     444        7194 :                     valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
     445             :                 }
     446             :             }
     447             : 
     448       10756 :             GlobalsToImport.insert(valueToImport);
     449             :             ilog(DEBUG1, "performing import of %s %s",
     450             :                  modPath.data(), SymbolName.data());
     451             : 
     452             :         }
     453             : 
     454        4854 :         if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
     455        2052 :                        [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {},
     456        9708 :                        /*IsPerformingImport=*/false))
     457           0 :             elog(FATAL, "function import failed with linker error");
     458             :     }
     459         998 : }
     460             : 
     461             : /*
     462             :  * Return a module identified by modPath, caching it in memory.
     463             :  *
     464             :  * Note that such a module may *not* be modified without copying, otherwise
     465             :  * the cache state would get corrupted.
     466             :  */
     467             : static llvm::Module*
     468        9864 : load_module_cached(LLVMContextRef lc, llvm::StringRef modPath)
     469             : {
     470        9864 :     auto it = module_cache->find(modPath);
     471        9864 :     if (it == module_cache->end())
     472             :     {
     473        5048 :         it = module_cache->insert(
     474        5048 :             std::make_pair(modPath, load_module(lc, modPath))).first;
     475             :     }
     476             : 
     477       19728 :     return it->second.get();
     478             : }
     479             : 
     480             : static std::unique_ptr<llvm::Module>
     481        5048 : load_module(LLVMContextRef lc, llvm::StringRef Identifier)
     482             : {
     483             :     LLVMMemoryBufferRef buf;
     484             :     LLVMModuleRef mod;
     485             :     char path[MAXPGPATH];
     486             :     char *msg;
     487             : 
     488        5048 :     snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
     489             : 
     490        5048 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
     491           0 :         elog(FATAL, "failed to open bitcode file \"%s\": %s",
     492             :              path, msg);
     493        5048 :     if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
     494           0 :         elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
     495             : 
     496             :     /*
     497             :      * Currently there's no use in more detailed debug info for JITed
     498             :      * code. Until that changes, not much point in wasting memory and cycles
     499             :      * on processing debuginfo.
     500             :      */
     501        5048 :     llvm::StripDebugInfo(*llvm::unwrap(mod));
     502             : 
     503       10096 :     return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
     504             : }
     505             : 
     506             : /*
     507             :  * Compute list of referenced variables, functions and the instruction count
     508             :  * for a function.
     509             :  */
     510             : static void
     511       14850 : function_references(llvm::Function &F,
     512             :                     int &running_instcount,
     513             :                     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     514             :                     llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
     515             : {
     516       29700 :     llvm::SmallPtrSet<const llvm::User *, 32> Visited;
     517             : 
     518      179230 :     for (llvm::BasicBlock &BB : F)
     519             :     {
     520     1047708 :         for (llvm::Instruction &I : BB)
     521             :         {
     522      883328 :             if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
     523           0 :                 continue;
     524             : 
     525     1766656 :             llvm::SmallVector<llvm::User *, 8> Worklist;
     526      883328 :             Worklist.push_back(&I);
     527             : 
     528      883328 :             running_instcount++;
     529             : 
     530     3270468 :             while (!Worklist.empty()) {
     531     2387140 :                 llvm::User *U = Worklist.pop_back_val();
     532             : 
     533             :                 /* visited before */
     534     2387140 :                 if (!Visited.insert(U).second)
     535     1245540 :                     continue;
     536             : 
     537     3153248 :                 for (auto &OI : U->operands()) {
     538     2011648 :                     llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
     539     2011648 :                     if (!Operand)
     540      562384 :                         continue;
     541     1661194 :                     if (llvm::isa<llvm::BlockAddress>(Operand))
     542           0 :                         continue;
     543     1661194 :                     if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
     544       56392 :                         referencedVars.insert(GV);
     545       56392 :                         if (GV->hasInitializer())
     546       54548 :                             Worklist.push_back(GV->getInitializer());
     547       56392 :                         continue;
     548             :                     }
     549     1604802 :                     if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
     550      155538 :                         referencedFunctions.insert(CF);
     551      155538 :                         continue;
     552             :                     }
     553     1449264 :                     Worklist.push_back(Operand);
     554             :                 }
     555             :             }
     556             :         }
     557             :     }
     558       14850 : }
     559             : 
     560             : /*
     561             :  * Check whether function F is inlinable and, if so, what globals need to be
     562             :  * imported.
     563             :  *
     564             :  * References to external functions from, potentially recursively, inlined
     565             :  * functions are added to the passed in worklist.
     566             :  */
     567             : static bool
     568       14944 : function_inlinable(llvm::Function &F,
     569             :                    int threshold,
     570             :                    FunctionInlineStates &functionStates,
     571             :                    InlineWorkList &worklist,
     572             :                    InlineSearchPath &searchpath,
     573             :                    llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     574             :                    int &running_instcount,
     575             :                    llvm::StringSet<> &importVars)
     576             : {
     577       14944 :     int subThreshold = threshold * inline_cost_decay_factor;
     578       29888 :     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
     579       29888 :     llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
     580             : 
     581             :     /* can't rely on what may be inlined */
     582       14944 :     if (F.isInterposable())
     583           0 :         return false;
     584             : 
     585             :     /*
     586             :      * Can't rely on function being present. Alternatively we could create a
     587             :      * static version of these functions?
     588             :      */
     589       14944 :     if (F.hasAvailableExternallyLinkage())
     590           0 :         return false;
     591             : 
     592             :     ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
     593             : 
     594       14944 :     if (F.materialize())
     595           0 :         elog(FATAL, "failed to materialize metadata");
     596             : 
     597             : #if LLVM_VERSION_MAJOR < 14
     598             : #define hasFnAttr hasFnAttribute
     599             : #endif
     600             : 
     601       14944 :     if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
     602             :     {
     603             :         ilog(DEBUG1, "ineligibile to import %s due to noinline",
     604             :              F.getName().data());
     605          94 :         return false;
     606             :     }
     607             : 
     608       14850 :     function_references(F, running_instcount, referencedVars, referencedFunctions);
     609             : 
     610       55660 :     for (llvm::GlobalVariable* rv: referencedVars)
     611             :     {
     612       42354 :         if (rv->materialize())
     613           0 :             elog(FATAL, "failed to materialize metadata");
     614             : 
     615             :         /*
     616             :          * Don't inline functions that access thread local variables.  That
     617             :          * doesn't work on current LLVM releases (but might in future).
     618             :          */
     619       42354 :         if (rv->isThreadLocal())
     620             :         {
     621             :             ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
     622             :                  F.getName().data(), rv->getName().data());
     623        1544 :             return false;
     624             :         }
     625             : 
     626             :         /*
     627             :          * Never want to inline externally visible vars, cheap enough to
     628             :          * reference.
     629             :          */
     630       42354 :         if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
     631         832 :             continue;
     632             : 
     633             :         /*
     634             :          * If variable is file-local, we need to inline it, to be able to
     635             :          * inline the function itself. Can't do that if the variable can be
     636             :          * modified, because they'd obviously get out of sync.
     637             :          *
     638             :          * XXX: Currently not a problem, but there'd be problems with
     639             :          * nontrivial initializers if they were allowed for postgres.
     640             :          */
     641       41522 :         if (!rv->isConstant())
     642             :         {
     643             :             ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
     644             :                  F.getName().data(), rv->getName().data());
     645        1544 :             return false;
     646             :         }
     647             : 
     648             :         ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
     649             :              rv->getName().data(), (int)rv->getLinkage());
     650             : 
     651       39978 :         importVars.insert(rv->getName());
     652             :         /* small cost attributed to each cloned global */
     653       39978 :         running_instcount += 5;
     654             :     }
     655             : 
     656       13306 :     visitedFunctions.insert(&F);
     657             : 
     658             :     /*
     659             :      * Check referenced functions. Check whether used static ones are
     660             :      * inlinable, and remember external ones for inlining.
     661             :      */
     662       83722 :     for (llvm::Function* referencedFunction: referencedFunctions)
     663             :     {
     664       70654 :         llvm::StringSet<> recImportVars;
     665             : 
     666       70654 :         if (referencedFunction->materialize())
     667           0 :             elog(FATAL, "failed to materialize metadata");
     668             : 
     669       70654 :         if (referencedFunction->isIntrinsic())
     670        9406 :             continue;
     671             : 
     672             :         /* if already visited skip, otherwise remember */
     673       61248 :         if (!visitedFunctions.insert(referencedFunction).second)
     674       22988 :             continue;
     675             : 
     676             :         /*
     677             :          * We don't inline external functions directly here, instead we put
     678             :          * them on the worklist if appropriate and check them from
     679             :          * llvm_build_inline_plan().
     680             :          */
     681       38260 :         if (referencedFunction->hasExternalLinkage())
     682             :         {
     683       33180 :             llvm::StringRef funcName = referencedFunction->getName();
     684             : 
     685             :             /*
     686             :              * Don't bother checking for inlining if remaining cost budget is
     687             :              * very small.
     688             :              */
     689       33180 :             if (subThreshold < 5)
     690        7012 :                 continue;
     691             : 
     692       26168 :             auto it = functionStates.find(funcName);
     693       26168 :             if (it == functionStates.end())
     694             :             {
     695             :                 FunctionInlineState inlineState;
     696             : 
     697       13910 :                 inlineState.costLimit = subThreshold;
     698       13910 :                 inlineState.processed = false;
     699       13910 :                 inlineState.inlined = false;
     700       13910 :                 inlineState.allowReconsidering = false;
     701             : 
     702       13910 :                 functionStates[funcName] = inlineState;
     703       13910 :                 worklist.push_back({funcName, searchpath});
     704             : 
     705             :                 ilog(DEBUG1,
     706             :                      "considering extern function %s at %d for inlining",
     707             :                      funcName.data(), subThreshold);
     708             :             }
     709       12258 :             else if (!it->second.inlined &&
     710       21060 :                      (!it->second.processed || it->second.allowReconsidering) &&
     711        8802 :                      it->second.costLimit < subThreshold)
     712             :             {
     713             :                 /*
     714             :                  * Update inlining threshold if higher. Need to re-queue
     715             :                  * to be processed if already processed with lower
     716             :                  * threshold.
     717             :                  */
     718         546 :                 if (it->second.processed)
     719             :                 {
     720             :                     ilog(DEBUG1,
     721             :                          "reconsidering extern function %s at %d for inlining, increasing from %d",
     722             :                          funcName.data(), subThreshold, it->second.costLimit);
     723             : 
     724         546 :                     it->second.processed = false;
     725         546 :                     it->second.allowReconsidering = false;
     726         546 :                     worklist.push_back({funcName, searchpath});
     727             :                 }
     728         546 :                 it->second.costLimit = subThreshold;
     729             :             }
     730       26168 :             continue;
     731             :         }
     732             : 
     733             :         /* can't rely on what may be inlined */
     734        5080 :         if (referencedFunction->isInterposable())
     735           0 :             return false;
     736             : 
     737        5080 :         if (!function_inlinable(*referencedFunction,
     738             :                                 subThreshold,
     739             :                                 functionStates,
     740             :                                 worklist,
     741             :                                 searchpath,
     742             :                                 visitedFunctions,
     743             :                                 running_instcount,
     744             :                                 recImportVars))
     745             :         {
     746             :             ilog(DEBUG1,
     747             :                  "cannot inline %s due to required function %s not being inlinable",
     748             :                  F.getName().data(), referencedFunction->getName().data());
     749         238 :             return false;
     750             :         }
     751             : 
     752             :         /* import referenced function itself */
     753        4842 :         importVars.insert(referencedFunction->getName());
     754             : 
     755             :         /* import referenced function and its dependents */
     756      102384 :         for (auto& recImportVar : recImportVars)
     757       97542 :             importVars.insert(recImportVar.first());
     758             :     }
     759             : 
     760       13068 :     return true;
     761             : }
     762             : 
     763             : /*
     764             :  * Attempt to load module summary located at path. Return empty pointer when
     765             :  * loading fails.
     766             :  */
     767             : static std::unique_ptr<llvm::ModuleSummaryIndex>
     768         504 : llvm_load_summary(llvm::StringRef path)
     769             : {
     770             :     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
     771        1008 :         llvm::MemoryBuffer::getFile(path);
     772             : 
     773         504 :     if (std::error_code EC = MBOrErr.getError())
     774             :     {
     775             :         ilog(DEBUG1, "failed to open %s: %s", path.data(),
     776             :              EC.message().c_str());
     777             :     }
     778             :     else
     779             :     {
     780         504 :         llvm::MemoryBufferRef ref(*MBOrErr.get().get());
     781             : 
     782             :         llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
     783        1008 :             llvm::getModuleSummaryIndex(ref);
     784         504 :         if (IndexOrErr)
     785         504 :             return std::move(IndexOrErr.get());
     786           0 :         elog(FATAL, "failed to load summary \"%s\": %s",
     787             :              path.data(),
     788             :              toString(IndexOrErr.takeError()).c_str());
     789             :     }
     790           0 :     return nullptr;
     791             : }
     792             : 
     793             : /*
     794             :  * Attempt to add modpath to the search path.
     795             :  */
     796             : static void
     797        1018 : add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
     798             : {
     799             :     /* only extension in libdir are candidates for inlining for now */
     800             : #if LLVM_VERSION_MAJOR < 16
     801             : #define starts_with startswith
     802             : #endif
     803        1018 :     if (!modpath.starts_with("$libdir/"))
     804           0 :         return;
     805             : 
     806             :     /* if there's no match, attempt to load */
     807        1018 :     auto it = summary_cache->find(modpath);
     808        1018 :     if (it == summary_cache->end())
     809             :     {
     810        1008 :         std::string path(modpath);
     811         504 :         path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
     812         504 :         path += ".index.bc";
     813         504 :         (*summary_cache)[modpath] = llvm_load_summary(path);
     814         504 :         it = summary_cache->find(modpath);
     815             :     }
     816             : 
     817             :     Assert(it != summary_cache->end());
     818             : 
     819             :     /* if the entry isn't NULL, it's validly loaded */
     820        1018 :     if (it->second)
     821        1018 :         searchpath.push_back(it->second.get());
     822             : }
     823             : 
     824             : /*
     825             :  * Search for all references for functions hashing to guid in the search path,
     826             :  * and return them in search path order.
     827             :  */
     828             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     829       18158 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
     830             : {
     831       18158 :     llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
     832             : 
     833       36608 :     for (auto index : path)
     834             :     {
     835       18450 :         llvm::ValueInfo funcVI = index->getValueInfo(guid);
     836             : 
     837             :         /* if index doesn't know function, we don't have a body, continue */
     838       18450 :         if (funcVI)
     839       34772 :             for (auto &gv : funcVI.getSummaryList())
     840       17386 :                 matches.push_back(gv.get());
     841             :     }
     842             : 
     843       18158 :     return matches;
     844             : }
     845             : 
     846             : /*
     847             :  * Create inline wrapper with the name Name, redirecting the call to F.
     848             :  */
     849             : static llvm::Function*
     850          18 : create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     851             :                             llvm::Function *F,
     852             :                             llvm::StringRef Name)
     853             : {
     854             :     typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     855             : 
     856          18 :     llvm::LLVMContext &Context = F->getContext();
     857          18 :     llvm::IRBuilder<> Builder(Context);
     858             :     llvm::Function *AF;
     859             :     llvm::BasicBlock *BB;
     860             :     llvm::CallInst *fwdcall;
     861             : #if LLVM_VERSION_MAJOR < 14
     862          18 :     llvm::Attribute inlineAttribute;
     863             : #endif
     864             : 
     865          36 :     AF = llvm::Function::Create(F->getFunctionType(),
     866             :                                 LinkageTypes::AvailableExternallyLinkage,
     867          18 :                                 Name, importMod.get());
     868          18 :     BB = llvm::BasicBlock::Create(Context, "entry", AF);
     869             : 
     870          18 :     Builder.SetInsertPoint(BB);
     871          18 :     fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
     872             : #if LLVM_VERSION_MAJOR < 14
     873             :     inlineAttribute = llvm::Attribute::get(Context,
     874          18 :                                            llvm::Attribute::AlwaysInline);
     875          18 :     fwdcall->addAttribute(~0U, inlineAttribute);
     876             : #else
     877             :     fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
     878             : #endif
     879          18 :     Builder.CreateRet(fwdcall);
     880             : 
     881          36 :     return AF;
     882             : }

Generated by: LCOV version 1.14