LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit_inline.cpp (source / functions) Hit Total Coverage
Test: PostgreSQL 18devel Lines: 210 251 83.7 %
Date: 2024-10-07 02:11:23 Functions: 11 13 84.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit_inline.cpp
       4             :  *    Cross module inlining suitable for postgres' JIT
       5             :  *
       6             :  * The inliner iterates over external functions referenced from the passed
       7             :  * module and attempts to inline those.  It does so by utilizing pre-built
       8             :  * indexes over both postgres core code and extension modules.  When a match
       9             :  * for an external function is found - not guaranteed! - the index will then
      10             :  * be used to judge their instruction count / inline worthiness. After doing
      11             :  * so for all external functions, all the referenced functions (and
      12             :  * prerequisites) will be imported.
      13             :  *
      14             :  * Copyright (c) 2016-2024, PostgreSQL Global Development Group
      15             :  *
      16             :  * IDENTIFICATION
      17             :  *    src/backend/lib/llvmjit/llvmjit_inline.cpp
      18             :  *
      19             :  *-------------------------------------------------------------------------
      20             :  */
      21             : 
      22             : extern "C"
      23             : {
      24             : #include "postgres.h"
      25             : }
      26             : 
      27             : #include "jit/llvmjit.h"
      28             : 
      29             : extern "C"
      30             : {
      31             : #include <fcntl.h>
      32             : #include <sys/mman.h>
      33             : #include <sys/stat.h>
      34             : #include <sys/types.h>
      35             : #include <unistd.h>
      36             : 
      37             : #include "common/string.h"
      38             : #include "miscadmin.h"
      39             : #include "storage/fd.h"
      40             : }
      41             : 
      42             : #include <llvm-c/Core.h>
      43             : #include <llvm-c/BitReader.h>
      44             : 
      45             : /* Avoid macro clash with LLVM's C++ headers */
      46             : #undef Min
      47             : 
      48             : #include <llvm/ADT/SetVector.h>
      49             : #include <llvm/ADT/StringSet.h>
      50             : #include <llvm/ADT/StringMap.h>
      51             : #include <llvm/Analysis/ModuleSummaryAnalysis.h>
      52             : #include <llvm/Bitcode/BitcodeReader.h>
      53             : #include <llvm/IR/Attributes.h>
      54             : #include <llvm/IR/DebugInfo.h>
      55             : #include <llvm/IR/IntrinsicInst.h>
      56             : #include <llvm/IR/IRBuilder.h>
      57             : #include <llvm/IR/ModuleSummaryIndex.h>
      58             : #include <llvm/Linker/IRMover.h>
      59             : #include <llvm/Support/ManagedStatic.h>
      60             : #include <llvm/Support/MemoryBuffer.h>
      61             : 
      62             : 
      63             : /*
      64             :  * Type used to represent modules InlineWorkListItem's subject is searched for
      65             :  * in.
      66             :  */
      67             : typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
      68             : 
      69             : /*
      70             :  * Item in queue of to-be-checked symbols and corresponding queue.
      71             :  */
      72             : typedef struct InlineWorkListItem
      73             : {
      74             :     llvm::StringRef symbolName;
      75             :     llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
      76             : } InlineWorkListItem;
      77             : typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
      78             : 
      79             : /*
      80             :  * Information about symbols processed during inlining. Used to prevent
      81             :  * repeated searches and provide additional information.
      82             :  */
      83             : typedef struct FunctionInlineState
      84             : {
      85             :     int costLimit;
      86             :     bool processed;
      87             :     bool inlined;
      88             :     bool allowReconsidering;
      89             : } FunctionInlineState;
      90             : typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
      91             : 
      92             : /*
      93             :  * Map of modules that should be inlined, with a list of the to-be inlined
      94             :  * symbols.
      95             :  */
      96             : typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
      97             : 
      98             : 
      99             : const float inline_cost_decay_factor = 0.5;
     100             : const int inline_initial_cost = 150;
     101             : 
     102             : /*
     103             :  * These are managed statics so LLVM knows to deallocate them during an
     104             :  * LLVMShutdown(), rather than after (which'd cause crashes).
     105             :  */
     106             : typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
     107             : llvm::ManagedStatic<ModuleCache> module_cache;
     108             : typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
     109             : llvm::ManagedStatic<SummaryCache> summary_cache;
     110             : 
     111             : 
     112             : static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod);
     113             : static void llvm_execute_inline_plan(llvm::Module *mod,
     114             :                                      ImportMapTy *globalsToInline);
     115             : 
     116             : static llvm::Module* load_module_cached(LLVMContextRef c, llvm::StringRef modPath);
     117             : static std::unique_ptr<llvm::Module> load_module(LLVMContextRef c, llvm::StringRef Identifier);
     118             : static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
     119             : 
     120             : 
     121             : static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     122             :                                                    llvm::Function *F,
     123             :                                                    llvm::StringRef Name);
     124             : 
     125             : static bool function_inlinable(llvm::Function &F,
     126             :                                int threshold,
     127             :                                FunctionInlineStates &functionState,
     128             :                                InlineWorkList &worklist,
     129             :                                InlineSearchPath &searchpath,
     130             :                                llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     131             :                                int &running_instcount,
     132             :                                llvm::StringSet<> &importVars);
     133             : static void function_references(llvm::Function &F,
     134             :                                 int &running_instcount,
     135             :                                 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     136             :                                 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
     137             : 
     138             : static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
     139             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     140             : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
     141             : 
     142             : /* verbose debugging for inliner development */
     143             : /* #define INLINE_DEBUG */
     144             : #ifdef INLINE_DEBUG
     145             : #define ilog        elog
     146             : #else
     147             : #define ilog(...)   (void) 0
     148             : #endif
     149             : 
     150             : /*
     151             :  * Reset inlining related state. This needs to be called before the currently
     152             :  * used LLVMContextRef is disposed (and a new one create), otherwise we would
     153             :  * have dangling references to deleted modules.
     154             :  */
     155             : void
     156           0 : llvm_inline_reset_caches(void)
     157             : {
     158           0 :     module_cache->clear();
     159           0 :     summary_cache->clear();
     160           0 : }
     161             : 
     162             : /*
     163             :  * Perform inlining of external function references in M based on a simple
     164             :  * cost based analysis.
     165             :  */
     166             : void
     167         552 : llvm_inline(LLVMModuleRef M)
     168             : {
     169         552 :     LLVMContextRef lc = LLVMGetModuleContext(M);
     170         552 :     llvm::Module *mod = llvm::unwrap(M);
     171             : 
     172         552 :     std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
     173         552 :     if (!globalsToInline)
     174           0 :         return;
     175         552 :     llvm_execute_inline_plan(mod, globalsToInline.get());
     176             : }
     177             : 
     178             : /*
     179             :  * Build information necessary for inlining external function references in
     180             :  * mod.
     181             :  */
     182             : static std::unique_ptr<ImportMapTy>
     183         552 : llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
     184             : {
     185        1104 :     std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
     186        1104 :     FunctionInlineStates functionStates;
     187        1104 :     InlineWorkList worklist;
     188             : 
     189        1104 :     InlineSearchPath defaultSearchPath;
     190             : 
     191             :     /* attempt to add module to search path */
     192         552 :     add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
     193             :     /* if postgres isn't available, no point continuing */
     194         552 :     if (defaultSearchPath.empty())
     195           0 :         return nullptr;
     196             : 
     197             :     /*
     198             :      * Start inlining with current references to external functions by putting
     199             :      * them on the inlining worklist. If, during inlining of those, new extern
     200             :      * functions need to be inlined, they'll also be put there, with a lower
     201             :      * priority.
     202             :      */
     203       10532 :     for (const llvm::Function &funcDecl : mod->functions())
     204             :     {
     205        9980 :         InlineWorkListItem item = {};
     206        9980 :         FunctionInlineState inlineState = {};
     207             : 
     208             :         /* already has a definition */
     209        9980 :         if (!funcDecl.isDeclaration())
     210        6846 :             continue;
     211             : 
     212             :         /* llvm provides implementation */
     213        3134 :         if (funcDecl.isIntrinsic())
     214         492 :             continue;
     215             : 
     216        2642 :         item.symbolName = funcDecl.getName();
     217        2642 :         item.searchpath = defaultSearchPath;
     218        2642 :         worklist.push_back(item);
     219        2642 :         inlineState.costLimit = inline_initial_cost;
     220        2642 :         inlineState.processed = false;
     221        2642 :         inlineState.inlined = false;
     222        2642 :         inlineState.allowReconsidering = false;
     223        2642 :         functionStates[funcDecl.getName()] = inlineState;
     224             :     }
     225             : 
     226             :     /*
     227             :      * Iterate over pending worklist items, look them up in index, check
     228             :      * whether they should be inlined.
     229             :      */
     230       12346 :     while (!worklist.empty())
     231             :     {
     232       11794 :         InlineWorkListItem item = worklist.pop_back_val();
     233       11794 :         llvm::StringRef symbolName = item.symbolName;
     234             :         char *cmodname;
     235             :         char *cfuncname;
     236       11794 :         FunctionInlineState &inlineState = functionStates[symbolName];
     237             :         llvm::GlobalValue::GUID funcGUID;
     238             : 
     239       11794 :         llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
     240             : 
     241       11794 :         funcGUID = llvm::GlobalValue::getGUID(cfuncname);
     242             : 
     243             :         /* already processed */
     244       11794 :         if (inlineState.processed)
     245           0 :             continue;
     246             : 
     247             : 
     248       11794 :         if (cmodname)
     249           0 :             add_module_to_inline_search_path(item.searchpath, cmodname);
     250             : 
     251             :         /*
     252             :          * Iterate over all known definitions of function, via the index. Then
     253             :          * look up module(s), check if function actually is defined (there
     254             :          * could be hash conflicts).
     255             :          */
     256       22690 :         for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
     257             :         {
     258             :             const llvm::FunctionSummary *fs;
     259       10896 :             llvm::StringRef modPath = gvs->modulePath();
     260             :             llvm::Module *defMod;
     261             :             llvm::Function *funcDef;
     262             : 
     263       10896 :             fs = llvm::cast<llvm::FunctionSummary>(gvs);
     264             : 
     265       10896 :             if (gvs->notEligibleToImport())
     266             :             {
     267             :                 ilog(DEBUG1, "ineligibile to import %s due to summary",
     268             :                      symbolName.data());
     269        5552 :                 continue;
     270             :             }
     271             : 
     272       10896 :             if ((int) fs->instCount() > inlineState.costLimit)
     273             :             {
     274             :                 ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
     275             :                      symbolName.data(), fs->instCount(), inlineState.costLimit);
     276        4850 :                 inlineState.allowReconsidering = true;
     277        4850 :                 continue;
     278             :             }
     279             : 
     280        6046 :             defMod = load_module_cached(lc, modPath);
     281        6046 :             if (defMod->materializeMetadata())
     282           0 :                 elog(FATAL, "failed to materialize metadata");
     283             : 
     284        6046 :             funcDef = defMod->getFunction(cfuncname);
     285             : 
     286             :             /*
     287             :              * This can happen e.g. in case of a hash collision of the
     288             :              * function's name.
     289             :              */
     290        6046 :             if (!funcDef)
     291           0 :                 continue;
     292             : 
     293        6046 :             if (funcDef->materialize())
     294           0 :                 elog(FATAL, "failed to materialize metadata");
     295             : 
     296             :             Assert(!funcDef->isDeclaration());
     297             :             Assert(funcDef->hasExternalLinkage());
     298             : 
     299        6046 :             llvm::StringSet<> importVars;
     300        6046 :             llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
     301        6046 :             int running_instcount = 0;
     302             : 
     303             :             /*
     304             :              * Check whether function, and objects it depends on, are
     305             :              * inlinable.
     306             :              */
     307        6046 :             if (function_inlinable(*funcDef,
     308             :                                    inlineState.costLimit,
     309             :                                    functionStates,
     310             :                                    worklist,
     311             :                                    item.searchpath,
     312             :                                    visitedFunctions,
     313             :                                    running_instcount,
     314             :                                    importVars))
     315             :             {
     316             :                 /*
     317             :                  * Check whether function and all its dependencies are too
     318             :                  * big. Dependencies already counted for other functions that
     319             :                  * will get inlined are not counted again. While this make
     320             :                  * things somewhat order dependent, I can't quite see a point
     321             :                  * in a different behaviour.
     322             :                  */
     323        5178 :                 if (running_instcount > inlineState.costLimit)
     324             :                 {
     325             :                     ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
     326             :                          symbolName.data(), running_instcount, inlineState.costLimit);
     327         702 :                     inlineState.allowReconsidering = true;
     328         702 :                     continue;
     329             :                 }
     330             : 
     331             :                 ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
     332             :                      symbolName.data(), running_instcount, fs->instCount());
     333             : 
     334             :                 /* import referenced function itself */
     335        4476 :                 importVars.insert(symbolName);
     336             : 
     337             :                 {
     338        4476 :                     llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
     339       11076 :                     for (auto& importVar : importVars)
     340        6600 :                         modGlobalsToInline.insert(importVar.first());
     341             :                     Assert(modGlobalsToInline.size() > 0);
     342             :                 }
     343             : 
     344             :                 /* mark function as inlined */
     345        4476 :                 inlineState.inlined = true;
     346             : 
     347             :                 /*
     348             :                  * Found definition to inline, don't look for further
     349             :                  * potential definitions.
     350             :                  */
     351        4476 :                 break;
     352             :             }
     353             :             else
     354             :             {
     355             :                 ilog(DEBUG1, "had to skip inlining %s",
     356             :                      symbolName.data());
     357             : 
     358             :                 /* It's possible there's another definition that's inlinable. */
     359             :             }
     360             :         }
     361             : 
     362             :         /*
     363             :          * Signal that we're done with symbol, whether successful (inlined =
     364             :          * true above) or not.
     365             :          */
     366       11794 :         inlineState.processed = true;
     367             :     }
     368             : 
     369         552 :     return globalsToInline;
     370             : }
     371             : 
     372             : /*
     373             :  * Perform the actual inlining of external functions (and their dependencies)
     374             :  * into mod.
     375             :  */
     376             : static void
     377         552 : llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
     378             : {
     379        1104 :     llvm::IRMover Mover(*mod);
     380             : 
     381        3604 :     for (const auto& toInline : *globalsToInline)
     382             :     {
     383        3052 :         const llvm::StringRef& modPath = toInline.first();
     384        3052 :         const llvm::StringSet<>& modGlobalsToInline = toInline.second;
     385        3052 :         llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
     386             : 
     387             :         Assert(module_cache->count(modPath));
     388        3052 :         std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
     389        3052 :         module_cache->erase(modPath);
     390             : 
     391        3052 :         if (modGlobalsToInline.empty())
     392           0 :             continue;
     393             : 
     394        9310 :         for (auto &glob: modGlobalsToInline)
     395             :         {
     396        6258 :             llvm::StringRef SymbolName = glob.first();
     397             :             char *modname;
     398             :             char *funcname;
     399             : 
     400        6258 :             llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
     401             : 
     402        6258 :             llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
     403             : 
     404        6258 :             if (!valueToImport)
     405           0 :                 elog(FATAL, "didn't refind value %s to import", SymbolName.data());
     406             : 
     407             :             /*
     408             :              * For functions (global vars are only inlined if already static),
     409             :              * mark imported variables as being clones from other
     410             :              * functions. That a) avoids symbol conflicts b) allows the
     411             :              * optimizer to perform inlining.
     412             :             */
     413        6258 :             if (llvm::isa<llvm::Function>(valueToImport))
     414             :             {
     415        4536 :                 llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
     416             :                 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     417             : 
     418             :                 /*
     419             :                  * Per-function info isn't necessarily stripped yet, as the
     420             :                  * module is lazy-loaded when stripped above.
     421             :                  */
     422        4536 :                 llvm::stripDebugInfo(*F);
     423             : 
     424             :                 /*
     425             :                  * If the to-be-imported function is one referenced including
     426             :                  * its module name, create a tiny inline function that just
     427             :                  * forwards the call. One might think a GlobalAlias would do
     428             :                  * the trick, but a) IRMover doesn't override a declaration
     429             :                  * with an alias pointing to a definition (instead renaming
     430             :                  * it), b) Aliases can't be AvailableExternally.
     431             :                  */
     432        4536 :                 if (modname)
     433             :                 {
     434             :                     llvm::Function *AF;
     435             : 
     436           0 :                     AF = create_redirection_function(importMod, F, SymbolName);
     437             : 
     438           0 :                     GlobalsToImport.insert(AF);
     439           0 :                     llvm::stripDebugInfo(*AF);
     440             :                 }
     441             : 
     442        4536 :                 if (valueToImport->hasExternalLinkage())
     443             :                 {
     444        4476 :                     valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
     445             :                 }
     446             :             }
     447             : 
     448        6258 :             GlobalsToImport.insert(valueToImport);
     449             :             ilog(DEBUG1, "performing import of %s %s",
     450             :                  modPath.data(), SymbolName.data());
     451             : 
     452             :         }
     453             : 
     454        3052 :         if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
     455        1096 :                        [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {},
     456        6104 :                        /*IsPerformingImport=*/false))
     457           0 :             elog(FATAL, "function import failed with linker error");
     458             :     }
     459         552 : }
     460             : 
     461             : /*
     462             :  * Return a module identified by modPath, caching it in memory.
     463             :  *
     464             :  * Note that such a module may *not* be modified without copying, otherwise
     465             :  * the cache state would get corrupted.
     466             :  */
     467             : static llvm::Module*
     468        6046 : load_module_cached(LLVMContextRef lc, llvm::StringRef modPath)
     469             : {
     470        6046 :     auto it = module_cache->find(modPath);
     471        6046 :     if (it == module_cache->end())
     472             :     {
     473        3166 :         it = module_cache->insert(
     474        3166 :             std::make_pair(modPath, load_module(lc, modPath))).first;
     475             :     }
     476             : 
     477        6046 :     return it->second.get();
     478             : }
     479             : 
     480             : static std::unique_ptr<llvm::Module>
     481        3166 : load_module(LLVMContextRef lc, llvm::StringRef Identifier)
     482             : {
     483             :     LLVMMemoryBufferRef buf;
     484             :     LLVMModuleRef mod;
     485             :     char path[MAXPGPATH];
     486             :     char *msg;
     487             : 
     488        3166 :     snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
     489             : 
     490        3166 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
     491           0 :         elog(FATAL, "failed to open bitcode file \"%s\": %s",
     492             :              path, msg);
     493        3166 :     if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
     494           0 :         elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
     495             : 
     496             :     /*
     497             :      * Currently there's no use in more detailed debug info for JITed
     498             :      * code. Until that changes, not much point in wasting memory and cycles
     499             :      * on processing debuginfo.
     500             :      */
     501        3166 :     llvm::StripDebugInfo(*llvm::unwrap(mod));
     502             : 
     503        6332 :     return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
     504             : }
     505             : 
     506             : /*
     507             :  * Compute list of referenced variables, functions and the instruction count
     508             :  * for a function.
     509             :  */
     510             : static void
     511        9742 : function_references(llvm::Function &F,
     512             :                     int &running_instcount,
     513             :                     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     514             :                     llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
     515             : {
     516       19484 :     llvm::SmallPtrSet<const llvm::User *, 32> Visited;
     517             : 
     518      129560 :     for (llvm::BasicBlock &BB : F)
     519             :     {
     520      760358 :         for (llvm::Instruction &I : BB)
     521             :         {
     522      640540 :             if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
     523           0 :                 continue;
     524             : 
     525     1281080 :             llvm::SmallVector<llvm::User *, 8> Worklist;
     526      640540 :             Worklist.push_back(&I);
     527             : 
     528      640540 :             running_instcount++;
     529             : 
     530     2275038 :             while (!Worklist.empty()) {
     531     1634498 :                 llvm::User *U = Worklist.pop_back_val();
     532             : 
     533             :                 /* visited before */
     534     1634498 :                 if (!Visited.insert(U).second)
     535      855026 :                     continue;
     536             : 
     537     2139780 :                 for (auto &OI : U->operands()) {
     538     1360308 :                     llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
     539     1360308 :                     if (!Operand)
     540      410870 :                         continue;
     541     1103962 :                     if (llvm::isa<llvm::BlockAddress>(Operand))
     542           0 :                         continue;
     543     1103962 :                     if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
     544       46376 :                         referencedVars.insert(GV);
     545       46376 :                         if (GV->hasInitializer())
     546       44520 :                             Worklist.push_back(GV->getInitializer());
     547       46376 :                         continue;
     548             :                     }
     549     1057586 :                     if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
     550      108148 :                         referencedFunctions.insert(CF);
     551      108148 :                         continue;
     552             :                     }
     553      949438 :                     Worklist.push_back(Operand);
     554             :                 }
     555             :             }
     556             :         }
     557             :     }
     558        9742 : }
     559             : 
     560             : /*
     561             :  * Check whether function F is inlinable and, if so, what globals need to be
     562             :  * imported.
     563             :  *
     564             :  * References to external functions from, potentially recursively, inlined
     565             :  * functions are added to the passed in worklist.
     566             :  */
     567             : static bool
     568        9790 : function_inlinable(llvm::Function &F,
     569             :                    int threshold,
     570             :                    FunctionInlineStates &functionStates,
     571             :                    InlineWorkList &worklist,
     572             :                    InlineSearchPath &searchpath,
     573             :                    llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     574             :                    int &running_instcount,
     575             :                    llvm::StringSet<> &importVars)
     576             : {
     577        9790 :     int subThreshold = threshold * inline_cost_decay_factor;
     578       19580 :     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
     579       19580 :     llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
     580             : 
     581             :     /* can't rely on what may be inlined */
     582        9790 :     if (F.isInterposable())
     583           0 :         return false;
     584             : 
     585             :     /*
     586             :      * Can't rely on function being present. Alternatively we could create a
     587             :      * static version of these functions?
     588             :      */
     589        9790 :     if (F.hasAvailableExternallyLinkage())
     590           0 :         return false;
     591             : 
     592             :     ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
     593             : 
     594        9790 :     if (F.materialize())
     595           0 :         elog(FATAL, "failed to materialize metadata");
     596             : 
     597        9790 :     if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
     598             :     {
     599             :         ilog(DEBUG1, "ineligibile to import %s due to noinline",
     600             :              F.getName().data());
     601          48 :         return false;
     602             :     }
     603             : 
     604        9742 :     function_references(F, running_instcount, referencedVars, referencedFunctions);
     605             : 
     606       39206 :     for (llvm::GlobalVariable* rv: referencedVars)
     607             :     {
     608       30284 :         if (rv->materialize())
     609           0 :             elog(FATAL, "failed to materialize metadata");
     610             : 
     611             :         /*
     612             :          * Don't inline functions that access thread local variables.  That
     613             :          * doesn't work on current LLVM releases (but might in future).
     614             :          */
     615       30284 :         if (rv->isThreadLocal())
     616             :         {
     617             :             ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
     618             :                  F.getName().data(), rv->getName().data());
     619         820 :             return false;
     620             :         }
     621             : 
     622             :         /*
     623             :          * Never want to inline externally visible vars, cheap enough to
     624             :          * reference.
     625             :          */
     626       30284 :         if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
     627         660 :             continue;
     628             : 
     629             :         /*
     630             :          * If variable is file-local, we need to inline it, to be able to
     631             :          * inline the function itself. Can't do that if the variable can be
     632             :          * modified, because they'd obviously get out of sync.
     633             :          *
     634             :          * XXX: Currently not a problem, but there'd be problems with
     635             :          * nontrivial initializers if they were allowed for postgres.
     636             :          */
     637       29624 :         if (!rv->isConstant())
     638             :         {
     639             :             ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
     640             :                  F.getName().data(), rv->getName().data());
     641         820 :             return false;
     642             :         }
     643             : 
     644             :         ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
     645             :              rv->getName().data(), (int)rv->getLinkage());
     646             : 
     647       28804 :         importVars.insert(rv->getName());
     648             :         /* small cost attributed to each cloned global */
     649       28804 :         running_instcount += 5;
     650             :     }
     651             : 
     652        8922 :     visitedFunctions.insert(&F);
     653             : 
     654             :     /*
     655             :      * Check referenced functions. Check whether used static ones are
     656             :      * inlinable, and remember external ones for inlining.
     657             :      */
     658       57542 :     for (llvm::Function* referencedFunction: referencedFunctions)
     659             :     {
     660       48798 :         llvm::StringSet<> recImportVars;
     661             : 
     662       48798 :         if (referencedFunction->materialize())
     663           0 :             elog(FATAL, "failed to materialize metadata");
     664             : 
     665       48798 :         if (referencedFunction->isIntrinsic())
     666        6360 :             continue;
     667             : 
     668             :         /* if already visited skip, otherwise remember */
     669       42438 :         if (!visitedFunctions.insert(referencedFunction).second)
     670       17586 :             continue;
     671             : 
     672             :         /*
     673             :          * We don't inline external functions directly here, instead we put
     674             :          * them on the worklist if appropriate and check them from
     675             :          * llvm_build_inline_plan().
     676             :          */
     677       24852 :         if (referencedFunction->hasExternalLinkage())
     678             :         {
     679       21108 :             llvm::StringRef funcName = referencedFunction->getName();
     680             : 
     681             :             /*
     682             :              * Don't bother checking for inlining if remaining cost budget is
     683             :              * very small.
     684             :              */
     685       21108 :             if (subThreshold < 5)
     686        5848 :                 continue;
     687             : 
     688       15260 :             auto it = functionStates.find(funcName);
     689       15260 :             if (it == functionStates.end())
     690             :             {
     691             :                 FunctionInlineState inlineState;
     692             : 
     693        8700 :                 inlineState.costLimit = subThreshold;
     694        8700 :                 inlineState.processed = false;
     695        8700 :                 inlineState.inlined = false;
     696        8700 :                 inlineState.allowReconsidering = false;
     697             : 
     698        8700 :                 functionStates[funcName] = inlineState;
     699        8700 :                 worklist.push_back({funcName, searchpath});
     700             : 
     701             :                 ilog(DEBUG1,
     702             :                      "considering extern function %s at %d for inlining",
     703             :                      funcName.data(), subThreshold);
     704             :             }
     705        6560 :             else if (!it->second.inlined &&
     706       11164 :                      (!it->second.processed || it->second.allowReconsidering) &&
     707       11164 :                      it->second.costLimit < subThreshold)
     708             :             {
     709             :                 /*
     710             :                  * Update inlining threshold if higher. Need to re-queue
     711             :                  * to be processed if already processed with lower
     712             :                  * threshold.
     713             :                  */
     714         494 :                 if (it->second.processed)
     715             :                 {
     716             :                     ilog(DEBUG1,
     717             :                          "reconsidering extern function %s at %d for inlining, increasing from %d",
     718             :                          funcName.data(), subThreshold, it->second.costLimit);
     719             : 
     720         452 :                     it->second.processed = false;
     721         452 :                     it->second.allowReconsidering = false;
     722         452 :                     worklist.push_back({funcName, searchpath});
     723             :                 }
     724         494 :                 it->second.costLimit = subThreshold;
     725             :             }
     726       15260 :             continue;
     727             :         }
     728             : 
     729             :         /* can't rely on what may be inlined */
     730        3744 :         if (referencedFunction->isInterposable())
     731           0 :             return false;
     732             : 
     733        3744 :         if (!function_inlinable(*referencedFunction,
     734             :                                 subThreshold,
     735             :                                 functionStates,
     736             :                                 worklist,
     737             :                                 searchpath,
     738             :                                 visitedFunctions,
     739             :                                 running_instcount,
     740             :                                 recImportVars))
     741             :         {
     742             :             ilog(DEBUG1,
     743             :                  "cannot inline %s due to required function %s not being inlinable",
     744             :                  F.getName().data(), referencedFunction->getName().data());
     745         178 :             return false;
     746             :         }
     747             : 
     748             :         /* import referenced function itself */
     749        3566 :         importVars.insert(referencedFunction->getName());
     750             : 
     751             :         /* import referenced function and its dependents */
     752       79026 :         for (auto& recImportVar : recImportVars)
     753       75460 :             importVars.insert(recImportVar.first());
     754             :     }
     755             : 
     756        8744 :     return true;
     757             : }
     758             : 
     759             : /*
     760             :  * Attempt to load module summary located at path. Return empty pointer when
     761             :  * loading fails.
     762             :  */
     763             : static std::unique_ptr<llvm::ModuleSummaryIndex>
     764         384 : llvm_load_summary(llvm::StringRef path)
     765             : {
     766             :     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
     767         768 :         llvm::MemoryBuffer::getFile(path);
     768             : 
     769         384 :     if (std::error_code EC = MBOrErr.getError())
     770             :     {
     771             :         ilog(DEBUG1, "failed to open %s: %s", path.data(),
     772             :              EC.message().c_str());
     773             :     }
     774             :     else
     775             :     {
     776         384 :         llvm::MemoryBufferRef ref(*MBOrErr.get().get());
     777             : 
     778             :         llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
     779         768 :             llvm::getModuleSummaryIndex(ref);
     780         384 :         if (IndexOrErr)
     781         384 :             return std::move(IndexOrErr.get());
     782           0 :         elog(FATAL, "failed to load summary \"%s\": %s",
     783             :              path.data(),
     784             :              toString(IndexOrErr.takeError()).c_str());
     785             :     }
     786           0 :     return nullptr;
     787             : }
     788             : 
     789             : /*
     790             :  * Attempt to add modpath to the search path.
     791             :  */
     792             : static void
     793         552 : add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
     794             : {
     795             :     /* only extension in libdir are candidates for inlining for now */
     796             : #if LLVM_VERSION_MAJOR < 16
     797             : #define starts_with startswith
     798             : #endif
     799         552 :     if (!modpath.starts_with("$libdir/"))
     800           0 :         return;
     801             : 
     802             :     /* if there's no match, attempt to load */
     803         552 :     auto it = summary_cache->find(modpath);
     804         552 :     if (it == summary_cache->end())
     805             :     {
     806         768 :         std::string path(modpath);
     807         384 :         path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
     808         384 :         path += ".index.bc";
     809         384 :         (*summary_cache)[modpath] = llvm_load_summary(path);
     810         384 :         it = summary_cache->find(modpath);
     811             :     }
     812             : 
     813             :     Assert(it != summary_cache->end());
     814             : 
     815             :     /* if the entry isn't NULL, it's validly loaded */
     816         552 :     if (it->second)
     817         552 :         searchpath.push_back(it->second.get());
     818             : }
     819             : 
     820             : /*
     821             :  * Search for all references for functions hashing to guid in the search path,
     822             :  * and return them in search path order.
     823             :  */
     824             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     825       11794 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
     826             : {
     827       11794 :     llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
     828             : 
     829       23588 :     for (auto index : path)
     830             :     {
     831       11794 :         llvm::ValueInfo funcVI = index->getValueInfo(guid);
     832             : 
     833             :         /* if index doesn't know function, we don't have a body, continue */
     834       11794 :         if (funcVI)
     835       21792 :             for (auto &gv : funcVI.getSummaryList())
     836       10896 :                 matches.push_back(gv.get());
     837             :     }
     838             : 
     839       11794 :     return matches;
     840             : }
     841             : 
     842             : /*
     843             :  * Create inline wrapper with the name Name, redirecting the call to F.
     844             :  */
     845             : static llvm::Function*
     846           0 : create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     847             :                             llvm::Function *F,
     848             :                             llvm::StringRef Name)
     849             : {
     850             :     typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     851             : 
     852           0 :     llvm::LLVMContext &Context = F->getContext();
     853           0 :     llvm::IRBuilder<> Builder(Context);
     854             :     llvm::Function *AF;
     855             :     llvm::BasicBlock *BB;
     856             :     llvm::CallInst *fwdcall;
     857             : 
     858           0 :     AF = llvm::Function::Create(F->getFunctionType(),
     859             :                                 LinkageTypes::AvailableExternallyLinkage,
     860           0 :                                 Name, importMod.get());
     861           0 :     BB = llvm::BasicBlock::Create(Context, "entry", AF);
     862             : 
     863           0 :     Builder.SetInsertPoint(BB);
     864           0 :     fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
     865           0 :     fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
     866           0 :     Builder.CreateRet(fwdcall);
     867             : 
     868           0 :     return AF;
     869             : }

Generated by: LCOV version 1.14