LCOV - code coverage report
Current view: top level - src/backend/jit/llvm - llvmjit_inline.cpp (source / functions) Hit Total Coverage
Test: PostgreSQL 13devel Lines: 226 249 90.8 %
Date: 2019-11-15 23:07:02 Functions: 17 17 100.0 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*-------------------------------------------------------------------------
       2             :  *
       3             :  * llvmjit_inline.cpp
       4             :  *    Cross module inlining suitable for postgres' JIT
       5             :  *
       6             :  * The inliner iterates over external functions referenced from the passed
       7             :  * module and attempts to inline those.  It does so by utilizing pre-built
       8             :  * indexes over both postgres core code and extension modules.  When a match
       9             :  * for an external function is found - not guaranteed! - the index will then
      10             :  * be used to judge their instruction count / inline worthiness. After doing
      11             :  * so for all external functions, all the referenced functions (and
      12             :  * prerequisites) will be imported.
      13             :  *
      14             :  * Copyright (c) 2016-2019, PostgreSQL Global Development Group
      15             :  *
      16             :  * IDENTIFICATION
      17             :  *    src/backend/lib/llvmjit/llvmjit_inline.cpp
      18             :  *
      19             :  *-------------------------------------------------------------------------
      20             :  */
      21             : 
      22             : extern "C"
      23             : {
      24             : #include "postgres.h"
      25             : }
      26             : 
      27             : #include "jit/llvmjit.h"
      28             : 
      29             : extern "C"
      30             : {
      31             : #include <fcntl.h>
      32             : #include <sys/mman.h>
      33             : #include <sys/stat.h>
      34             : #include <sys/types.h>
      35             : #include <unistd.h>
      36             : 
      37             : #include "common/string.h"
      38             : #include "miscadmin.h"
      39             : #include "storage/fd.h"
      40             : }
      41             : 
      42             : #include <llvm-c/Core.h>
      43             : #include <llvm-c/BitReader.h>
      44             : 
      45             : /* Avoid macro clash with LLVM's C++ headers */
      46             : #undef Min
      47             : 
      48             : #include <llvm/ADT/SetVector.h>
      49             : #include <llvm/ADT/StringSet.h>
      50             : #include <llvm/ADT/StringMap.h>
      51             : #include <llvm/Analysis/ModuleSummaryAnalysis.h>
      52             : #if LLVM_VERSION_MAJOR > 3
      53             : #include <llvm/Bitcode/BitcodeReader.h>
      54             : #else
      55             : #include <llvm/Bitcode/ReaderWriter.h>
      56             : #include <llvm/Support/Error.h>
      57             : #endif
      58             : #include <llvm/IR/Attributes.h>
      59             : #include <llvm/IR/CallSite.h>
      60             : #include <llvm/IR/DebugInfo.h>
      61             : #include <llvm/IR/IntrinsicInst.h>
      62             : #include <llvm/IR/IRBuilder.h>
      63             : #include <llvm/IR/ModuleSummaryIndex.h>
      64             : #include <llvm/Linker/IRMover.h>
      65             : #include <llvm/Support/ManagedStatic.h>
      66             : 
      67             : 
      68             : /*
      69             :  * Type used to represent modules InlineWorkListItem's subject is searched for
      70             :  * in.
      71             :  */
      72             : typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
      73             : 
      74             : /*
      75             :  * Item in queue of to-be-checked symbols and corresponding queue.
      76             :  */
      77       28330 : typedef struct InlineWorkListItem
      78             : {
      79             :     llvm::StringRef symbolName;
      80             :     llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
      81             : } InlineWorkListItem;
      82             : typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
      83             : 
      84             : /*
      85             :  * Information about symbols processed during inlining. Used to prevent
      86             :  * repeated searches and provide additional information.
      87             :  */
      88             : typedef struct FunctionInlineState
      89             : {
      90             :     int costLimit;
      91             :     bool processed;
      92             :     bool inlined;
      93             :     bool allowReconsidering;
      94             : } FunctionInlineState;
      95             : typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
      96             : 
      97             : /*
      98             :  * Map of modules that should be inlined, with a list of the to-be inlined
      99             :  * symbols.
     100             :  */
     101             : typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
     102             : 
     103             : 
     104             : const float inline_cost_decay_factor = 0.5;
     105             : const int inline_initial_cost = 150;
     106             : 
     107             : /*
     108             :  * These are managed statics so LLVM knows to deallocate them during an
     109             :  * LLVMShutdown(), rather than after (which'd cause crashes).
     110             :  */
     111             : typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
     112        1162 : llvm::ManagedStatic<ModuleCache> module_cache;
     113             : typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
     114        1162 : llvm::ManagedStatic<SummaryCache> summary_cache;
     115             : 
     116             : 
     117             : static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(llvm::Module *mod);
     118             : static void llvm_execute_inline_plan(llvm::Module *mod,
     119             :                                      ImportMapTy *globalsToInline);
     120             : 
     121             : static llvm::Module* load_module_cached(llvm::StringRef modPath);
     122             : static std::unique_ptr<llvm::Module> load_module(llvm::StringRef Identifier);
     123             : static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
     124             : 
     125             : 
     126             : static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     127             :                                                    llvm::Function *F,
     128             :                                                    llvm::StringRef Name);
     129             : 
     130             : static bool function_inlinable(llvm::Function &F,
     131             :                                int threshold,
     132             :                                FunctionInlineStates &functionState,
     133             :                                InlineWorkList &worklist,
     134             :                                InlineSearchPath &searchpath,
     135             :                                llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     136             :                                int &running_instcount,
     137             :                                llvm::StringSet<> &importVars);
     138             : static void function_references(llvm::Function &F,
     139             :                                 int &running_instcount,
     140             :                                 llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     141             :                                 llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
     142             : 
     143             : static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
     144             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     145             : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
     146             : 
     147             : /* verbose debugging for inliner development */
     148             : /* #define INLINE_DEBUG */
     149             : #ifdef INLINE_DEBUG
     150             : #define ilog        elog
     151             : #else
     152             : #define ilog(...)   (void) 0
     153             : #endif
     154             : 
     155             : /*
     156             :  * Perform inlining of external function references in M based on a simple
     157             :  * cost based analysis.
     158             :  */
     159             : void
     160         594 : llvm_inline(LLVMModuleRef M)
     161             : {
     162         594 :     llvm::Module *mod = llvm::unwrap(M);
     163             : 
     164        1188 :     std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(mod);
     165         594 :     if (!globalsToInline)
     166           0 :         return;
     167         594 :     llvm_execute_inline_plan(mod, globalsToInline.get());
     168             : }
     169             : 
     170             : /*
     171             :  * Build information necessary for inlining external function references in
     172             :  * mod.
     173             :  */
     174             : static std::unique_ptr<ImportMapTy>
     175         594 : llvm_build_inline_plan(llvm::Module *mod)
     176             : {
     177        1188 :     std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
     178        1188 :     FunctionInlineStates functionStates;
     179        1188 :     InlineWorkList worklist;
     180             : 
     181        1188 :     InlineSearchPath defaultSearchPath;
     182             : 
     183             :     /* attempt to add module to search path */
     184         594 :     add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
     185             :     /* if postgres isn't available, no point continuing */
     186         594 :     if (defaultSearchPath.empty())
     187           0 :         return nullptr;
     188             : 
     189             :     /*
     190             :      * Start inlining with current references to external functions by putting
     191             :      * them on the inlining worklist. If, during inlining of those, new extern
     192             :      * functions need to be inlined, they'll also be put there, with a lower
     193             :      * priority.
     194             :      */
     195        7900 :     for (const llvm::Function &funcDecl : mod->functions())
     196             :     {
     197        9242 :         InlineWorkListItem item = {};
     198        7306 :         FunctionInlineState inlineState = {};
     199             : 
     200             :         /* already has a definition */
     201        7306 :         if (!funcDecl.isDeclaration())
     202        4858 :             continue;
     203             : 
     204             :         /* llvm provides implementation */
     205        2448 :         if (funcDecl.isIntrinsic())
     206         512 :             continue;
     207             : 
     208        1936 :         item.symbolName = funcDecl.getName();
     209        1936 :         item.searchpath = defaultSearchPath;
     210        1936 :         worklist.push_back(item);
     211        1936 :         inlineState.costLimit = inline_initial_cost;
     212        1936 :         inlineState.processed = false;
     213        1936 :         inlineState.inlined = false;
     214        1936 :         inlineState.allowReconsidering = false;
     215        1936 :         functionStates[funcDecl.getName()] = inlineState;
     216             :     }
     217             : 
     218             :     /*
     219             :      * Iterate over pending worklist items, look them up in index, check
     220             :      * whether they should be inlined.
     221             :      */
     222        9778 :     while (!worklist.empty())
     223             :     {
     224        9184 :         InlineWorkListItem item = worklist.pop_back_val();
     225        4592 :         llvm::StringRef symbolName = item.symbolName;
     226             :         char *cmodname;
     227             :         char *cfuncname;
     228        4592 :         FunctionInlineState &inlineState = functionStates[symbolName];
     229             :         llvm::GlobalValue::GUID funcGUID;
     230             : 
     231        4592 :         llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
     232             : 
     233        9184 :         funcGUID = llvm::GlobalValue::getGUID(cfuncname);
     234             : 
     235             :         /* already processed */
     236        4592 :         if (inlineState.processed)
     237           0 :             continue;
     238             : 
     239             : 
     240        4592 :         if (cmodname)
     241          52 :             add_module_to_inline_search_path(item.searchpath, cmodname);
     242             : 
     243             :         /*
     244             :          * Iterate over all known definitions of function, via the index. Then
     245             :          * look up module(s), check if function actually is defined (there
     246             :          * could be hash conflicts).
     247             :          */
     248       16042 :         for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
     249             :         {
     250             :             const llvm::FunctionSummary *fs;
     251        4462 :             llvm::StringRef modPath = gvs->modulePath();
     252             :             llvm::Module *defMod;
     253             :             llvm::Function *funcDef;
     254             : 
     255        4462 :             fs = llvm::cast<llvm::FunctionSummary>(gvs);
     256             : 
     257             : #if LLVM_VERSION_MAJOR > 3
     258        4462 :             if (gvs->notEligibleToImport())
     259             :             {
     260             :                 ilog(DEBUG1, "ineligibile to import %s due to summary",
     261             :                      symbolName.data());
     262        2564 :                 continue;
     263             :             }
     264             : #endif
     265             : 
     266        3722 :             if ((int) fs->instCount() > inlineState.costLimit)
     267             :             {
     268             :                 ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
     269             :                      symbolName.data(), fs->instCount(), inlineState.costLimit);
     270         914 :                 inlineState.allowReconsidering = true;
     271         914 :                 continue;
     272             :             }
     273             : 
     274        2808 :             defMod = load_module_cached(modPath);
     275        2808 :             if (defMod->materializeMetadata())
     276           0 :                 elog(FATAL, "failed to materialize metadata");
     277             : 
     278        5616 :             funcDef = defMod->getFunction(cfuncname);
     279             : 
     280             :             /*
     281             :              * This can happen e.g. in case of a hash collision of the
     282             :              * function's name.
     283             :              */
     284        2808 :             if (!funcDef)
     285           0 :                 continue;
     286             : 
     287        2808 :             if (funcDef->materialize())
     288           0 :                 elog(FATAL, "failed to materialize metadata");
     289             : 
     290             :             Assert(!funcDef->isDeclaration());
     291             :             Assert(funcDef->hasExternalLinkage());
     292             : 
     293        3250 :             llvm::StringSet<> importVars;
     294        3250 :             llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
     295        2808 :             int running_instcount = 0;
     296             : 
     297             :             /*
     298             :              * Check whether function, and objects it depends on, are
     299             :              * inlinable.
     300             :              */
     301        2808 :             if (function_inlinable(*funcDef,
     302             :                                    inlineState.costLimit,
     303             :                                    functionStates,
     304             :                                    worklist,
     305             :                                    item.searchpath,
     306             :                                    visitedFunctions,
     307             :                                    running_instcount,
     308             :                                    importVars))
     309             :             {
     310             :                 /*
     311             :                  * Check whether function and all its dependencies are too
     312             :                  * big. Dependencies already counted for other functions that
     313             :                  * will get inlined are not counted again. While this make
     314             :                  * things somewhat order dependent, I can't quite see a point
     315             :                  * in a different behaviour.
     316             :                  */
     317        2366 :                 if (running_instcount > inlineState.costLimit)
     318             :                 {
     319             :                     ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
     320             :                          symbolName.data(), running_instcount, inlineState.costLimit);
     321         170 :                     inlineState.allowReconsidering = true;
     322         170 :                     continue;
     323             :                 }
     324             : 
     325             :                 ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
     326             :                      symbolName.data(), running_instcount, fs->instCount());
     327             : 
     328             :                 /* import referenced function itself */
     329        2196 :                 importVars.insert(symbolName);
     330             : 
     331             :                 {
     332        2196 :                     llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
     333        5970 :                     for (auto& importVar : importVars)
     334        3774 :                         modGlobalsToInline.insert(importVar.first());
     335             :                     Assert(modGlobalsToInline.size() > 0);
     336             :                 }
     337             : 
     338             :                 /* mark function as inlined */
     339        2196 :                 inlineState.inlined = true;
     340             : 
     341             :                 /*
     342             :                  * Found definition to inline, don't look for further
     343             :                  * potential definitions.
     344             :                  */
     345        2196 :                 break;
     346             :             }
     347             :             else
     348             :             {
     349             :                 ilog(DEBUG1, "had to skip inlining %s",
     350             :                      symbolName.data());
     351             : 
     352             :                 /* It's possible there's another definition that's inlinable. */
     353             :             }
     354             :         }
     355             : 
     356             :         /*
     357             :          * Signal that we're done with symbol, whether successful (inlined =
     358             :          * true above) or not.
     359             :          */
     360        4592 :         inlineState.processed = true;
     361             :     }
     362             : 
     363         594 :     return globalsToInline;
     364             : }
     365             : 
     366             : /*
     367             :  * Perform the actual inlining of external functions (and their dependencies)
     368             :  * into mod.
     369             :  */
     370             : static void
     371         594 : llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
     372             : {
     373        1188 :     llvm::IRMover Mover(*mod);
     374             : 
     375        2452 :     for (const auto& toInline : *globalsToInline)
     376             :     {
     377        1858 :         const llvm::StringRef& modPath = toInline.first();
     378        1858 :         const llvm::StringSet<>& modGlobalsToInline = toInline.second;
     379        3716 :         llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
     380             : 
     381             :         Assert(module_cache->count(modPath));
     382        3716 :         std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
     383        1858 :         module_cache->erase(modPath);
     384             : 
     385        1858 :         if (modGlobalsToInline.empty())
     386           0 :             continue;
     387             : 
     388        5524 :         for (auto &glob: modGlobalsToInline)
     389             :         {
     390        3666 :             llvm::StringRef SymbolName = glob.first();
     391             :             char *modname;
     392             :             char *funcname;
     393             : 
     394        3666 :             llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
     395             : 
     396        7332 :             llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
     397             : 
     398        3666 :             if (!valueToImport)
     399           0 :                 elog(FATAL, "didn't refind value %s to import", SymbolName.data());
     400             : 
     401             :             /*
     402             :              * For functions (global vars are only inlined if already static),
     403             :              * mark imported variables as being clones from other
     404             :              * functions. That a) avoids symbol conflicts b) allows the
     405             :              * optimizer to perform inlining.
     406             :             */
     407        3666 :             if (llvm::isa<llvm::Function>(valueToImport))
     408             :             {
     409        2250 :                 llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
     410             :                 typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     411             : 
     412             :                 /*
     413             :                  * Per-function info isn't necessarily stripped yet, as the
     414             :                  * module is lazy-loaded when stripped above.
     415             :                  */
     416        2250 :                 llvm::stripDebugInfo(*F);
     417             : 
     418             :                 /*
     419             :                  * If the to-be-imported function is one referenced including
     420             :                  * its module name, create a tiny inline function that just
     421             :                  * forwards the call. One might think a GlobalAlias would do
     422             :                  * the trick, but a) IRMover doesn't override a declaration
     423             :                  * with an alias pointing to a definition (instead renaming
     424             :                  * it), b) Aliases can't be AvailableExternally.
     425             :                  */
     426        2250 :                 if (modname)
     427             :                 {
     428             :                     llvm::Function *AF;
     429             : 
     430          22 :                     AF = create_redirection_function(importMod, F, SymbolName);
     431             : 
     432          22 :                     GlobalsToImport.insert(AF);
     433          22 :                     llvm::stripDebugInfo(*AF);
     434             :                 }
     435             : 
     436        2250 :                 if (valueToImport->hasExternalLinkage())
     437             :                 {
     438        2196 :                     valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
     439             :                 }
     440             :             }
     441             : 
     442        3666 :             GlobalsToImport.insert(valueToImport);
     443             :             ilog(DEBUG1, "performing import of %s %s",
     444             :                  modPath.data(), SymbolName.data());
     445             : 
     446             :         }
     447             : 
     448             : #if LLVM_VERSION_MAJOR > 4
     449             : #define IRMOVE_PARAMS , /*IsPerformingImport=*/false
     450             : #elif LLVM_VERSION_MAJOR > 3
     451             : #define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
     452             : #else
     453             : #define IRMOVE_PARAMS
     454             : #endif
     455        1858 :         if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
     456         168 :                        [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
     457             :                        IRMOVE_PARAMS))
     458           0 :             elog(FATAL, "function import failed with linker error");
     459             :     }
     460         594 : }
     461             : 
     462             : /*
     463             :  * Return a module identified by modPath, caching it in memory.
     464             :  *
     465             :  * Note that such a module may *not* be modified without copying, otherwise
     466             :  * the cache state would get corrupted.
     467             :  */
     468             : static llvm::Module*
     469        2808 : load_module_cached(llvm::StringRef modPath)
     470             : {
     471        2808 :     auto it = module_cache->find(modPath);
     472        2808 :     if (it == module_cache->end())
     473             :     {
     474        1994 :         it = module_cache->insert(
     475        3988 :             std::make_pair(modPath, load_module(modPath))).first;
     476             :     }
     477             : 
     478        2808 :     return it->second.get();
     479             : }
     480             : 
     481             : static std::unique_ptr<llvm::Module>
     482        1994 : load_module(llvm::StringRef Identifier)
     483             : {
     484             :     LLVMMemoryBufferRef buf;
     485             :     LLVMModuleRef mod;
     486             :     char path[MAXPGPATH];
     487             :     char *msg;
     488             : 
     489        1994 :     snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
     490             : 
     491        1994 :     if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
     492           0 :         elog(FATAL, "failed to open bitcode file \"%s\": %s",
     493             :              path, msg);
     494        1994 :     if (LLVMGetBitcodeModuleInContext2(LLVMGetGlobalContext(), buf, &mod))
     495           0 :         elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
     496             : 
     497             :     /*
     498             :      * Currently there's no use in more detailed debug info for JITed
     499             :      * code. Until that changes, not much point in wasting memory and cycles
     500             :      * on processing debuginfo.
     501             :      */
     502        1994 :     llvm::StripDebugInfo(*llvm::unwrap(mod));
     503             : 
     504        1994 :     return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
     505             : }
     506             : 
     507             : /*
     508             :  * Compute list of referenced variables, functions and the instruction count
     509             :  * for a function.
     510             :  */
     511             : static void
     512        3070 : function_references(llvm::Function &F,
     513             :                     int &running_instcount,
     514             :                     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
     515             :                     llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
     516             : {
     517        6140 :     llvm::SmallPtrSet<const llvm::User *, 32> Visited;
     518             : 
     519       21468 :     for (llvm::BasicBlock &BB : F)
     520             :     {
     521      122380 :         for (llvm::Instruction &I : BB)
     522             :         {
     523      103982 :             if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
     524           0 :                 continue;
     525             : 
     526      207964 :             llvm::SmallVector<llvm::User *, 8> Worklist;
     527      103982 :             Worklist.push_back(&I);
     528             : 
     529      103982 :             running_instcount++;
     530             : 
     531      668442 :             while (!Worklist.empty()) {
     532      282230 :                 llvm::User *U = Worklist.pop_back_val();
     533             : 
     534             :                 /* visited before */
     535      282230 :                 if (!Visited.insert(U).second)
     536      137644 :                     continue;
     537             : 
     538      373784 :                 for (auto &OI : U->operands()) {
     539      229198 :                     llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
     540      229198 :                     if (!Operand)
     541       97224 :                         continue;
     542      189274 :                     if (llvm::isa<llvm::BlockAddress>(Operand))
     543           0 :                         continue;
     544      189274 :                     if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
     545        7048 :                         referencedVars.insert(GV);
     546        7048 :                         if (GV->hasInitializer())
     547        6350 :                             Worklist.push_back(GV->getInitializer());
     548        7048 :                         continue;
     549             :                     }
     550      182226 :                     if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
     551       10328 :                         referencedFunctions.insert(CF);
     552       10328 :                         continue;
     553             :                     }
     554      171898 :                     Worklist.push_back(Operand);
     555             :                 }
     556             :             }
     557             :         }
     558             :     }
     559        3070 : }
     560             : 
     561             : /*
     562             :  * Check whether function F is inlinable and, if so, what globals need to be
     563             :  * imported.
     564             :  *
     565             :  * References to external functions from, potentially recursively, inlined
     566             :  * functions are added to the passed in worklist.
     567             :  */
     568             : static bool
     569        3070 : function_inlinable(llvm::Function &F,
     570             :                    int threshold,
     571             :                    FunctionInlineStates &functionStates,
     572             :                    InlineWorkList &worklist,
     573             :                    InlineSearchPath &searchpath,
     574             :                    llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
     575             :                    int &running_instcount,
     576             :                    llvm::StringSet<> &importVars)
     577             : {
     578        3070 :     int subThreshold = threshold * inline_cost_decay_factor;
     579        6140 :     llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
     580        6140 :     llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
     581             : 
     582             :     /* can't rely on what may be inlined */
     583        3070 :     if (F.isInterposable())
     584           0 :         return false;
     585             : 
     586             :     /*
     587             :      * Can't rely on function being present. Alternatively we could create a
     588             :      * static version of these functions?
     589             :      */
     590        3070 :     if (F.hasAvailableExternallyLinkage())
     591           0 :         return false;
     592             : 
     593             :     ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
     594             : 
     595        3070 :     if (F.materialize())
     596           0 :         elog(FATAL, "failed to materialize metadata");
     597             : 
     598        3070 :     if (F.getAttributes().hasFnAttribute(llvm::Attribute::NoInline))
     599             :     {
     600             :         ilog(DEBUG1, "ineligibile to import %s due to noinline",
     601             :              F.getName().data());
     602           0 :         return false;
     603             :     }
     604             : 
     605        3070 :     function_references(F, running_instcount, referencedVars, referencedFunctions);
     606             : 
     607        6346 :     for (llvm::GlobalVariable* rv: referencedVars)
     608             :     {
     609        3718 :         if (rv->materialize())
     610           0 :             elog(FATAL, "failed to materialize metadata");
     611             : 
     612             :         /*
     613             :          * Never want to inline externally visible vars, cheap enough to
     614             :          * reference.
     615             :          */
     616        3718 :         if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
     617         438 :             continue;
     618             : 
     619             :         /*
     620             :          * If variable is file-local, we need to inline it, to be able to
     621             :          * inline the function itself. Can't do that if the variable can be
     622             :          * modified, because they'd obviously get out of sync.
     623             :          *
     624             :          * XXX: Currently not a problem, but there'd be problems with
     625             :          * nontrivial initializers if they were allowed for postgres.
     626             :          */
     627        3280 :         if (!rv->isConstant())
     628             :         {
     629             :             ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
     630             :                  F.getName().data(), rv->getName().data());
     631         442 :             return false;
     632             :         }
     633             : 
     634             :         ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
     635             :              rv->getName().data(), (int)rv->getLinkage());
     636             : 
     637        2838 :         importVars.insert(rv->getName());
     638             :         /* small cost attributed to each cloned global */
     639        2838 :         running_instcount += 5;
     640             :     }
     641             : 
     642        2628 :     visitedFunctions.insert(&F);
     643             : 
     644             :     /*
     645             :      * Check referenced functions. Check whether used static ones are
     646             :      * inlinable, and remember external ones for inlining.
     647             :      */
     648        8442 :     for (llvm::Function* referencedFunction: referencedFunctions)
     649             :     {
     650        6076 :         llvm::StringSet<> recImportVars;
     651             : 
     652        5846 :         if (referencedFunction->materialize())
     653           0 :             elog(FATAL, "failed to materialize metadata");
     654             : 
     655        5846 :         if (referencedFunction->isIntrinsic())
     656        1446 :             continue;
     657             : 
     658             :         /* if already visited skip, otherwise remember */
     659        4400 :         if (!visitedFunctions.insert(referencedFunction).second)
     660         270 :             continue;
     661             : 
     662             :         /*
     663             :          * We don't inline external functions directly here, instead we put
     664             :          * them on the worklist if appropriate and check them from
     665             :          * llvm_build_inline_plan().
     666             :          */
     667        4130 :         if (referencedFunction->hasExternalLinkage())
     668             :         {
     669        3868 :             llvm::StringRef funcName = referencedFunction->getName();
     670             : 
     671             :             /*
     672             :              * Don't bother checking for inlining if remaining cost budget is
     673             :              * very small.
     674             :              */
     675        3868 :             if (subThreshold < 5)
     676         260 :                 continue;
     677             : 
     678        3608 :             auto it = functionStates.find(funcName);
     679        3608 :             if (it == functionStates.end())
     680             :             {
     681             :                 FunctionInlineState inlineState;
     682             : 
     683        2644 :                 inlineState.costLimit = subThreshold;
     684        2644 :                 inlineState.processed = false;
     685        2644 :                 inlineState.inlined = false;
     686        2644 :                 inlineState.allowReconsidering = false;
     687             : 
     688        2644 :                 functionStates[funcName] = inlineState;
     689        2644 :                 worklist.push_back({funcName, searchpath});
     690             : 
     691             :                 ilog(DEBUG1,
     692             :                      "considering extern function %s at %d for inlining",
     693             :                      funcName.data(), subThreshold);
     694             :             }
     695        2862 :             else if (!it->second.inlined &&
     696        2122 :                      (!it->second.processed || it->second.allowReconsidering) &&
     697         292 :                      it->second.costLimit < subThreshold)
     698             :             {
     699             :                 /*
     700             :                  * Update inlining threshold if higher. Need to re-queue
     701             :                  * to be processed if already processed with lower
     702             :                  * threshold.
     703             :                  */
     704          12 :                 if (it->second.processed)
     705             :                 {
     706             :                     ilog(DEBUG1,
     707             :                          "reconsidering extern function %s at %d for inlining, increasing from %d",
     708             :                          funcName.data(), subThreshold, it->second.costLimit);
     709             : 
     710          12 :                     it->second.processed = false;
     711          12 :                     it->second.allowReconsidering = false;
     712          12 :                     worklist.push_back({funcName, searchpath});
     713             :                 }
     714          12 :                 it->second.costLimit = subThreshold;
     715             :             }
     716        3608 :             continue;
     717             :         }
     718             : 
     719             :         /* can't rely on what may be inlined */
     720         262 :         if (referencedFunction->isInterposable())
     721           0 :             return false;
     722             : 
     723         262 :         if (!function_inlinable(*referencedFunction,
     724             :                                 subThreshold,
     725             :                                 functionStates,
     726             :                                 worklist,
     727             :                                 searchpath,
     728             :                                 visitedFunctions,
     729             :                                 running_instcount,
     730             :                                 recImportVars))
     731             :         {
     732             :             ilog(DEBUG1,
     733             :                  "cannot inline %s due to required function %s not being inlinable",
     734             :                  F.getName().data(), referencedFunction->getName().data());
     735          32 :             return false;
     736             :         }
     737             : 
     738             :         /* import referenced function itself */
     739         230 :         importVars.insert(referencedFunction->getName());
     740             : 
     741             :         /* import referenced function and its dependants */
     742        1190 :         for (auto& recImportVar : recImportVars)
     743         960 :             importVars.insert(recImportVar.first());
     744             :     }
     745             : 
     746        2596 :     return true;
     747             : }
     748             : 
     749             : /*
     750             :  * Attempt to load module summary located at path. Return empty pointer when
     751             :  * loading fails.
     752             :  */
     753             : static std::unique_ptr<llvm::ModuleSummaryIndex>
     754         302 : llvm_load_summary(llvm::StringRef path)
     755             : {
     756             :     llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
     757         604 :         llvm::MemoryBuffer::getFile(path);
     758             : 
     759         302 :     if (std::error_code EC = MBOrErr.getError())
     760             :     {
     761             :         ilog(DEBUG1, "failed to open %s: %s", path.data(),
     762             :              EC.message().c_str());
     763             :     }
     764             :     else
     765             :     {
     766         302 :         llvm::MemoryBufferRef ref(*MBOrErr.get().get());
     767             : 
     768             : #if LLVM_VERSION_MAJOR > 3
     769             :         llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
     770         604 :             llvm::getModuleSummaryIndex(ref);
     771         302 :         if (IndexOrErr)
     772         302 :             return std::move(IndexOrErr.get());
     773           0 :         elog(FATAL, "failed to load summary \"%s\": %s",
     774             :              path.data(),
     775             :              toString(IndexOrErr.takeError()).c_str());
     776             : #else
     777             :         llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
     778             :             llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {});
     779             :         if (IndexOrErr)
     780             :             return std::move(IndexOrErr.get());
     781             :         elog(FATAL, "failed to load summary \"%s\": %s",
     782             :              path.data(),
     783             :              IndexOrErr.getError().message().c_str());
     784             : #endif
     785             :     }
     786           0 :     return nullptr;
     787             : }
     788             : 
     789             : /*
     790             :  * Attempt to add modpath to the search path.
     791             :  */
     792             : static void
     793         620 : add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
     794             : {
     795             :     /* only extension in libdir are candidates for inlining for now */
     796         620 :     if (!modpath.startswith("$libdir/"))
     797           0 :         return;
     798             : 
     799             :     /* if there's no match, attempt to load */
     800         620 :     auto it = summary_cache->find(modpath);
     801         620 :     if (it == summary_cache->end())
     802             :     {
     803         604 :         std::string path(modpath);
     804         302 :         path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
     805         302 :         path += ".index.bc";
     806         302 :         (*summary_cache)[modpath] = llvm_load_summary(path);
     807         302 :         it = summary_cache->find(modpath);
     808             :     }
     809             : 
     810             :     Assert(it != summary_cache->end());
     811             : 
     812             :     /* if the entry isn't NULL, it's validly loaded */
     813         620 :     if (it->second)
     814         620 :         searchpath.push_back(it->second.get());
     815             : }
     816             : 
     817             : /*
     818             :  * Search for all references for functions hashing to guid in the search path,
     819             :  * and return them in search path order.
     820             :  */
     821             : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
     822        4592 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
     823             : {
     824        4592 :     llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
     825             : 
     826       18608 :     for (auto index : path)
     827             :     {
     828             : #if LLVM_VERSION_MAJOR > 4
     829        4832 :         llvm::ValueInfo funcVI = index->getValueInfo(guid);
     830             : 
     831             :         /* if index doesn't know function, we don't have a body, continue */
     832        4832 :         if (funcVI)
     833        8924 :             for (auto &gv : funcVI.getSummaryList())
     834        4462 :                 matches.push_back(gv.get());
     835             : #else
     836             :         const llvm::const_gvsummary_iterator &I =
     837             :             index->findGlobalValueSummaryList(guid);
     838             :         if (I != index->end())
     839             :         {
     840             :             for (auto &gv : I->second)
     841             :                 matches.push_back(gv.get());
     842             :         }
     843             : #endif
     844             :     }
     845             : 
     846        4592 :     return matches;
     847             : }
     848             : 
     849             : /*
     850             :  * Create inline wrapper with the name Name, redirecting the call to F.
     851             :  */
     852             : static llvm::Function*
     853          22 : create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
     854             :                             llvm::Function *F,
     855             :                             llvm::StringRef Name)
     856             : {
     857             :     typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
     858             : 
     859          22 :     llvm::LLVMContext &Context = F->getContext();
     860          44 :     llvm::IRBuilder<> Builder(Context);
     861             :     llvm::Function *AF;
     862             :     llvm::BasicBlock *BB;
     863             :     llvm::CallInst *fwdcall;
     864          22 :     llvm::Attribute inlineAttribute;
     865             : 
     866          44 :     AF = llvm::Function::Create(F->getFunctionType(),
     867             :                                 LinkageTypes::AvailableExternallyLinkage,
     868          22 :                                 Name, importMod.get());
     869          22 :     BB = llvm::BasicBlock::Create(Context, "entry", AF);
     870             : 
     871          22 :     Builder.SetInsertPoint(BB);
     872          22 :     fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
     873             :     inlineAttribute = llvm::Attribute::get(Context,
     874          22 :                                            llvm::Attribute::AlwaysInline);
     875          22 :     fwdcall->addAttribute(~0U, inlineAttribute);
     876          22 :     Builder.CreateRet(fwdcall);
     877             : 
     878          44 :     return AF;
     879        3486 : }

Generated by: LCOV version 1.13