Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit_inline.cpp
4 : * Cross module inlining suitable for postgres' JIT
5 : *
6 : * The inliner iterates over external functions referenced from the passed
7 : * module and attempts to inline those. It does so by utilizing pre-built
8 : * indexes over both postgres core code and extension modules. When a match
9 : * for an external function is found - not guaranteed! - the index will then
10 : * be used to judge their instruction count / inline worthiness. After doing
11 : * so for all external functions, all the referenced functions (and
12 : * prerequisites) will be imported.
13 : *
14 : * Copyright (c) 2016-2023, PostgreSQL Global Development Group
15 : *
16 : * IDENTIFICATION
17 : * src/backend/lib/llvmjit/llvmjit_inline.cpp
18 : *
19 : *-------------------------------------------------------------------------
20 : */
21 :
22 : extern "C"
23 : {
24 : #include "postgres.h"
25 : }
26 :
27 : #include "jit/llvmjit.h"
28 :
29 : extern "C"
30 : {
31 : #include <fcntl.h>
32 : #include <sys/mman.h>
33 : #include <sys/stat.h>
34 : #include <sys/types.h>
35 : #include <unistd.h>
36 :
37 : #include "common/string.h"
38 : #include "miscadmin.h"
39 : #include "storage/fd.h"
40 : }
41 :
42 : #include <llvm-c/Core.h>
43 : #include <llvm-c/BitReader.h>
44 :
45 : /* Avoid macro clash with LLVM's C++ headers */
46 : #undef Min
47 :
48 : #include <llvm/ADT/SetVector.h>
49 : #include <llvm/ADT/StringSet.h>
50 : #include <llvm/ADT/StringMap.h>
51 : #include <llvm/Analysis/ModuleSummaryAnalysis.h>
52 : #if LLVM_VERSION_MAJOR > 3
53 : #include <llvm/Bitcode/BitcodeReader.h>
54 : #else
55 : #include <llvm/Bitcode/ReaderWriter.h>
56 : #include <llvm/Support/Error.h>
57 : #endif
58 : #include <llvm/IR/Attributes.h>
59 : #include <llvm/IR/DebugInfo.h>
60 : #include <llvm/IR/IntrinsicInst.h>
61 : #include <llvm/IR/IRBuilder.h>
62 : #include <llvm/IR/ModuleSummaryIndex.h>
63 : #include <llvm/Linker/IRMover.h>
64 : #include <llvm/Support/ManagedStatic.h>
65 : #include <llvm/Support/MemoryBuffer.h>
66 :
67 :
68 : /*
69 : * Type used to represent modules InlineWorkListItem's subject is searched for
70 : * in.
71 : */
72 : typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
73 :
74 : /*
75 : * Item in queue of to-be-checked symbols and corresponding queue.
76 : */
77 : typedef struct InlineWorkListItem
78 : {
79 : llvm::StringRef symbolName;
80 : llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
81 : } InlineWorkListItem;
82 : typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
83 :
84 : /*
85 : * Information about symbols processed during inlining. Used to prevent
86 : * repeated searches and provide additional information.
87 : */
88 : typedef struct FunctionInlineState
89 : {
90 : int costLimit;
91 : bool processed;
92 : bool inlined;
93 : bool allowReconsidering;
94 : } FunctionInlineState;
95 : typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
96 :
97 : /*
98 : * Map of modules that should be inlined, with a list of the to-be inlined
99 : * symbols.
100 : */
101 : typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
102 :
103 :
104 : const float inline_cost_decay_factor = 0.5;
105 : const int inline_initial_cost = 150;
106 :
107 : /*
108 : * These are managed statics so LLVM knows to deallocate them during an
109 : * LLVMShutdown(), rather than after (which'd cause crashes).
110 : */
111 : typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
112 : llvm::ManagedStatic<ModuleCache> module_cache;
113 : typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
114 : llvm::ManagedStatic<SummaryCache> summary_cache;
115 :
116 :
117 : static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod);
118 : static void llvm_execute_inline_plan(llvm::Module *mod,
119 : ImportMapTy *globalsToInline);
120 :
121 : static llvm::Module* load_module_cached(LLVMContextRef c, llvm::StringRef modPath);
122 : static std::unique_ptr<llvm::Module> load_module(LLVMContextRef c, llvm::StringRef Identifier);
123 : static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
124 :
125 :
126 : static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
127 : llvm::Function *F,
128 : llvm::StringRef Name);
129 :
130 : static bool function_inlinable(llvm::Function &F,
131 : int threshold,
132 : FunctionInlineStates &functionState,
133 : InlineWorkList &worklist,
134 : InlineSearchPath &searchpath,
135 : llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
136 : int &running_instcount,
137 : llvm::StringSet<> &importVars);
138 : static void function_references(llvm::Function &F,
139 : int &running_instcount,
140 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
141 : llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
142 :
143 : static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
144 : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
145 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
146 :
147 : /* verbose debugging for inliner development */
148 : /* #define INLINE_DEBUG */
149 : #ifdef INLINE_DEBUG
150 : #define ilog elog
151 : #else
152 : #define ilog(...) (void) 0
153 : #endif
154 :
155 : /*
156 : * Reset inlining related state. This needs to be called before the currently
157 : * used LLVMContextRef is disposed (and a new one create), otherwise we would
158 : * have dangling references to deleted modules.
159 : */
160 : void
161 0 : llvm_inline_reset_caches(void)
162 : {
163 0 : module_cache->clear();
164 0 : summary_cache->clear();
165 0 : }
166 :
167 : /*
168 : * Perform inlining of external function references in M based on a simple
169 : * cost based analysis.
170 : */
171 : void
172 978 : llvm_inline(LLVMModuleRef M)
173 : {
174 978 : LLVMContextRef lc = LLVMGetModuleContext(M);
175 978 : llvm::Module *mod = llvm::unwrap(M);
176 :
177 978 : std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
178 978 : if (!globalsToInline)
179 0 : return;
180 978 : llvm_execute_inline_plan(mod, globalsToInline.get());
181 : }
182 :
183 : /*
184 : * Build information necessary for inlining external function references in
185 : * mod.
186 : */
187 : static std::unique_ptr<ImportMapTy>
188 978 : llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
189 : {
190 1956 : std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
191 1956 : FunctionInlineStates functionStates;
192 1956 : InlineWorkList worklist;
193 :
194 1956 : InlineSearchPath defaultSearchPath;
195 :
196 : /* attempt to add module to search path */
197 978 : add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
198 : /* if postgres isn't available, no point continuing */
199 978 : if (defaultSearchPath.empty())
200 0 : return nullptr;
201 :
202 : /*
203 : * Start inlining with current references to external functions by putting
204 : * them on the inlining worklist. If, during inlining of those, new extern
205 : * functions need to be inlined, they'll also be put there, with a lower
206 : * priority.
207 : */
208 15128 : for (const llvm::Function &funcDecl : mod->functions())
209 : {
210 14150 : InlineWorkListItem item = {};
211 14150 : FunctionInlineState inlineState = {};
212 :
213 : /* already has a definition */
214 14150 : if (!funcDecl.isDeclaration())
215 9696 : continue;
216 :
217 : /* llvm provides implementation */
218 4454 : if (funcDecl.isIntrinsic())
219 794 : continue;
220 :
221 3660 : item.symbolName = funcDecl.getName();
222 3660 : item.searchpath = defaultSearchPath;
223 3660 : worklist.push_back(item);
224 3660 : inlineState.costLimit = inline_initial_cost;
225 3660 : inlineState.processed = false;
226 3660 : inlineState.inlined = false;
227 3660 : inlineState.allowReconsidering = false;
228 3660 : functionStates[funcDecl.getName()] = inlineState;
229 : }
230 :
231 : /*
232 : * Iterate over pending worklist items, look them up in index, check
233 : * whether they should be inlined.
234 : */
235 18504 : while (!worklist.empty())
236 : {
237 17526 : InlineWorkListItem item = worklist.pop_back_val();
238 17526 : llvm::StringRef symbolName = item.symbolName;
239 : char *cmodname;
240 : char *cfuncname;
241 17526 : FunctionInlineState &inlineState = functionStates[symbolName];
242 : llvm::GlobalValue::GUID funcGUID;
243 :
244 17526 : llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
245 :
246 17526 : funcGUID = llvm::GlobalValue::getGUID(cfuncname);
247 :
248 : /* already processed */
249 17526 : if (inlineState.processed)
250 0 : continue;
251 :
252 :
253 17526 : if (cmodname)
254 38 : add_module_to_inline_search_path(item.searchpath, cmodname);
255 :
256 : /*
257 : * Iterate over all known definitions of function, via the index. Then
258 : * look up module(s), check if function actually is defined (there
259 : * could be hash conflicts).
260 : */
261 34308 : for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
262 : {
263 : const llvm::FunctionSummary *fs;
264 16782 : llvm::StringRef modPath = gvs->modulePath();
265 : llvm::Module *defMod;
266 : llvm::Function *funcDef;
267 :
268 16782 : fs = llvm::cast<llvm::FunctionSummary>(gvs);
269 :
270 : #if LLVM_VERSION_MAJOR > 3
271 16782 : if (gvs->notEligibleToImport())
272 : {
273 : ilog(DEBUG1, "ineligibile to import %s due to summary",
274 : symbolName.data());
275 8420 : continue;
276 : }
277 : #endif
278 :
279 16782 : if ((int) fs->instCount() > inlineState.costLimit)
280 : {
281 : ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
282 : symbolName.data(), fs->instCount(), inlineState.costLimit);
283 7378 : inlineState.allowReconsidering = true;
284 7378 : continue;
285 : }
286 :
287 9404 : defMod = load_module_cached(lc, modPath);
288 9404 : if (defMod->materializeMetadata())
289 0 : elog(FATAL, "failed to materialize metadata");
290 :
291 9404 : funcDef = defMod->getFunction(cfuncname);
292 :
293 : /*
294 : * This can happen e.g. in case of a hash collision of the
295 : * function's name.
296 : */
297 9404 : if (!funcDef)
298 0 : continue;
299 :
300 9404 : if (funcDef->materialize())
301 0 : elog(FATAL, "failed to materialize metadata");
302 :
303 : Assert(!funcDef->isDeclaration());
304 : Assert(funcDef->hasExternalLinkage());
305 :
306 9404 : llvm::StringSet<> importVars;
307 9404 : llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
308 9404 : int running_instcount = 0;
309 :
310 : /*
311 : * Check whether function, and objects it depends on, are
312 : * inlinable.
313 : */
314 9404 : if (function_inlinable(*funcDef,
315 : inlineState.costLimit,
316 : functionStates,
317 : worklist,
318 : item.searchpath,
319 : visitedFunctions,
320 : running_instcount,
321 : importVars))
322 : {
323 : /*
324 : * Check whether function and all its dependencies are too
325 : * big. Dependencies already counted for other functions that
326 : * will get inlined are not counted again. While this make
327 : * things somewhat order dependent, I can't quite see a point
328 : * in a different behaviour.
329 : */
330 7910 : if (running_instcount > inlineState.costLimit)
331 : {
332 : ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
333 : symbolName.data(), running_instcount, inlineState.costLimit);
334 1042 : inlineState.allowReconsidering = true;
335 1042 : continue;
336 : }
337 :
338 : ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
339 : symbolName.data(), running_instcount, fs->instCount());
340 :
341 : /* import referenced function itself */
342 6868 : importVars.insert(symbolName);
343 :
344 : {
345 6868 : llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
346 17082 : for (auto& importVar : importVars)
347 10214 : modGlobalsToInline.insert(importVar.first());
348 : Assert(modGlobalsToInline.size() > 0);
349 : }
350 :
351 : /* mark function as inlined */
352 6868 : inlineState.inlined = true;
353 :
354 : /*
355 : * Found definition to inline, don't look for further
356 : * potential definitions.
357 : */
358 6868 : break;
359 : }
360 : else
361 : {
362 : ilog(DEBUG1, "had to skip inlining %s",
363 : symbolName.data());
364 :
365 : /* It's possible there's another definition that's inlinable. */
366 : }
367 : }
368 :
369 : /*
370 : * Signal that we're done with symbol, whether successful (inlined =
371 : * true above) or not.
372 : */
373 17526 : inlineState.processed = true;
374 : }
375 :
376 978 : return globalsToInline;
377 : }
378 :
379 : /*
380 : * Perform the actual inlining of external functions (and their dependencies)
381 : * into mod.
382 : */
383 : static void
384 978 : llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
385 : {
386 1956 : llvm::IRMover Mover(*mod);
387 :
388 5754 : for (const auto& toInline : *globalsToInline)
389 : {
390 4776 : const llvm::StringRef& modPath = toInline.first();
391 4776 : const llvm::StringSet<>& modGlobalsToInline = toInline.second;
392 4776 : llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
393 :
394 : Assert(module_cache->count(modPath));
395 4776 : std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
396 4776 : module_cache->erase(modPath);
397 :
398 4776 : if (modGlobalsToInline.empty())
399 0 : continue;
400 :
401 14814 : for (auto &glob: modGlobalsToInline)
402 : {
403 10038 : llvm::StringRef SymbolName = glob.first();
404 : char *modname;
405 : char *funcname;
406 :
407 10038 : llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
408 :
409 10038 : llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
410 :
411 10038 : if (!valueToImport)
412 0 : elog(FATAL, "didn't refind value %s to import", SymbolName.data());
413 :
414 : /*
415 : * For functions (global vars are only inlined if already static),
416 : * mark imported variables as being clones from other
417 : * functions. That a) avoids symbol conflicts b) allows the
418 : * optimizer to perform inlining.
419 : */
420 10038 : if (llvm::isa<llvm::Function>(valueToImport))
421 : {
422 6950 : llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
423 : typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
424 :
425 : /*
426 : * Per-function info isn't necessarily stripped yet, as the
427 : * module is lazy-loaded when stripped above.
428 : */
429 6950 : llvm::stripDebugInfo(*F);
430 :
431 : /*
432 : * If the to-be-imported function is one referenced including
433 : * its module name, create a tiny inline function that just
434 : * forwards the call. One might think a GlobalAlias would do
435 : * the trick, but a) IRMover doesn't override a declaration
436 : * with an alias pointing to a definition (instead renaming
437 : * it), b) Aliases can't be AvailableExternally.
438 : */
439 6950 : if (modname)
440 : {
441 : llvm::Function *AF;
442 :
443 34 : AF = create_redirection_function(importMod, F, SymbolName);
444 :
445 34 : GlobalsToImport.insert(AF);
446 34 : llvm::stripDebugInfo(*AF);
447 : }
448 :
449 6950 : if (valueToImport->hasExternalLinkage())
450 : {
451 6868 : valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
452 : }
453 : }
454 :
455 10038 : GlobalsToImport.insert(valueToImport);
456 : ilog(DEBUG1, "performing import of %s %s",
457 : modPath.data(), SymbolName.data());
458 :
459 : }
460 :
461 : #if LLVM_VERSION_MAJOR > 4
462 : #define IRMOVE_PARAMS , /*IsPerformingImport=*/false
463 : #elif LLVM_VERSION_MAJOR > 3
464 : #define IRMOVE_PARAMS , /*LinkModuleInlineAsm=*/false, /*IsPerformingImport=*/false
465 : #else
466 : #define IRMOVE_PARAMS
467 : #endif
468 4776 : if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
469 2380 : [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {}
470 9552 : IRMOVE_PARAMS))
471 0 : elog(FATAL, "function import failed with linker error");
472 : }
473 978 : }
474 :
475 : /*
476 : * Return a module identified by modPath, caching it in memory.
477 : *
478 : * Note that such a module may *not* be modified without copying, otherwise
479 : * the cache state would get corrupted.
480 : */
481 : static llvm::Module*
482 9404 : load_module_cached(LLVMContextRef lc, llvm::StringRef modPath)
483 : {
484 9404 : auto it = module_cache->find(modPath);
485 9404 : if (it == module_cache->end())
486 : {
487 4962 : it = module_cache->insert(
488 4962 : std::make_pair(modPath, load_module(lc, modPath))).first;
489 : }
490 :
491 18808 : return it->second.get();
492 : }
493 :
494 : static std::unique_ptr<llvm::Module>
495 4962 : load_module(LLVMContextRef lc, llvm::StringRef Identifier)
496 : {
497 : LLVMMemoryBufferRef buf;
498 : LLVMModuleRef mod;
499 : char path[MAXPGPATH];
500 : char *msg;
501 :
502 4962 : snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
503 :
504 4962 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
505 0 : elog(FATAL, "failed to open bitcode file \"%s\": %s",
506 : path, msg);
507 4962 : if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
508 0 : elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
509 :
510 : /*
511 : * Currently there's no use in more detailed debug info for JITed
512 : * code. Until that changes, not much point in wasting memory and cycles
513 : * on processing debuginfo.
514 : */
515 4962 : llvm::StripDebugInfo(*llvm::unwrap(mod));
516 :
517 9924 : return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
518 : }
519 :
520 : /*
521 : * Compute list of referenced variables, functions and the instruction count
522 : * for a function.
523 : */
524 : static void
525 14324 : function_references(llvm::Function &F,
526 : int &running_instcount,
527 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
528 : llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
529 : {
530 28648 : llvm::SmallPtrSet<const llvm::User *, 32> Visited;
531 :
532 169554 : for (llvm::BasicBlock &BB : F)
533 : {
534 1004592 : for (llvm::Instruction &I : BB)
535 : {
536 849362 : if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
537 0 : continue;
538 :
539 1698724 : llvm::SmallVector<llvm::User *, 8> Worklist;
540 849362 : Worklist.push_back(&I);
541 :
542 849362 : running_instcount++;
543 :
544 3140752 : while (!Worklist.empty()) {
545 2291390 : llvm::User *U = Worklist.pop_back_val();
546 :
547 : /* visited before */
548 2291390 : if (!Visited.insert(U).second)
549 1193412 : continue;
550 :
551 3023484 : for (auto &OI : U->operands()) {
552 1925506 : llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
553 1925506 : if (!Operand)
554 535062 : continue;
555 1593162 : if (llvm::isa<llvm::BlockAddress>(Operand))
556 0 : continue;
557 1593162 : if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
558 53932 : referencedVars.insert(GV);
559 53932 : if (GV->hasInitializer())
560 51584 : Worklist.push_back(GV->getInitializer());
561 53932 : continue;
562 : }
563 1539230 : if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
564 148786 : referencedFunctions.insert(CF);
565 148786 : continue;
566 : }
567 1390444 : Worklist.push_back(Operand);
568 : }
569 : }
570 : }
571 : }
572 14324 : }
573 :
574 : /*
575 : * Check whether function F is inlinable and, if so, what globals need to be
576 : * imported.
577 : *
578 : * References to external functions from, potentially recursively, inlined
579 : * functions are added to the passed in worklist.
580 : */
581 : static bool
582 14336 : function_inlinable(llvm::Function &F,
583 : int threshold,
584 : FunctionInlineStates &functionStates,
585 : InlineWorkList &worklist,
586 : InlineSearchPath &searchpath,
587 : llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
588 : int &running_instcount,
589 : llvm::StringSet<> &importVars)
590 : {
591 14336 : int subThreshold = threshold * inline_cost_decay_factor;
592 28672 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
593 28672 : llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
594 :
595 : /* can't rely on what may be inlined */
596 14336 : if (F.isInterposable())
597 0 : return false;
598 :
599 : /*
600 : * Can't rely on function being present. Alternatively we could create a
601 : * static version of these functions?
602 : */
603 14336 : if (F.hasAvailableExternallyLinkage())
604 0 : return false;
605 :
606 : ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
607 :
608 14336 : if (F.materialize())
609 0 : elog(FATAL, "failed to materialize metadata");
610 :
611 : #if LLVM_VERSION_MAJOR < 14
612 : #define hasFnAttr hasFnAttribute
613 : #endif
614 :
615 14336 : if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
616 : {
617 : ilog(DEBUG1, "ineligibile to import %s due to noinline",
618 : F.getName().data());
619 12 : return false;
620 : }
621 :
622 14324 : function_references(F, running_instcount, referencedVars, referencedFunctions);
623 :
624 52700 : for (llvm::GlobalVariable* rv: referencedVars)
625 : {
626 39858 : if (rv->materialize())
627 0 : elog(FATAL, "failed to materialize metadata");
628 :
629 : /*
630 : * Don't inline functions that access thread local variables. That
631 : * doesn't work on current LLVM releases (but might in future).
632 : */
633 39858 : if (rv->isThreadLocal())
634 : {
635 : ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
636 : F.getName().data(), rv->getName().data());
637 1482 : return false;
638 : }
639 :
640 : /*
641 : * Never want to inline externally visible vars, cheap enough to
642 : * reference.
643 : */
644 39858 : if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
645 1406 : continue;
646 :
647 : /*
648 : * If variable is file-local, we need to inline it, to be able to
649 : * inline the function itself. Can't do that if the variable can be
650 : * modified, because they'd obviously get out of sync.
651 : *
652 : * XXX: Currently not a problem, but there'd be problems with
653 : * nontrivial initializers if they were allowed for postgres.
654 : */
655 38452 : if (!rv->isConstant())
656 : {
657 : ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
658 : F.getName().data(), rv->getName().data());
659 1482 : return false;
660 : }
661 :
662 : ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
663 : rv->getName().data(), (int)rv->getLinkage());
664 :
665 36970 : importVars.insert(rv->getName());
666 : /* small cost attributed to each cloned global */
667 36970 : running_instcount += 5;
668 : }
669 :
670 12842 : visitedFunctions.insert(&F);
671 :
672 : /*
673 : * Check referenced functions. Check whether used static ones are
674 : * inlinable, and remember external ones for inlining.
675 : */
676 79838 : for (llvm::Function* referencedFunction: referencedFunctions)
677 : {
678 67130 : llvm::StringSet<> recImportVars;
679 :
680 67130 : if (referencedFunction->materialize())
681 0 : elog(FATAL, "failed to materialize metadata");
682 :
683 67130 : if (referencedFunction->isIntrinsic())
684 9972 : continue;
685 :
686 : /* if already visited skip, otherwise remember */
687 57158 : if (!visitedFunctions.insert(referencedFunction).second)
688 21976 : continue;
689 :
690 : /*
691 : * We don't inline external functions directly here, instead we put
692 : * them on the worklist if appropriate and check them from
693 : * llvm_build_inline_plan().
694 : */
695 35182 : if (referencedFunction->hasExternalLinkage())
696 : {
697 30250 : llvm::StringRef funcName = referencedFunction->getName();
698 :
699 : /*
700 : * Don't bother checking for inlining if remaining cost budget is
701 : * very small.
702 : */
703 30250 : if (subThreshold < 5)
704 6676 : continue;
705 :
706 23574 : auto it = functionStates.find(funcName);
707 23574 : if (it == functionStates.end())
708 : {
709 : FunctionInlineState inlineState;
710 :
711 13232 : inlineState.costLimit = subThreshold;
712 13232 : inlineState.processed = false;
713 13232 : inlineState.inlined = false;
714 13232 : inlineState.allowReconsidering = false;
715 :
716 13232 : functionStates[funcName] = inlineState;
717 13232 : worklist.push_back({funcName, searchpath});
718 :
719 : ilog(DEBUG1,
720 : "considering extern function %s at %d for inlining",
721 : funcName.data(), subThreshold);
722 : }
723 10342 : else if (!it->second.inlined &&
724 17596 : (!it->second.processed || it->second.allowReconsidering) &&
725 7254 : it->second.costLimit < subThreshold)
726 : {
727 : /*
728 : * Update inlining threshold if higher. Need to re-queue
729 : * to be processed if already processed with lower
730 : * threshold.
731 : */
732 642 : if (it->second.processed)
733 : {
734 : ilog(DEBUG1,
735 : "reconsidering extern function %s at %d for inlining, increasing from %d",
736 : funcName.data(), subThreshold, it->second.costLimit);
737 :
738 634 : it->second.processed = false;
739 634 : it->second.allowReconsidering = false;
740 634 : worklist.push_back({funcName, searchpath});
741 : }
742 642 : it->second.costLimit = subThreshold;
743 : }
744 23574 : continue;
745 : }
746 :
747 : /* can't rely on what may be inlined */
748 4932 : if (referencedFunction->isInterposable())
749 0 : return false;
750 :
751 4932 : if (!function_inlinable(*referencedFunction,
752 : subThreshold,
753 : functionStates,
754 : worklist,
755 : searchpath,
756 : visitedFunctions,
757 : running_instcount,
758 : recImportVars))
759 : {
760 : ilog(DEBUG1,
761 : "cannot inline %s due to required function %s not being inlinable",
762 : F.getName().data(), referencedFunction->getName().data());
763 134 : return false;
764 : }
765 :
766 : /* import referenced function itself */
767 4798 : importVars.insert(referencedFunction->getName());
768 :
769 : /* import referenced function and its dependents */
770 94420 : for (auto& recImportVar : recImportVars)
771 89622 : importVars.insert(recImportVar.first());
772 : }
773 :
774 12708 : return true;
775 : }
776 :
777 : /*
778 : * Attempt to load module summary located at path. Return empty pointer when
779 : * loading fails.
780 : */
781 : static std::unique_ptr<llvm::ModuleSummaryIndex>
782 488 : llvm_load_summary(llvm::StringRef path)
783 : {
784 : llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
785 976 : llvm::MemoryBuffer::getFile(path);
786 :
787 488 : if (std::error_code EC = MBOrErr.getError())
788 : {
789 : ilog(DEBUG1, "failed to open %s: %s", path.data(),
790 : EC.message().c_str());
791 : }
792 : else
793 : {
794 488 : llvm::MemoryBufferRef ref(*MBOrErr.get().get());
795 :
796 : #if LLVM_VERSION_MAJOR > 3
797 : llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
798 976 : llvm::getModuleSummaryIndex(ref);
799 488 : if (IndexOrErr)
800 488 : return std::move(IndexOrErr.get());
801 0 : elog(FATAL, "failed to load summary \"%s\": %s",
802 : path.data(),
803 : toString(IndexOrErr.takeError()).c_str());
804 : #else
805 : llvm::ErrorOr<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
806 : llvm::getModuleSummaryIndex(ref, [](const llvm::DiagnosticInfo &) {});
807 : if (IndexOrErr)
808 : return std::move(IndexOrErr.get());
809 : elog(FATAL, "failed to load summary \"%s\": %s",
810 : path.data(),
811 : IndexOrErr.getError().message().c_str());
812 : #endif
813 : }
814 0 : return nullptr;
815 : }
816 :
817 : /*
818 : * Attempt to add modpath to the search path.
819 : */
820 : static void
821 1016 : add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
822 : {
823 : /* only extension in libdir are candidates for inlining for now */
824 1016 : if (!modpath.startswith("$libdir/"))
825 0 : return;
826 :
827 : /* if there's no match, attempt to load */
828 1016 : auto it = summary_cache->find(modpath);
829 1016 : if (it == summary_cache->end())
830 : {
831 976 : std::string path(modpath);
832 488 : path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
833 488 : path += ".index.bc";
834 488 : (*summary_cache)[modpath] = llvm_load_summary(path);
835 488 : it = summary_cache->find(modpath);
836 : }
837 :
838 : Assert(it != summary_cache->end());
839 :
840 : /* if the entry isn't NULL, it's validly loaded */
841 1016 : if (it->second)
842 1016 : searchpath.push_back(it->second.get());
843 : }
844 :
845 : /*
846 : * Search for all references for functions hashing to guid in the search path,
847 : * and return them in search path order.
848 : */
849 : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
850 17526 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
851 : {
852 17526 : llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
853 :
854 36204 : for (auto index : path)
855 : {
856 : #if LLVM_VERSION_MAJOR > 4
857 18678 : llvm::ValueInfo funcVI = index->getValueInfo(guid);
858 :
859 : /* if index doesn't know function, we don't have a body, continue */
860 18678 : if (funcVI)
861 33564 : for (auto &gv : funcVI.getSummaryList())
862 16782 : matches.push_back(gv.get());
863 : #else
864 : const llvm::const_gvsummary_iterator &I =
865 : index->findGlobalValueSummaryList(guid);
866 : if (I != index->end())
867 : {
868 : for (auto &gv : I->second)
869 : matches.push_back(gv.get());
870 : }
871 : #endif
872 : }
873 :
874 17526 : return matches;
875 : }
876 :
877 : /*
878 : * Create inline wrapper with the name Name, redirecting the call to F.
879 : */
880 : static llvm::Function*
881 34 : create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
882 : llvm::Function *F,
883 : llvm::StringRef Name)
884 : {
885 : typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
886 :
887 34 : llvm::LLVMContext &Context = F->getContext();
888 34 : llvm::IRBuilder<> Builder(Context);
889 : llvm::Function *AF;
890 : llvm::BasicBlock *BB;
891 : llvm::CallInst *fwdcall;
892 : #if LLVM_VERSION_MAJOR < 14
893 34 : llvm::Attribute inlineAttribute;
894 : #endif
895 :
896 68 : AF = llvm::Function::Create(F->getFunctionType(),
897 : LinkageTypes::AvailableExternallyLinkage,
898 34 : Name, importMod.get());
899 34 : BB = llvm::BasicBlock::Create(Context, "entry", AF);
900 :
901 34 : Builder.SetInsertPoint(BB);
902 34 : fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
903 : #if LLVM_VERSION_MAJOR < 14
904 : inlineAttribute = llvm::Attribute::get(Context,
905 34 : llvm::Attribute::AlwaysInline);
906 34 : fwdcall->addAttribute(~0U, inlineAttribute);
907 : #else
908 : fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
909 : #endif
910 34 : Builder.CreateRet(fwdcall);
911 :
912 68 : return AF;
913 : }
|
