Line data Source code
1 : /*-------------------------------------------------------------------------
2 : *
3 : * llvmjit_inline.cpp
4 : * Cross module inlining suitable for postgres' JIT
5 : *
6 : * The inliner iterates over external functions referenced from the passed
7 : * module and attempts to inline those. It does so by utilizing pre-built
8 : * indexes over both postgres core code and extension modules. When a match
9 : * for an external function is found - not guaranteed! - the index will then
10 : * be used to judge their instruction count / inline worthiness. After doing
11 : * so for all external functions, all the referenced functions (and
12 : * prerequisites) will be imported.
13 : *
14 : * Copyright (c) 2016-2024, PostgreSQL Global Development Group
15 : *
16 : * IDENTIFICATION
17 : * src/backend/lib/llvmjit/llvmjit_inline.cpp
18 : *
19 : *-------------------------------------------------------------------------
20 : */
21 :
22 : extern "C"
23 : {
24 : #include "postgres.h"
25 : }
26 :
27 : #include "jit/llvmjit.h"
28 :
29 : extern "C"
30 : {
31 : #include <fcntl.h>
32 : #include <sys/mman.h>
33 : #include <sys/stat.h>
34 : #include <sys/types.h>
35 : #include <unistd.h>
36 :
37 : #include "common/string.h"
38 : #include "miscadmin.h"
39 : #include "storage/fd.h"
40 : }
41 :
42 : #include <llvm-c/Core.h>
43 : #include <llvm-c/BitReader.h>
44 :
45 : /* Avoid macro clash with LLVM's C++ headers */
46 : #undef Min
47 :
48 : #include <llvm/ADT/SetVector.h>
49 : #include <llvm/ADT/StringSet.h>
50 : #include <llvm/ADT/StringMap.h>
51 : #include <llvm/Analysis/ModuleSummaryAnalysis.h>
52 : #include <llvm/Bitcode/BitcodeReader.h>
53 : #include <llvm/IR/Attributes.h>
54 : #include <llvm/IR/DebugInfo.h>
55 : #include <llvm/IR/IntrinsicInst.h>
56 : #include <llvm/IR/IRBuilder.h>
57 : #include <llvm/IR/ModuleSummaryIndex.h>
58 : #include <llvm/Linker/IRMover.h>
59 : #include <llvm/Support/ManagedStatic.h>
60 : #include <llvm/Support/MemoryBuffer.h>
61 :
62 :
63 : /*
64 : * Type used to represent modules InlineWorkListItem's subject is searched for
65 : * in.
66 : */
67 : typedef llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> InlineSearchPath;
68 :
69 : /*
70 : * Item in queue of to-be-checked symbols and corresponding queue.
71 : */
72 : typedef struct InlineWorkListItem
73 : {
74 : llvm::StringRef symbolName;
75 : llvm::SmallVector<llvm::ModuleSummaryIndex *, 2> searchpath;
76 : } InlineWorkListItem;
77 : typedef llvm::SmallVector<InlineWorkListItem, 128> InlineWorkList;
78 :
79 : /*
80 : * Information about symbols processed during inlining. Used to prevent
81 : * repeated searches and provide additional information.
82 : */
83 : typedef struct FunctionInlineState
84 : {
85 : int costLimit;
86 : bool processed;
87 : bool inlined;
88 : bool allowReconsidering;
89 : } FunctionInlineState;
90 : typedef llvm::StringMap<FunctionInlineState> FunctionInlineStates;
91 :
92 : /*
93 : * Map of modules that should be inlined, with a list of the to-be inlined
94 : * symbols.
95 : */
96 : typedef llvm::StringMap<llvm::StringSet<> > ImportMapTy;
97 :
98 :
99 : const float inline_cost_decay_factor = 0.5;
100 : const int inline_initial_cost = 150;
101 :
102 : /*
103 : * These are managed statics so LLVM knows to deallocate them during an
104 : * LLVMShutdown(), rather than after (which'd cause crashes).
105 : */
106 : typedef llvm::StringMap<std::unique_ptr<llvm::Module> > ModuleCache;
107 : llvm::ManagedStatic<ModuleCache> module_cache;
108 : typedef llvm::StringMap<std::unique_ptr<llvm::ModuleSummaryIndex> > SummaryCache;
109 : llvm::ManagedStatic<SummaryCache> summary_cache;
110 :
111 :
112 : static std::unique_ptr<ImportMapTy> llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod);
113 : static void llvm_execute_inline_plan(llvm::Module *mod,
114 : ImportMapTy *globalsToInline);
115 :
116 : static llvm::Module* load_module_cached(LLVMContextRef c, llvm::StringRef modPath);
117 : static std::unique_ptr<llvm::Module> load_module(LLVMContextRef c, llvm::StringRef Identifier);
118 : static std::unique_ptr<llvm::ModuleSummaryIndex> llvm_load_summary(llvm::StringRef path);
119 :
120 :
121 : static llvm::Function* create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
122 : llvm::Function *F,
123 : llvm::StringRef Name);
124 :
125 : static bool function_inlinable(llvm::Function &F,
126 : int threshold,
127 : FunctionInlineStates &functionState,
128 : InlineWorkList &worklist,
129 : InlineSearchPath &searchpath,
130 : llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
131 : int &running_instcount,
132 : llvm::StringSet<> &importVars);
133 : static void function_references(llvm::Function &F,
134 : int &running_instcount,
135 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
136 : llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions);
137 :
138 : static void add_module_to_inline_search_path(InlineSearchPath& path, llvm::StringRef modpath);
139 : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
140 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid);
141 :
142 : /* verbose debugging for inliner development */
143 : /* #define INLINE_DEBUG */
144 : #ifdef INLINE_DEBUG
145 : #define ilog elog
146 : #else
147 : #define ilog(...) (void) 0
148 : #endif
149 :
150 : /*
151 : * Reset inlining related state. This needs to be called before the currently
152 : * used LLVMContextRef is disposed (and a new one create), otherwise we would
153 : * have dangling references to deleted modules.
154 : */
155 : void
156 0 : llvm_inline_reset_caches(void)
157 : {
158 0 : module_cache->clear();
159 0 : summary_cache->clear();
160 0 : }
161 :
162 : /*
163 : * Perform inlining of external function references in M based on a simple
164 : * cost based analysis.
165 : */
166 : void
167 982 : llvm_inline(LLVMModuleRef M)
168 : {
169 982 : LLVMContextRef lc = LLVMGetModuleContext(M);
170 982 : llvm::Module *mod = llvm::unwrap(M);
171 :
172 982 : std::unique_ptr<ImportMapTy> globalsToInline = llvm_build_inline_plan(lc, mod);
173 982 : if (!globalsToInline)
174 0 : return;
175 982 : llvm_execute_inline_plan(mod, globalsToInline.get());
176 : }
177 :
178 : /*
179 : * Build information necessary for inlining external function references in
180 : * mod.
181 : */
182 : static std::unique_ptr<ImportMapTy>
183 982 : llvm_build_inline_plan(LLVMContextRef lc, llvm::Module *mod)
184 : {
185 1964 : std::unique_ptr<ImportMapTy> globalsToInline(new ImportMapTy());
186 1964 : FunctionInlineStates functionStates;
187 1964 : InlineWorkList worklist;
188 :
189 1964 : InlineSearchPath defaultSearchPath;
190 :
191 : /* attempt to add module to search path */
192 982 : add_module_to_inline_search_path(defaultSearchPath, "$libdir/postgres");
193 : /* if postgres isn't available, no point continuing */
194 982 : if (defaultSearchPath.empty())
195 0 : return nullptr;
196 :
197 : /*
198 : * Start inlining with current references to external functions by putting
199 : * them on the inlining worklist. If, during inlining of those, new extern
200 : * functions need to be inlined, they'll also be put there, with a lower
201 : * priority.
202 : */
203 15286 : for (const llvm::Function &funcDecl : mod->functions())
204 : {
205 14304 : InlineWorkListItem item = {};
206 14304 : FunctionInlineState inlineState = {};
207 :
208 : /* already has a definition */
209 14304 : if (!funcDecl.isDeclaration())
210 9826 : continue;
211 :
212 : /* llvm provides implementation */
213 4478 : if (funcDecl.isIntrinsic())
214 798 : continue;
215 :
216 3680 : item.symbolName = funcDecl.getName();
217 3680 : item.searchpath = defaultSearchPath;
218 3680 : worklist.push_back(item);
219 3680 : inlineState.costLimit = inline_initial_cost;
220 3680 : inlineState.processed = false;
221 3680 : inlineState.inlined = false;
222 3680 : inlineState.allowReconsidering = false;
223 3680 : functionStates[funcDecl.getName()] = inlineState;
224 : }
225 :
226 : /*
227 : * Iterate over pending worklist items, look them up in index, check
228 : * whether they should be inlined.
229 : */
230 19072 : while (!worklist.empty())
231 : {
232 18090 : InlineWorkListItem item = worklist.pop_back_val();
233 18090 : llvm::StringRef symbolName = item.symbolName;
234 : char *cmodname;
235 : char *cfuncname;
236 18090 : FunctionInlineState &inlineState = functionStates[symbolName];
237 : llvm::GlobalValue::GUID funcGUID;
238 :
239 18090 : llvm_split_symbol_name(symbolName.data(), &cmodname, &cfuncname);
240 :
241 18090 : funcGUID = llvm::GlobalValue::getGUID(cfuncname);
242 :
243 : /* already processed */
244 18090 : if (inlineState.processed)
245 0 : continue;
246 :
247 :
248 18090 : if (cmodname)
249 20 : add_module_to_inline_search_path(item.searchpath, cmodname);
250 :
251 : /*
252 : * Iterate over all known definitions of function, via the index. Then
253 : * look up module(s), check if function actually is defined (there
254 : * could be hash conflicts).
255 : */
256 35458 : for (const auto &gvs : summaries_for_guid(item.searchpath, funcGUID))
257 : {
258 : const llvm::FunctionSummary *fs;
259 17368 : llvm::StringRef modPath = gvs->modulePath();
260 : llvm::Module *defMod;
261 : llvm::Function *funcDef;
262 :
263 17368 : fs = llvm::cast<llvm::FunctionSummary>(gvs);
264 :
265 17368 : if (gvs->notEligibleToImport())
266 : {
267 : ilog(DEBUG1, "ineligibile to import %s due to summary",
268 : symbolName.data());
269 8536 : continue;
270 : }
271 :
272 17368 : if ((int) fs->instCount() > inlineState.costLimit)
273 : {
274 : ilog(DEBUG1, "ineligibile to import %s due to early threshold: %u vs %u",
275 : symbolName.data(), fs->instCount(), inlineState.costLimit);
276 7500 : inlineState.allowReconsidering = true;
277 7500 : continue;
278 : }
279 :
280 9868 : defMod = load_module_cached(lc, modPath);
281 9868 : if (defMod->materializeMetadata())
282 0 : elog(FATAL, "failed to materialize metadata");
283 :
284 9868 : funcDef = defMod->getFunction(cfuncname);
285 :
286 : /*
287 : * This can happen e.g. in case of a hash collision of the
288 : * function's name.
289 : */
290 9868 : if (!funcDef)
291 0 : continue;
292 :
293 9868 : if (funcDef->materialize())
294 0 : elog(FATAL, "failed to materialize metadata");
295 :
296 : Assert(!funcDef->isDeclaration());
297 : Assert(funcDef->hasExternalLinkage());
298 :
299 9868 : llvm::StringSet<> importVars;
300 9868 : llvm::SmallPtrSet<const llvm::Function *, 8> visitedFunctions;
301 9868 : int running_instcount = 0;
302 :
303 : /*
304 : * Check whether function, and objects it depends on, are
305 : * inlinable.
306 : */
307 9868 : if (function_inlinable(*funcDef,
308 : inlineState.costLimit,
309 : functionStates,
310 : worklist,
311 : item.searchpath,
312 : visitedFunctions,
313 : running_instcount,
314 : importVars))
315 : {
316 : /*
317 : * Check whether function and all its dependencies are too
318 : * big. Dependencies already counted for other functions that
319 : * will get inlined are not counted again. While this make
320 : * things somewhat order dependent, I can't quite see a point
321 : * in a different behaviour.
322 : */
323 8220 : if (running_instcount > inlineState.costLimit)
324 : {
325 : ilog(DEBUG1, "skipping inlining of %s due to late threshold %d vs %d",
326 : symbolName.data(), running_instcount, inlineState.costLimit);
327 1036 : inlineState.allowReconsidering = true;
328 1036 : continue;
329 : }
330 :
331 : ilog(DEBUG1, "inline top function %s total_instcount: %d, partial: %d",
332 : symbolName.data(), running_instcount, fs->instCount());
333 :
334 : /* import referenced function itself */
335 7184 : importVars.insert(symbolName);
336 :
337 : {
338 7184 : llvm::StringSet<> &modGlobalsToInline = (*globalsToInline)[modPath];
339 18632 : for (auto& importVar : importVars)
340 11448 : modGlobalsToInline.insert(importVar.first());
341 : Assert(modGlobalsToInline.size() > 0);
342 : }
343 :
344 : /* mark function as inlined */
345 7184 : inlineState.inlined = true;
346 :
347 : /*
348 : * Found definition to inline, don't look for further
349 : * potential definitions.
350 : */
351 7184 : break;
352 : }
353 : else
354 : {
355 : ilog(DEBUG1, "had to skip inlining %s",
356 : symbolName.data());
357 :
358 : /* It's possible there's another definition that's inlinable. */
359 : }
360 : }
361 :
362 : /*
363 : * Signal that we're done with symbol, whether successful (inlined =
364 : * true above) or not.
365 : */
366 18090 : inlineState.processed = true;
367 : }
368 :
369 982 : return globalsToInline;
370 : }
371 :
372 : /*
373 : * Perform the actual inlining of external functions (and their dependencies)
374 : * into mod.
375 : */
376 : static void
377 982 : llvm_execute_inline_plan(llvm::Module *mod, ImportMapTy *globalsToInline)
378 : {
379 1964 : llvm::IRMover Mover(*mod);
380 :
381 5802 : for (const auto& toInline : *globalsToInline)
382 : {
383 4820 : const llvm::StringRef& modPath = toInline.first();
384 4820 : const llvm::StringSet<>& modGlobalsToInline = toInline.second;
385 4820 : llvm::SetVector<llvm::GlobalValue *> GlobalsToImport;
386 :
387 : Assert(module_cache->count(modPath));
388 4820 : std::unique_ptr<llvm::Module> importMod(std::move((*module_cache)[modPath]));
389 4820 : module_cache->erase(modPath);
390 :
391 4820 : if (modGlobalsToInline.empty())
392 0 : continue;
393 :
394 15558 : for (auto &glob: modGlobalsToInline)
395 : {
396 10738 : llvm::StringRef SymbolName = glob.first();
397 : char *modname;
398 : char *funcname;
399 :
400 10738 : llvm_split_symbol_name(SymbolName.data(), &modname, &funcname);
401 :
402 10738 : llvm::GlobalValue *valueToImport = importMod->getNamedValue(funcname);
403 :
404 10738 : if (!valueToImport)
405 0 : elog(FATAL, "didn't refind value %s to import", SymbolName.data());
406 :
407 : /*
408 : * For functions (global vars are only inlined if already static),
409 : * mark imported variables as being clones from other
410 : * functions. That a) avoids symbol conflicts b) allows the
411 : * optimizer to perform inlining.
412 : */
413 10738 : if (llvm::isa<llvm::Function>(valueToImport))
414 : {
415 7250 : llvm::Function *F = llvm::dyn_cast<llvm::Function>(valueToImport);
416 : typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
417 :
418 : /*
419 : * Per-function info isn't necessarily stripped yet, as the
420 : * module is lazy-loaded when stripped above.
421 : */
422 7250 : llvm::stripDebugInfo(*F);
423 :
424 : /*
425 : * If the to-be-imported function is one referenced including
426 : * its module name, create a tiny inline function that just
427 : * forwards the call. One might think a GlobalAlias would do
428 : * the trick, but a) IRMover doesn't override a declaration
429 : * with an alias pointing to a definition (instead renaming
430 : * it), b) Aliases can't be AvailableExternally.
431 : */
432 7250 : if (modname)
433 : {
434 : llvm::Function *AF;
435 :
436 18 : AF = create_redirection_function(importMod, F, SymbolName);
437 :
438 18 : GlobalsToImport.insert(AF);
439 18 : llvm::stripDebugInfo(*AF);
440 : }
441 :
442 7250 : if (valueToImport->hasExternalLinkage())
443 : {
444 7184 : valueToImport->setLinkage(LinkageTypes::AvailableExternallyLinkage);
445 : }
446 : }
447 :
448 10738 : GlobalsToImport.insert(valueToImport);
449 : ilog(DEBUG1, "performing import of %s %s",
450 : modPath.data(), SymbolName.data());
451 :
452 : }
453 :
454 4820 : if (Mover.move(std::move(importMod), GlobalsToImport.getArrayRef(),
455 2040 : [](llvm::GlobalValue &, llvm::IRMover::ValueAdder) {},
456 9640 : /*IsPerformingImport=*/false))
457 0 : elog(FATAL, "function import failed with linker error");
458 : }
459 982 : }
460 :
461 : /*
462 : * Return a module identified by modPath, caching it in memory.
463 : *
464 : * Note that such a module may *not* be modified without copying, otherwise
465 : * the cache state would get corrupted.
466 : */
467 : static llvm::Module*
468 9868 : load_module_cached(LLVMContextRef lc, llvm::StringRef modPath)
469 : {
470 9868 : auto it = module_cache->find(modPath);
471 9868 : if (it == module_cache->end())
472 : {
473 5018 : it = module_cache->insert(
474 5018 : std::make_pair(modPath, load_module(lc, modPath))).first;
475 : }
476 :
477 19736 : return it->second.get();
478 : }
479 :
480 : static std::unique_ptr<llvm::Module>
481 5018 : load_module(LLVMContextRef lc, llvm::StringRef Identifier)
482 : {
483 : LLVMMemoryBufferRef buf;
484 : LLVMModuleRef mod;
485 : char path[MAXPGPATH];
486 : char *msg;
487 :
488 5018 : snprintf(path, MAXPGPATH,"%s/bitcode/%s", pkglib_path, Identifier.data());
489 :
490 5018 : if (LLVMCreateMemoryBufferWithContentsOfFile(path, &buf, &msg))
491 0 : elog(FATAL, "failed to open bitcode file \"%s\": %s",
492 : path, msg);
493 5018 : if (LLVMGetBitcodeModuleInContext2(lc, buf, &mod))
494 0 : elog(FATAL, "failed to parse bitcode in file \"%s\"", path);
495 :
496 : /*
497 : * Currently there's no use in more detailed debug info for JITed
498 : * code. Until that changes, not much point in wasting memory and cycles
499 : * on processing debuginfo.
500 : */
501 5018 : llvm::StripDebugInfo(*llvm::unwrap(mod));
502 :
503 10036 : return std::unique_ptr<llvm::Module>(llvm::unwrap(mod));
504 : }
505 :
506 : /*
507 : * Compute list of referenced variables, functions and the instruction count
508 : * for a function.
509 : */
510 : static void
511 14906 : function_references(llvm::Function &F,
512 : int &running_instcount,
513 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> &referencedVars,
514 : llvm::SmallPtrSet<llvm::Function *, 8> &referencedFunctions)
515 : {
516 29812 : llvm::SmallPtrSet<const llvm::User *, 32> Visited;
517 :
518 179516 : for (llvm::BasicBlock &BB : F)
519 : {
520 1048352 : for (llvm::Instruction &I : BB)
521 : {
522 883742 : if (llvm::isa<llvm::DbgInfoIntrinsic>(I))
523 0 : continue;
524 :
525 1767484 : llvm::SmallVector<llvm::User *, 8> Worklist;
526 883742 : Worklist.push_back(&I);
527 :
528 883742 : running_instcount++;
529 :
530 3272452 : while (!Worklist.empty()) {
531 2388710 : llvm::User *U = Worklist.pop_back_val();
532 :
533 : /* visited before */
534 2388710 : if (!Visited.insert(U).second)
535 1245892 : continue;
536 :
537 3156144 : for (auto &OI : U->operands()) {
538 2013326 : llvm::User *Operand = llvm::dyn_cast<llvm::User>(OI);
539 2013326 : if (!Operand)
540 563036 : continue;
541 1662646 : if (llvm::isa<llvm::BlockAddress>(Operand))
542 0 : continue;
543 1662646 : if (auto *GV = llvm::dyn_cast<llvm::GlobalVariable>(Operand)) {
544 56516 : referencedVars.insert(GV);
545 56516 : if (GV->hasInitializer())
546 54678 : Worklist.push_back(GV->getInitializer());
547 56516 : continue;
548 : }
549 1606130 : if (auto *CF = llvm::dyn_cast<llvm::Function>(Operand)) {
550 155840 : referencedFunctions.insert(CF);
551 155840 : continue;
552 : }
553 1450290 : Worklist.push_back(Operand);
554 : }
555 : }
556 : }
557 : }
558 14906 : }
559 :
560 : /*
561 : * Check whether function F is inlinable and, if so, what globals need to be
562 : * imported.
563 : *
564 : * References to external functions from, potentially recursively, inlined
565 : * functions are added to the passed in worklist.
566 : */
567 : static bool
568 15002 : function_inlinable(llvm::Function &F,
569 : int threshold,
570 : FunctionInlineStates &functionStates,
571 : InlineWorkList &worklist,
572 : InlineSearchPath &searchpath,
573 : llvm::SmallPtrSet<const llvm::Function *, 8> &visitedFunctions,
574 : int &running_instcount,
575 : llvm::StringSet<> &importVars)
576 : {
577 15002 : int subThreshold = threshold * inline_cost_decay_factor;
578 30004 : llvm::SmallPtrSet<llvm::GlobalVariable *, 8> referencedVars;
579 30004 : llvm::SmallPtrSet<llvm::Function *, 8> referencedFunctions;
580 :
581 : /* can't rely on what may be inlined */
582 15002 : if (F.isInterposable())
583 0 : return false;
584 :
585 : /*
586 : * Can't rely on function being present. Alternatively we could create a
587 : * static version of these functions?
588 : */
589 15002 : if (F.hasAvailableExternallyLinkage())
590 0 : return false;
591 :
592 : ilog(DEBUG1, "checking inlinability of %s", F.getName().data());
593 :
594 15002 : if (F.materialize())
595 0 : elog(FATAL, "failed to materialize metadata");
596 :
597 : #if LLVM_VERSION_MAJOR < 14
598 : #define hasFnAttr hasFnAttribute
599 : #endif
600 :
601 15002 : if (F.getAttributes().hasFnAttr(llvm::Attribute::NoInline))
602 : {
603 : ilog(DEBUG1, "ineligibile to import %s due to noinline",
604 : F.getName().data());
605 96 : return false;
606 : }
607 :
608 14906 : function_references(F, running_instcount, referencedVars, referencedFunctions);
609 :
610 55812 : for (llvm::GlobalVariable* rv: referencedVars)
611 : {
612 42458 : if (rv->materialize())
613 0 : elog(FATAL, "failed to materialize metadata");
614 :
615 : /*
616 : * Don't inline functions that access thread local variables. That
617 : * doesn't work on current LLVM releases (but might in future).
618 : */
619 42458 : if (rv->isThreadLocal())
620 : {
621 : ilog(DEBUG1, "cannot inline %s due to thread-local variable %s",
622 : F.getName().data(), rv->getName().data());
623 1552 : return false;
624 : }
625 :
626 : /*
627 : * Never want to inline externally visible vars, cheap enough to
628 : * reference.
629 : */
630 42458 : if (rv->hasExternalLinkage() || rv->hasAvailableExternallyLinkage())
631 816 : continue;
632 :
633 : /*
634 : * If variable is file-local, we need to inline it, to be able to
635 : * inline the function itself. Can't do that if the variable can be
636 : * modified, because they'd obviously get out of sync.
637 : *
638 : * XXX: Currently not a problem, but there'd be problems with
639 : * nontrivial initializers if they were allowed for postgres.
640 : */
641 41642 : if (!rv->isConstant())
642 : {
643 : ilog(DEBUG1, "cannot inline %s due to uncloneable variable %s",
644 : F.getName().data(), rv->getName().data());
645 1552 : return false;
646 : }
647 :
648 : ilog(DEBUG1, "memorizing global var %s linkage %d for inlining",
649 : rv->getName().data(), (int)rv->getLinkage());
650 :
651 40090 : importVars.insert(rv->getName());
652 : /* small cost attributed to each cloned global */
653 40090 : running_instcount += 5;
654 : }
655 :
656 13354 : visitedFunctions.insert(&F);
657 :
658 : /*
659 : * Check referenced functions. Check whether used static ones are
660 : * inlinable, and remember external ones for inlining.
661 : */
662 84146 : for (llvm::Function* referencedFunction: referencedFunctions)
663 : {
664 71056 : llvm::StringSet<> recImportVars;
665 :
666 71056 : if (referencedFunction->materialize())
667 0 : elog(FATAL, "failed to materialize metadata");
668 :
669 71056 : if (referencedFunction->isIntrinsic())
670 9400 : continue;
671 :
672 : /* if already visited skip, otherwise remember */
673 61656 : if (!visitedFunctions.insert(referencedFunction).second)
674 23042 : continue;
675 :
676 : /*
677 : * We don't inline external functions directly here, instead we put
678 : * them on the worklist if appropriate and check them from
679 : * llvm_build_inline_plan().
680 : */
681 38614 : if (referencedFunction->hasExternalLinkage())
682 : {
683 33480 : llvm::StringRef funcName = referencedFunction->getName();
684 :
685 : /*
686 : * Don't bother checking for inlining if remaining cost budget is
687 : * very small.
688 : */
689 33480 : if (subThreshold < 5)
690 7118 : continue;
691 :
692 26362 : auto it = functionStates.find(funcName);
693 26362 : if (it == functionStates.end())
694 : {
695 : FunctionInlineState inlineState;
696 :
697 13822 : inlineState.costLimit = subThreshold;
698 13822 : inlineState.processed = false;
699 13822 : inlineState.inlined = false;
700 13822 : inlineState.allowReconsidering = false;
701 :
702 13822 : functionStates[funcName] = inlineState;
703 13822 : worklist.push_back({funcName, searchpath});
704 :
705 : ilog(DEBUG1,
706 : "considering extern function %s at %d for inlining",
707 : funcName.data(), subThreshold);
708 : }
709 12540 : else if (!it->second.inlined &&
710 21448 : (!it->second.processed || it->second.allowReconsidering) &&
711 8908 : it->second.costLimit < subThreshold)
712 : {
713 : /*
714 : * Update inlining threshold if higher. Need to re-queue
715 : * to be processed if already processed with lower
716 : * threshold.
717 : */
718 588 : if (it->second.processed)
719 : {
720 : ilog(DEBUG1,
721 : "reconsidering extern function %s at %d for inlining, increasing from %d",
722 : funcName.data(), subThreshold, it->second.costLimit);
723 :
724 588 : it->second.processed = false;
725 588 : it->second.allowReconsidering = false;
726 588 : worklist.push_back({funcName, searchpath});
727 : }
728 588 : it->second.costLimit = subThreshold;
729 : }
730 26362 : continue;
731 : }
732 :
733 : /* can't rely on what may be inlined */
734 5134 : if (referencedFunction->isInterposable())
735 0 : return false;
736 :
737 5134 : if (!function_inlinable(*referencedFunction,
738 : subThreshold,
739 : functionStates,
740 : worklist,
741 : searchpath,
742 : visitedFunctions,
743 : running_instcount,
744 : recImportVars))
745 : {
746 : ilog(DEBUG1,
747 : "cannot inline %s due to required function %s not being inlinable",
748 : F.getName().data(), referencedFunction->getName().data());
749 264 : return false;
750 : }
751 :
752 : /* import referenced function itself */
753 4870 : importVars.insert(referencedFunction->getName());
754 :
755 : /* import referenced function and its dependents */
756 101752 : for (auto& recImportVar : recImportVars)
757 96882 : importVars.insert(recImportVar.first());
758 : }
759 :
760 13090 : return true;
761 : }
762 :
763 : /*
764 : * Attempt to load module summary located at path. Return empty pointer when
765 : * loading fails.
766 : */
767 : static std::unique_ptr<llvm::ModuleSummaryIndex>
768 492 : llvm_load_summary(llvm::StringRef path)
769 : {
770 : llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer> > MBOrErr =
771 984 : llvm::MemoryBuffer::getFile(path);
772 :
773 492 : if (std::error_code EC = MBOrErr.getError())
774 : {
775 : ilog(DEBUG1, "failed to open %s: %s", path.data(),
776 : EC.message().c_str());
777 : }
778 : else
779 : {
780 492 : llvm::MemoryBufferRef ref(*MBOrErr.get().get());
781 :
782 : llvm::Expected<std::unique_ptr<llvm::ModuleSummaryIndex> > IndexOrErr =
783 984 : llvm::getModuleSummaryIndex(ref);
784 492 : if (IndexOrErr)
785 492 : return std::move(IndexOrErr.get());
786 0 : elog(FATAL, "failed to load summary \"%s\": %s",
787 : path.data(),
788 : toString(IndexOrErr.takeError()).c_str());
789 : }
790 0 : return nullptr;
791 : }
792 :
793 : /*
794 : * Attempt to add modpath to the search path.
795 : */
796 : static void
797 1002 : add_module_to_inline_search_path(InlineSearchPath& searchpath, llvm::StringRef modpath)
798 : {
799 : /* only extension in libdir are candidates for inlining for now */
800 : #if LLVM_VERSION_MAJOR < 16
801 : #define starts_with startswith
802 : #endif
803 1002 : if (!modpath.starts_with("$libdir/"))
804 0 : return;
805 :
806 : /* if there's no match, attempt to load */
807 1002 : auto it = summary_cache->find(modpath);
808 1002 : if (it == summary_cache->end())
809 : {
810 984 : std::string path(modpath);
811 492 : path = path.replace(0, strlen("$libdir"), std::string(pkglib_path) + "/bitcode");
812 492 : path += ".index.bc";
813 492 : (*summary_cache)[modpath] = llvm_load_summary(path);
814 492 : it = summary_cache->find(modpath);
815 : }
816 :
817 : Assert(it != summary_cache->end());
818 :
819 : /* if the entry isn't NULL, it's validly loaded */
820 1002 : if (it->second)
821 1002 : searchpath.push_back(it->second.get());
822 : }
823 :
824 : /*
825 : * Search for all references for functions hashing to guid in the search path,
826 : * and return them in search path order.
827 : */
828 : static llvm::SmallVector<llvm::GlobalValueSummary *, 1>
829 18090 : summaries_for_guid(const InlineSearchPath& path, llvm::GlobalValue::GUID guid)
830 : {
831 18090 : llvm::SmallVector<llvm::GlobalValueSummary *, 1> matches;
832 :
833 36472 : for (auto index : path)
834 : {
835 18382 : llvm::ValueInfo funcVI = index->getValueInfo(guid);
836 :
837 : /* if index doesn't know function, we don't have a body, continue */
838 18382 : if (funcVI)
839 34736 : for (auto &gv : funcVI.getSummaryList())
840 17368 : matches.push_back(gv.get());
841 : }
842 :
843 18090 : return matches;
844 : }
845 :
846 : /*
847 : * Create inline wrapper with the name Name, redirecting the call to F.
848 : */
849 : static llvm::Function*
850 18 : create_redirection_function(std::unique_ptr<llvm::Module> &importMod,
851 : llvm::Function *F,
852 : llvm::StringRef Name)
853 : {
854 : typedef llvm::GlobalValue::LinkageTypes LinkageTypes;
855 :
856 18 : llvm::LLVMContext &Context = F->getContext();
857 18 : llvm::IRBuilder<> Builder(Context);
858 : llvm::Function *AF;
859 : llvm::BasicBlock *BB;
860 : llvm::CallInst *fwdcall;
861 : #if LLVM_VERSION_MAJOR < 14
862 18 : llvm::Attribute inlineAttribute;
863 : #endif
864 :
865 36 : AF = llvm::Function::Create(F->getFunctionType(),
866 : LinkageTypes::AvailableExternallyLinkage,
867 18 : Name, importMod.get());
868 18 : BB = llvm::BasicBlock::Create(Context, "entry", AF);
869 :
870 18 : Builder.SetInsertPoint(BB);
871 18 : fwdcall = Builder.CreateCall(F, &*AF->arg_begin());
872 : #if LLVM_VERSION_MAJOR < 14
873 : inlineAttribute = llvm::Attribute::get(Context,
874 18 : llvm::Attribute::AlwaysInline);
875 18 : fwdcall->addAttribute(~0U, inlineAttribute);
876 : #else
877 : fwdcall->addFnAttr(llvm::Attribute::AlwaysInline);
878 : #endif
879 18 : Builder.CreateRet(fwdcall);
880 :
881 36 : return AF;
882 : }
|
