llvm.org GIT mirror llvm / 3efe561
[ORC] Refactor definition-generation, add a generator for static libraries. This patch replaces the JITDylib::DefinitionGenerator typedef with a class of the same name, and adds support for attaching a sequence of DefinitionGeneration objects to a JITDylib. This patch also adds a new definition generator, StaticLibraryDefinitionGenerator, that can be used to add symbols fom a static library to a JITDylib. An object from the static library will be added (via a supplied ObjectLayer reference) whenever a symbol from that object is referenced. To enable testing, lli is updated to add support for the --extra-archive option when running in -jit-kind=orc-lazy mode. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@368707 91177308-0d34-0410-b5e6-96231b3b80d8 Lang Hames a month ago
8 changed file(s) with 277 addition(s) and 57 deletion(s). Raw diff Collapse all Expand all
410410 Expected
411411 buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols);
412412
413 /// ReexportsGenerator can be used with JITDylib::setGenerator to automatically
414 /// re-export a subset of the source JITDylib's symbols in the target.
415 class ReexportsGenerator {
416 public:
417 using SymbolPredicate = std::function;
418
419 /// Create a reexports generator. If an Allow predicate is passed, only
420 /// symbols for which the predicate returns true will be reexported. If no
421 /// Allow predicate is passed, all symbols will be exported.
422 ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false,
423 SymbolPredicate Allow = SymbolPredicate());
424
425 Expected operator()(JITDylib &JD, const SymbolNameSet &Names);
426
427 private:
428 JITDylib &SourceJD;
429 bool MatchNonExported = false;
430 SymbolPredicate Allow;
431 };
432
433413 /// Represents the state that a symbol has reached during materialization.
434414 enum class SymbolState : uint8_t {
435415 Invalid, /// No symbol should be in this state.
501481 friend class ExecutionSession;
502482 friend class MaterializationResponsibility;
503483 public:
504 using GeneratorFunction = std::function(
505 JITDylib &Parent, const SymbolNameSet &Names)>;
484 class DefinitionGenerator {
485 public:
486 virtual ~DefinitionGenerator();
487 virtual Expected
488 tryToGenerate(JITDylib &Parent, const SymbolNameSet &Names) = 0;
489 };
506490
507491 using AsynchronousSymbolQuerySet =
508492 std::set>;
518502 /// Get a reference to the ExecutionSession for this JITDylib.
519503 ExecutionSession &getExecutionSession() const { return ES; }
520504
521 /// Set a definition generator. If set, whenever a symbol fails to resolve
522 /// within this JITDylib, lookup and lookupFlags will pass the unresolved
523 /// symbols set to the definition generator. The generator can optionally
524 /// add a definition for the unresolved symbols to the dylib.
525 void setGenerator(GeneratorFunction DefGenerator) {
526 this->DefGenerator = std::move(DefGenerator);
527 }
505 /// Adds a definition generator to this JITDylib and returns a referenece to
506 /// it.
507 ///
508 /// When JITDylibs are searched during lookup, if no existing definition of
509 /// a symbol is found, then any generators that have been added are run (in
510 /// the order that they were added) to potentially generate a definition.
511 template
512 GeneratorT &addGenerator(std::unique_ptr DefGenerator);
513
514 /// Remove a definition generator from this JITDylib.
515 ///
516 /// The given generator must exist in this JITDylib's generators list (i.e.
517 /// have been added and not yet removed).
518 void removeGenerator(DefinitionGenerator &G);
528519
529520 /// Set the search order to be used when fixing up definitions in JITDylib.
530521 /// This will replace the previous search order, and apply to any symbol
743734 SymbolTable Symbols;
744735 UnmaterializedInfosMap UnmaterializedInfos;
745736 MaterializingInfosMap MaterializingInfos;
746 GeneratorFunction DefGenerator;
737 std::vector> DefGenerators;
747738 JITDylibSearchList SearchOrder;
748739 };
749740
931922 OutstandingMUs;
932923 };
933924
925 template
926 GeneratorT &JITDylib::addGenerator(std::unique_ptr DefGenerator) {
927 auto &G = *DefGenerator;
928 ES.runSessionLocked(
929 [&]() { DefGenerators.push_back(std::move(DefGenerator)); });
930 return G;
931 }
932
934933 template
935934 auto JITDylib::withSearchOrderDo(Func &&F)
936935 -> decltype(F(std::declval())) {
970969 });
971970 }
972971
972 /// ReexportsGenerator can be used with JITDylib::setGenerator to automatically
973 /// re-export a subset of the source JITDylib's symbols in the target.
974 class ReexportsGenerator : public JITDylib::DefinitionGenerator {
975 public:
976 using SymbolPredicate = std::function;
977
978 /// Create a reexports generator. If an Allow predicate is passed, only
979 /// symbols for which the predicate returns true will be reexported. If no
980 /// Allow predicate is passed, all symbols will be exported.
981 ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false,
982 SymbolPredicate Allow = SymbolPredicate());
983
984 Expected tryToGenerate(JITDylib &JD,
985 const SymbolNameSet &Names) override;
986
987 private:
988 JITDylib &SourceJD;
989 bool MatchNonExported = false;
990 SymbolPredicate Allow;
991 };
992
973993 /// Mangles symbol names then uniques them in the context of an
974994 /// ExecutionSession.
975995 class MangleAndInterner {
1818 #include "llvm/ExecutionEngine/Orc/Core.h"
1919 #include "llvm/ExecutionEngine/Orc/OrcError.h"
2020 #include "llvm/ExecutionEngine/RuntimeDyld.h"
21 #include "llvm/Object/Archive.h"
2122 #include "llvm/Support/DynamicLibrary.h"
2223 #include
2324 #include
3536 class Value;
3637
3738 namespace orc {
39
40 class ObjectLayer;
3841
3942 /// This iterator provides a convenient way to iterate over the elements
4043 /// of an llvm.global_ctors/llvm.global_dtors instance.
236239 /// If an instance of this class is attached to a JITDylib as a fallback
237240 /// definition generator, then any symbol found in the given DynamicLibrary that
238241 /// passes the 'Allow' predicate will be added to the JITDylib.
239 class DynamicLibrarySearchGenerator {
242 class DynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator {
240243 public:
241244 using SymbolPredicate = std::function;
242245
252255 /// Permanently loads the library at the given path and, on success, returns
253256 /// a DynamicLibrarySearchGenerator that will search it for symbol definitions
254257 /// in the library. On failure returns the reason the library failed to load.
255 static Expected<DynamicLibrarySearchGenerator>
258 static Expected<std::unique_ptr>
256259 Load(const char *FileName, char GlobalPrefix,
257260 SymbolPredicate Allow = SymbolPredicate());
258261
259262 /// Creates a DynamicLibrarySearchGenerator that searches for symbols in
260263 /// the current process.
261 static Expected<DynamicLibrarySearchGenerator>
264 static Expected<std::unique_ptr>
262265 GetForCurrentProcess(char GlobalPrefix,
263266 SymbolPredicate Allow = SymbolPredicate()) {
264267 return Load(nullptr, GlobalPrefix, std::move(Allow));
265268 }
266269
267 Expected operator()(JITDylib &JD, const SymbolNameSet &Names);
270 Expected tryToGenerate(JITDylib &JD,
271 const SymbolNameSet &Names) override;
268272
269273 private:
270274 sys::DynamicLibrary Dylib;
272276 char GlobalPrefix;
273277 };
274278
279 /// A utility class to expose symbols from a static library.
280 ///
281 /// If an instance of this class is attached to a JITDylib as a fallback
282 /// definition generator, then any symbol found in the archive will result in
283 /// the containing object being added to the JITDylib.
284 class StaticLibraryDefinitionGenerator : public JITDylib::DefinitionGenerator {
285 public:
286 /// Try to create a StaticLibraryDefinitionGenerator from the given path.
287 ///
288 /// This call will succeed if the file at the given path is a static library
289 /// is a valid archive, otherwise it will return an error.
290 static Expected>
291 Load(ObjectLayer &L, const char *FileName);
292
293 /// Try to create a StaticLibrarySearchGenerator from the given memory buffer.
294 /// Thhis call will succeed if the buffer contains a valid archive, otherwise
295 /// it will return an error.
296 static Expected>
297 Create(ObjectLayer &L, std::unique_ptr ArchiveBuffer);
298
299 Expected tryToGenerate(JITDylib &JD,
300 const SymbolNameSet &Names) override;
301
302 private:
303 StaticLibraryDefinitionGenerator(ObjectLayer &L,
304 std::unique_ptr ArchiveBuffer,
305 Error &Err);
306
307 ObjectLayer &L;
308 std::unique_ptr ArchiveBuffer;
309 object::Archive Archive;
310 size_t UnrealizedObjects = 0;
311 };
312
275313 } // end namespace orc
276314 } // end namespace llvm
277315
693693 Allow(std::move(Allow)) {}
694694
695695 Expected
696 ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) {
696 ReexportsGenerator::tryToGenerate(JITDylib &JD, const SymbolNameSet &Names) {
697697 orc::SymbolNameSet Added;
698698 orc::SymbolAliasMap AliasMap;
699699
713713 cantFail(JD.define(reexports(SourceJD, AliasMap, MatchNonExported)));
714714
715715 return Added;
716 }
717
718 JITDylib::DefinitionGenerator::~DefinitionGenerator() {}
719
720 void JITDylib::removeGenerator(DefinitionGenerator &G) {
721 ES.runSessionLocked([&]() {
722 auto I = std::find_if(DefGenerators.begin(), DefGenerators.end(),
723 [&](const std::unique_ptr &H) {
724 return H.get() == &G;
725 });
726 assert(I != DefGenerators.end() && "Generator not found");
727 DefGenerators.erase(I);
728 });
716729 }
717730
718731 Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) {
11581171 if (!Unresolved)
11591172 return Unresolved.takeError();
11601173
1161 if (DefGenerator && !Unresolved->empty()) {
1162 auto NewDefs = DefGenerator(*this, *Unresolved);
1174 /// Run any definition generators.
1175 for (auto &DG : DefGenerators) {
1176
1177 // Bail out early if we've resolved everything.
1178 if (Unresolved->empty())
1179 break;
1180
1181 // Run this generator.
1182 auto NewDefs = DG->tryToGenerate(*this, *Unresolved);
11631183 if (!NewDefs)
11641184 return NewDefs.takeError();
1185
11651186 if (!NewDefs->empty()) {
11661187 auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs);
11671188 if (!Unresolved2)
11701191 assert(Unresolved2->empty() &&
11711192 "All fallback defs should have been found by lookupFlagsImpl");
11721193 }
1173 };
1194
1195 for (auto &Name : *NewDefs)
1196 Unresolved->erase(Name);
1197 }
11741198 return Result;
11751199 });
11761200 }
11971221 assert(Q && "Query can not be null");
11981222
11991223 lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs);
1200 if (DefGenerator && !Unresolved.empty()) {
1201 auto NewDefs = DefGenerator(*this, Unresolved);
1224
1225 // Run any definition generators.
1226 for (auto &DG : DefGenerators) {
1227
1228 // Bail out early if we have resolved everything.
1229 if (Unresolved.empty())
1230 break;
1231
1232 // Run the generator.
1233 auto NewDefs = DG->tryToGenerate(*this, Unresolved);
1234
12021235 if (!NewDefs)
12031236 return NewDefs.takeError();
1237
1238 llvm::dbgs() << "NewDefs is " << *NewDefs << "\n";
12041239 if (!NewDefs->empty()) {
12051240 for (auto &D : *NewDefs)
12061241 Unresolved.erase(D);
1242 llvm::dbgs() << "NewDefs is now " << *NewDefs << "\n";
12071243 lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs);
12081244 assert(NewDefs->empty() &&
12091245 "All fallback defs should have been found by lookupImpl");
12911327 SymbolNameSet Unresolved = std::move(Names);
12921328 auto Err = ES.runSessionLocked([&, this]() -> Error {
12931329 QueryComplete = lookupImpl(Q, MUs, Unresolved);
1294 if (DefGenerator && !Unresolved.empty()) {
1330
1331 // Run any definition generators.
1332 for (auto &DG : DefGenerators) {
1333
1334 // Bail out early if we have resolved everything.
1335 if (Unresolved.empty())
1336 break;
1337
12951338 assert(!QueryComplete && "query complete but unresolved symbols remain?");
1296 auto NewDefs = DefGenerator(*this, Unresolved);
1339 auto NewDefs = DG->tryToGenerate(*this, Unresolved);
12971340 if (!NewDefs)
12981341 return NewDefs.takeError();
12991342 if (!NewDefs->empty()) {
77
88 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
99
10 #include "llvm/ExecutionEngine/Orc/Layer.h"
1011 #include "llvm/IR/Constants.h"
1112 #include "llvm/IR/Function.h"
1213 #include "llvm/IR/GlobalVariable.h"
177178 : Dylib(std::move(Dylib)), Allow(std::move(Allow)),
178179 GlobalPrefix(GlobalPrefix) {}
179180
180 Expected<DynamicLibrarySearchGenerator>
181 Expected<std::unique_ptr>
181182 DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix,
182183 SymbolPredicate Allow) {
183184 std::string ErrMsg;
184185 auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg);
185186 if (!Lib.isValid())
186187 return make_error(std::move(ErrMsg), inconvertibleErrorCode());
187 return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix,
188 std::move(Allow));
188 return llvm::make_unique(
189 std::move(Lib), GlobalPrefix, std::move(Allow));
189190 }
190191
191192 Expected
192 DynamicLibrarySearchGenerator::operator()(JITDylib &JD,
193 const SymbolNameSet &Names) {
193 DynamicLibrarySearchGenerator::tryToGenerate(JITDylib &JD,
194 const SymbolNameSet &Names) {
194195 orc::SymbolNameSet Added;
195196 orc::SymbolMap NewSymbols;
196197
225226 return Added;
226227 }
227228
229 Expected>
230 StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName) {
231 auto ArchiveBuffer = errorOrToExpected(MemoryBuffer::getFile(FileName));
232
233 if (!ArchiveBuffer)
234 return ArchiveBuffer.takeError();
235
236 return Create(L, std::move(*ArchiveBuffer));
237 }
238
239 Expected>
240 StaticLibraryDefinitionGenerator::Create(
241 ObjectLayer &L, std::unique_ptr ArchiveBuffer) {
242 Error Err = Error::success();
243
244 std::unique_ptr ADG(
245 new StaticLibraryDefinitionGenerator(L, std::move(ArchiveBuffer), Err));
246
247 if (Err)
248 return std::move(Err);
249
250 return std::move(ADG);
251 }
252
253 Expected
254 StaticLibraryDefinitionGenerator::tryToGenerate(JITDylib &JD,
255 const SymbolNameSet &Names) {
256
257 DenseSet> ChildBufferInfos;
258 SymbolNameSet NewDefs;
259
260 for (const auto &Name : Names) {
261 auto Child = Archive.findSym(*Name);
262 if (!Child)
263 return Child.takeError();
264 if (*Child == None)
265 continue;
266 auto ChildBuffer = (*Child)->getMemoryBufferRef();
267 if (!ChildBuffer)
268 return ChildBuffer.takeError();
269 ChildBufferInfos.insert(
270 {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()});
271 NewDefs.insert(Name);
272 }
273
274 for (auto ChildBufferInfo : ChildBufferInfos) {
275 MemoryBufferRef ChildBufferRef(ChildBufferInfo.first,
276 ChildBufferInfo.second);
277
278 if (auto Err =
279 L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef), VModuleKey()))
280 return std::move(Err);
281
282 --UnrealizedObjects;
283 }
284
285 return NewDefs;
286 }
287
288 StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(
289 ObjectLayer &L, std::unique_ptr ArchiveBuffer, Error &Err)
290 : L(L), ArchiveBuffer(std::move(ArchiveBuffer)),
291 Archive(*this->ArchiveBuffer, Err) {
292
293 if (Err)
294 return;
295
296 Error Err2 = Error::success();
297 for (auto _ : Archive.children(Err2)) {
298 (void)_;
299 ++UnrealizedObjects;
300 }
301
302 // No need to check this: We will leave it to the caller.
303 Err = std::move(Err2);
304 }
305
228306 } // End namespace orc.
229307 } // End namespace llvm.
0 ; This first line will generate the .o files for the next run line
1 ; RUN: llc -filetype=obj -o %t.o %p/Inputs/basic-object-source.ll
2 ; RUN: llvm-ar r %t.a %t.o
3 ; RUN: lli -jit-kind=orc-lazy -extra-archive %t.a %s
4
5 declare i32 @foo()
6
7 define i32 @main() {
8 %r = call i32 @foo( ) ; [#uses=1]
9 ret i32 %r
10 }
791791 });
792792 return TSM;
793793 });
794 J->getMainJITDylib().setGenerator(
794 J->getMainJITDylib().addGenerator(
795795 ExitOnErr(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
796796 J->getDataLayout().getGlobalPrefix())));
797797
830830 auto &JD = *JDItr->second;
831831 ExitOnErr(
832832 J->addLazyIRModule(JD, orc::ThreadSafeModule(std::move(M), TSCtx)));
833 }
834
835 for (auto EAItr = ExtraArchives.begin(), EAEnd = ExtraArchives.end();
836 EAItr != EAEnd; ++EAItr) {
837 auto EAIdx = ExtraArchives.getPosition(EAItr - ExtraArchives.begin());
838 assert(EAIdx != 0 && "ExtraArchive should have index > 0");
839 auto JDItr = std::prev(IdxToDylib.lower_bound(EAIdx));
840 auto &JD = *JDItr->second;
841 JD.addGenerator(ExitOnErr(orc::StaticLibraryDefinitionGenerator::Load(
842 J->getObjLinkingLayer(), EAItr->c_str())));
833843 }
834844 }
835845
379379 auto FilterMainEntryPoint = [InternedEntryPointName](SymbolStringPtr Name) {
380380 return Name != InternedEntryPointName;
381381 };
382 S.ES.getMainJITDylib().setGenerator(
382 S.ES.getMainJITDylib().addGenerator(
383383 ExitOnErr(orc::DynamicLibrarySearchGenerator::GetForCurrentProcess(
384384 GlobalPrefix, FilterMainEntryPoint)));
385385
242242
243243 TEST_F(CoreAPIsStandardTest, LookupWithGeneratorFailure) {
244244
245 class BadGenerator {
245 class BadGenerator : public JITDylib::DefinitionGenerator {
246246 public:
247 Expected operator()(JITDylib &, const SymbolNameSet &) {
247 Expected tryToGenerate(JITDylib &,
248 const SymbolNameSet &) override {
248249 return make_error("BadGenerator", inconvertibleErrorCode());
249250 }
250251 };
251252
252 JD.setGenerator(BadGenerator());
253 JD.addGenerator(llvm::make_unique());
253254
254255 EXPECT_THAT_ERROR(JD.lookupFlags({Foo}).takeError(), Failed())
255256 << "Generator failure did not propagate through lookupFlags";
342343
343344 auto Filter = [this](SymbolStringPtr Name) { return Name != Bar; };
344345
345 JD.setGenerator(ReexportsGenerator(JD2, false, Filter));
346 JD.addGenerator(llvm::make_unique(JD2, false, Filter));
346347
347348 auto Flags = cantFail(JD.lookupFlags({Foo, Bar, Baz}));
348349 EXPECT_EQ(Flags.size(), 1U) << "Unexpected number of results";
666667 TEST_F(CoreAPIsStandardTest, GeneratorTest) {
667668 cantFail(JD.define(absoluteSymbols({{Foo, FooSym}})));
668669
669 JD.setGenerator([&](JITDylib &JD2, const SymbolNameSet &Names) {
670 cantFail(JD2.define(absoluteSymbols({{Bar, BarSym}})));
671 return SymbolNameSet({Bar});
672 });
670 class TestGenerator : public JITDylib::DefinitionGenerator {
671 public:
672 TestGenerator(SymbolMap Symbols) : Symbols(std::move(Symbols)) {}
673 Expected tryToGenerate(JITDylib &JD,
674 const SymbolNameSet &Names) {
675 SymbolMap NewDefs;
676 SymbolNameSet NewNames;
677
678 for (auto &Name : Names) {
679 if (Symbols.count(Name)) {
680 NewDefs[Name] = Symbols[Name];
681 NewNames.insert(Name);
682 }
683 }
684 cantFail(JD.define(absoluteSymbols(std::move(NewDefs))));
685 return NewNames;
686 };
687
688 private:
689 SymbolMap Symbols;
690 };
691
692 JD.addGenerator(llvm::make_unique(SymbolMap({{Bar, BarSym}})));
673693
674694 auto Result =
675695 cantFail(ES.lookup(JITDylibSearchList({{&JD, false}}), {Foo, Bar}));