diff --git a/bindings/pyroot/pythonizations/test/numbadeclare.py b/bindings/pyroot/pythonizations/test/numbadeclare.py index ee705398135ed..debce01c06a62 100644 --- a/bindings/pyroot/pythonizations/test/numbadeclare.py +++ b/bindings/pyroot/pythonizations/test/numbadeclare.py @@ -312,6 +312,11 @@ class NumbaDeclareArray(unittest.TestCase): test_inputs = [default_test_inputs] + # The global module index does not have RVec entities preloaded and + # gInterpreter.Declare is not allowed to load libROOTVecOps for RVec. + # Preload the library now. + ROOT.gSystem.Load("libROOTVecOps") + @unittest.skipIf(skip, skip_reason) def test_wrapper_in_vecf(self): """ diff --git a/bindings/pyroot/pythonizations/test/rdataframe_asnumpy.py b/bindings/pyroot/pythonizations/test/rdataframe_asnumpy.py index 90a1a303d2d0f..f2adb6a66f916 100644 --- a/bindings/pyroot/pythonizations/test/rdataframe_asnumpy.py +++ b/bindings/pyroot/pythonizations/test/rdataframe_asnumpy.py @@ -156,6 +156,11 @@ def test_read_tlorentzvector(self): """ Testing reading a TLorentzVector """ + + # The global module index does not have it preloaded and + # gInterpreter.Declare is not allowed to load libPhysics for + # TLorentzVector. Preload the library now. + ROOT.gSystem.Load("libPhysics") ROOT.gInterpreter.Declare(""" TLorentzVector create_tlorentzvector() { auto v = TLorentzVector(); diff --git a/bindings/pyroot_legacy/test/rdataframe_asnumpy.py b/bindings/pyroot_legacy/test/rdataframe_asnumpy.py index a1d818cf60954..816cb11f76e2b 100644 --- a/bindings/pyroot_legacy/test/rdataframe_asnumpy.py +++ b/bindings/pyroot_legacy/test/rdataframe_asnumpy.py @@ -118,6 +118,10 @@ def test_read_vector_variablesize(self): self.assertIn("ROOT.vector", str(type(npy["x"][0]))) def test_read_tlorentzvector(self): + # The global module index does not have it preloaded and + # gInterpreter.Declare is not allowed to load libPhysics for + # TLorentzVector. Preload the library now. + ROOT.gSystem.Load("libPhysics") ROOT.gInterpreter.Declare(""" TLorentzVector create_tlorentzvector() { auto v = TLorentzVector(); diff --git a/core/metacling/src/CMakeLists.txt b/core/metacling/src/CMakeLists.txt index 86c7b83ba268f..2923345f7cb84 100644 --- a/core/metacling/src/CMakeLists.txt +++ b/core/metacling/src/CMakeLists.txt @@ -31,11 +31,11 @@ endif() ROOT_OBJECT_LIBRARY(MetaCling rootclingTCling.cxx + TCling.cxx TClingBaseClassInfo.cxx TClingCallbacks.cxx TClingCallFunc.cxx TClingClassInfo.cxx - TCling.cxx TClingDataMemberInfo.cxx TClingDeclInfo.cxx TClingMethodArgInfo.cxx diff --git a/core/metacling/src/TCling.cxx b/core/metacling/src/TCling.cxx index c1fb4071edc02..96bdddf0fa4cd 100644 --- a/core/metacling/src/TCling.cxx +++ b/core/metacling/src/TCling.cxx @@ -1122,7 +1122,63 @@ static GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc, cling RecreateIndex = true; } if (RecreateIndex) { - GlobalModuleIndex::writeIndex(CI.getFileManager(), CI.getPCHContainerReader(), ModuleIndexPath); + cling::Interpreter::PushTransactionRAII deserRAII(&interp); + clang::GlobalModuleIndex::UserDefinedInterestingIDs IDs; + + struct DefinitionFinder : public RecursiveASTVisitor { + DefinitionFinder(clang::GlobalModuleIndex::UserDefinedInterestingIDs& IDs, + clang::TranslationUnitDecl* TU) : DefinitionIDs(IDs) { + TraverseDecl(TU); + } + bool VisitNamedDecl(NamedDecl *ND) { + for (auto R : ND->redecls()) { + if (!R->isFromASTFile()) + continue; + if (TagDecl *TD = llvm::dyn_cast(R)) { + if (TD->isCompleteDefinition()) + Register(TD); + } else if (NamespaceDecl *NSD = llvm::dyn_cast(R)) + Register(NSD, /*AddSingleEntry=*/ false); + else if (TypedefNameDecl *TND = dyn_cast(R)) + Register(TND); + // FIXME: Add the rest... + } + return true; // continue decending + } + private: + clang::GlobalModuleIndex::UserDefinedInterestingIDs &DefinitionIDs; + void Register(NamedDecl* ND, bool AddSingleEntry = true) { + assert(ND->isFromASTFile()); + // FIXME: All decls should have an owning module once rootcling + // updates its generated decls from within the LookupHelper & co. + if (!ND->hasOwningModule()) { +#ifndef NDEBUG + SourceManager &SM = ND->getASTContext().getSourceManager(); + SourceLocation Loc = ND->getLocation(); + const FileEntry *FE = SM.getFileEntryForID(SM.getFileID(Loc)); + (void)FE; + assert(FE->getName().contains("input_line_")); +#endif + return; + } + + Module *OwningModule = ND->getOwningModule()->getTopLevelModule(); + assert(OwningModule); + if (AddSingleEntry && DefinitionIDs.count(ND->getName())) + return; + // FIXME: The FileEntry in not stable to serialize. + // FIXME: We might end up with many times with the same module. + // FIXME: We might end up two modules containing a definition. + // FIXME: What do we do if no definition is found. + DefinitionIDs[ND->getName()].push_back(OwningModule->getASTFile()); + } + }; + DefinitionFinder defFinder(IDs, CI.getASTContext().getTranslationUnitDecl()); + + GlobalModuleIndex::writeIndex(CI.getFileManager(), + CI.getPCHContainerReader(), + ModuleIndexPath, + &IDs); ModuleManager->resetForReload(); ModuleManager->loadGlobalIndex(); GlobalIndex = ModuleManager->getGlobalIndex(); @@ -1161,6 +1217,7 @@ static void RegisterCxxModules(cling::Interpreter &clingInterp) "ROOT_Rtypes", "ROOT_Foundation_Stage1_NoRTTI", "Core", + "Rint", "RIO"}; LoadModules(CoreModules, clingInterp); @@ -1171,19 +1228,27 @@ static void RegisterCxxModules(cling::Interpreter &clingInterp) LoadModules(CommonModules, clingInterp); // These modules should not be preloaded but they fix issues. - std::vector FIXMEModules = {"Hist", "Gpad", "Graf", - "GenVector", "Smatrix", "Tree", - "TreePlayer", "Physics", - "Proof", "Geom"}; + // FIXME: Hist is not a core module but is very entangled to MathCore and + // causes issues. + std::vector FIXMEModules = {"Hist"}; LoadModules(FIXMEModules, clingInterp); clang::CompilerInstance &CI = *clingInterp.getCI(); GlobalModuleIndex *GlobalIndex = nullptr; + // Conservatively enable platform by platform. + bool supportedPlatform = false; + // Allow forcefully enabling the GMI. const char *experimentalGMI = gSystem->Getenv("ROOT_EXPERIMENTAL_GMI"); - if (experimentalGMI && strcmp(experimentalGMI,"false") != 0) { + if (experimentalGMI && strcmp(experimentalGMI,"false") != 0) + supportedPlatform = true; + + if (supportedPlatform && !gSystem->Getenv("ROOT_DISABLE_GMI")) { loadGlobalModuleIndex(SourceLocation(), clingInterp); + // FIXME: The ASTReader still calls loadGlobalIndex and loads the file + // We should investigate how to suppress it completely. GlobalIndex = CI.getModuleManager()->getGlobalIndex(); } + llvm::StringSet<> KnownModuleFileNames; if (GlobalIndex) GlobalIndex->getKnownModuleFileNames(KnownModuleFileNames); @@ -1771,6 +1836,7 @@ void TCling::LoadPCM(std::string pcmFileNameFullPath) std::string RDictFileOpts = pcmFileNameFullPath + "?filetype=pcm"; TMemFile pcmMemFile(RDictFileOpts.c_str(), range); + cling::Interpreter::PushTransactionRAII deserRAII(GetInterpreterImpl()); LoadPCMImpl(pcmMemFile); fPendingRdicts.erase(pendingRdict); diff --git a/core/metacling/src/TClingCallbacks.cxx b/core/metacling/src/TClingCallbacks.cxx index 6a450e9be1f6f..307469d5657ff 100644 --- a/core/metacling/src/TClingCallbacks.cxx +++ b/core/metacling/src/TClingCallbacks.cxx @@ -288,6 +288,9 @@ bool TClingCallbacks::LookupObject(LookupResult &R, Scope *S) { bool TClingCallbacks::findInGlobalModuleIndex(DeclarationName Name, bool loadFirstMatchOnly /*=true*/) { + if (::getenv("ROOT_DISABLE_GMI")) + return false; + const CompilerInstance *CI = m_Interpreter->getCI(); const LangOptions &LangOpts = CI->getPreprocessor().getLangOpts(); @@ -315,6 +318,10 @@ bool TClingCallbacks::findInGlobalModuleIndex(DeclarationName Name, bool loadFir for (auto FileName : FoundModules) { StringRef ModuleName = llvm::sys::path::stem(*FileName); fIsLoadingModule = true; + if (gDebug > 2) + llvm::errs() << "Loading " << ModuleName << " on demand" + << " for " << Name.getAsString() << "\n"; + m_Interpreter->loadModule(ModuleName); fIsLoadingModule = false; if (loadFirstMatchOnly) diff --git a/interpreter/llvm/src/tools/clang/include/clang/Serialization/GlobalModuleIndex.h b/interpreter/llvm/src/tools/clang/include/clang/Serialization/GlobalModuleIndex.h index 57e6297ecc7cc..0db0f5904dab6 100644 --- a/interpreter/llvm/src/tools/clang/include/clang/Serialization/GlobalModuleIndex.h +++ b/interpreter/llvm/src/tools/clang/include/clang/Serialization/GlobalModuleIndex.h @@ -141,6 +141,9 @@ class GlobalModuleIndex { EC_IOError }; + using UserDefinedInterestingIDs = + llvm::StringMap>; + /// \brief Read a global index file for the given directory. /// /// \param Path The path to the specific module cache where the module files @@ -205,9 +208,11 @@ class GlobalModuleIndex { /// creating modules. /// \param Path The path to the directory containing module files, into /// which the global index will be written. + /// \param Optionally pass already precomputed interesting identifiers. static ErrorCode writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, - StringRef Path); + StringRef Path, + UserDefinedInterestingIDs *ExternalIDs = nullptr); }; } diff --git a/interpreter/llvm/src/tools/clang/lib/Serialization/GlobalModuleIndex.cpp b/interpreter/llvm/src/tools/clang/lib/Serialization/GlobalModuleIndex.cpp index d6e07a83f08ac..7406342fd2ac6 100644 --- a/interpreter/llvm/src/tools/clang/lib/Serialization/GlobalModuleIndex.cpp +++ b/interpreter/llvm/src/tools/clang/lib/Serialization/GlobalModuleIndex.cpp @@ -422,9 +422,6 @@ namespace { /// \brief Builder that generates the global module index file. class GlobalModuleIndexBuilder { - FileManager &FileMgr; - const PCHContainerReader &PCHContainerRdr; - /// Mapping from files to module file information. typedef llvm::MapVector ModuleFilesMap; @@ -464,14 +461,20 @@ namespace { } public: - explicit GlobalModuleIndexBuilder( - FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr) - : FileMgr(FileMgr), PCHContainerRdr(PCHContainerRdr) {} + explicit GlobalModuleIndexBuilder(GlobalModuleIndex::UserDefinedInterestingIDs* ExternalIDs) { + if (!ExternalIDs) + return; + + for (const auto & I : *ExternalIDs) + for (const FileEntry * J : I.getValue()) + InterestingIdentifiers[I.getKey()].push_back(getModuleFileInfo(J).ID); + } /// \brief Load the contents of the given module file into the builder. /// /// \returns true if an error occurred, false otherwise. - bool loadModuleFile(const FileEntry *File); + bool loadModuleFile(const FileEntry *File, FileManager &FileMgr, + const PCHContainerReader &PCHContainerRdr); /// \brief Write the index to the given bitstream. /// \returns true if an error occurred, false otherwise. @@ -542,7 +545,9 @@ namespace { }; } -bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) { +bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File, + FileManager &FileMgr, + const PCHContainerReader &PCHContainerRdr) { // Open the module file. auto Buffer = FileMgr.getBufferForFile(File, /*isVolatile=*/true); @@ -693,7 +698,10 @@ bool GlobalModuleIndexBuilder::loadModuleFile(const FileEntry *File) { DEnd = Table->data_end(); D != DEnd; ++D) { std::pair Ident = *D; - InterestingIdentifiers[Ident.first].push_back(ID); + if (Ident.second) + InterestingIdentifiers[Ident.first].push_back(ID); + else + (void)InterestingIdentifiers[Ident.first]; } } @@ -755,14 +763,15 @@ bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { for (auto MapEntry : ImportedModuleFiles) { auto *File = MapEntry.first; ImportedModuleFileInfo &Info = MapEntry.second; - // if (getModuleFileInfo(File).Signature) { - // if (getModuleFileInfo(File).Signature != Info.StoredSignature) - // // Verify Signature. - // return true; - // } else if (Info.StoredSize != File->getSize() || - // Info.StoredModTime != File->getModificationTime()) - // // Verify Size and ModTime. - // return true; + if (getModuleFileInfo(File).Signature) { + if (getModuleFileInfo(File).Signature != Info.StoredSignature) + // Verify Signature. + return true; + } else if (Info.StoredSize != File->getSize() || + (Info.StoredModTime && + Info.StoredModTime != File->getModificationTime())) + // Verify Size and ModTime. + return true; } using namespace llvm; @@ -846,7 +855,9 @@ bool GlobalModuleIndexBuilder::writeIndex(llvm::BitstreamWriter &Stream) { GlobalModuleIndex::ErrorCode GlobalModuleIndex::writeIndex(FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, - StringRef Path) { + StringRef Path, + UserDefinedInterestingIDs *ExternalIDs /* = nullptr */) { + llvm::SmallString<128> IndexPath; IndexPath += Path; llvm::sys::path::append(IndexPath, IndexFileName); @@ -869,32 +880,34 @@ GlobalModuleIndex::writeIndex(FileManager &FileMgr, } // The module index builder. - GlobalModuleIndexBuilder Builder(FileMgr, PCHContainerRdr); - - // Load each of the module files. - std::error_code EC; - for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd; - D != DEnd && !EC; - D.increment(EC)) { - // If this isn't a module file, we don't care. - if (llvm::sys::path::extension(D->path()) != ".pcm") { - // ... unless it's a .pcm.lock file, which indicates that someone is - // in the process of rebuilding a module. They'll rebuild the index - // at the end of that translation unit, so we don't have to. - if (llvm::sys::path::extension(D->path()) == ".pcm.lock") - return EC_Building; + GlobalModuleIndexBuilder Builder(ExternalIDs); + + if (!ExternalIDs) { + // Load each of the module files. + std::error_code EC; + for (llvm::sys::fs::directory_iterator D(Path, EC), DEnd; + D != DEnd && !EC; + D.increment(EC)) { + // If this isn't a module file, we don't care. + if (llvm::sys::path::extension(D->path()) != ".pcm") { + // ... unless it's a .pcm.lock file, which indicates that someone is + // in the process of rebuilding a module. They'll rebuild the index + // at the end of that translation unit, so we don't have to. + if (llvm::sys::path::extension(D->path()) == ".pcm.lock") + return EC_Building; - continue; - } + continue; + } - // If we can't find the module file, skip it. - const FileEntry *ModuleFile = FileMgr.getFile(D->path()); - if (!ModuleFile) - continue; + // If we can't find the module file, skip it. + const FileEntry *ModuleFile = FileMgr.getFile(D->path()); + if (!ModuleFile) + continue; - // Load this module file. - if (Builder.loadModuleFile(ModuleFile)) - return EC_IOError; + // Load this module file. + if (Builder.loadModuleFile(ModuleFile, FileMgr, PCHContainerRdr)) + return EC_IOError; + } } // The output buffer, into which the global index will be written.