Skip to content

[cxxmodules] Implement global module indexing to improve performance. #4016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vgvassilev merged 20 commits into from
Feb 24, 2020
Merged

[cxxmodules] Implement global module indexing to improve performance. #4016

Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
916823d
[cxxmodules] Implement global module indexing to improve performance.
vgvassilev Feb 20, 2019
e4bb1a8
[cxxmodules] Do not cache the file lookup failure.
vgvassilev Jun 30, 2019
7c14fcf
[cxxmodules] Preload only common modules.
vgvassilev Jun 30, 2019
3bda1e6
[cxxmodules][cling] Add a callback for start/finish code generation.
vgvassilev Jul 4, 2019
3db4b28
[tcling] Modernize header file virtual -> override.
vgvassilev Jul 4, 2019
e96522a
[tcling] Modernize header file: use inline initialization.
vgvassilev Jul 4, 2019
66d6e47
[cxxmodules] Tighten the findInGlobalModuleIndex routine.
vgvassilev Jul 5, 2019
9822d02
[cxxmodules] Preload Hist because it has the same issue as Gpad.
vgvassilev Jul 5, 2019
0766260
Preload Graf for the same reason as Hist and Gpad
arpi-r Jul 12, 2019
a77d5ed
Add GenVector FIXMEModules
arpi-r Jul 25, 2019
661f9dd
Do not resolve from require complete type.
vgvassilev Jan 25, 2020
1d59f46
If the identifier is unknown, return false.
vgvassilev Jan 28, 2020
4b38143
Rename the interface, add documentation, make it bool.
vgvassilev Jan 28, 2020
9c06b0d
Preload modules which are not in the index.
vgvassilev Jan 28, 2020
d84e4a0
Add help how to automatically incorporate clang-format changes.
vgvassilev Jan 28, 2020
da1ec92
Preload Tree to fix the messy roottest-root-io-newstl-make
vgvassilev Jan 29, 2020
c0014fd
Do not load recursively modules.
vgvassilev Feb 3, 2020
7d936ae
Preload module Physics.
vgvassilev Feb 3, 2020
0eea44a
Preload Smatrix, TreePlayer, Proof and Geom.
vgvassilev Feb 11, 2020
774c08f
[cxxmodules] Make the global module index opt-in.
vgvassilev Feb 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
[cxxmodules] Preload only common modules.
  • Loading branch information
@vgvassilev
vgvassilev committed Feb 24, 2020
commit 7c14fcf30b3d82a0763537259afe39af8c52badb
156 changes: 57 additions & 99 deletions core/metacling/src/TCling.cxx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1090,7 +1090,50 @@ static std::string GetModuleNameAsString(clang::Module *M, const clang::Preproce
return std::string(llvm::sys::path::stem(ModuleName));
}

static void RegisterCxxModules(cling::Interpreter &clingInterp)
static bool HaveFullGlobalModuleIndex = false;
static GlobalModuleIndex *loadGlobalModuleIndex(SourceLocation TriggerLoc, cling::Interpreter &interp)
{
CompilerInstance &CI = *interp.getCI();
Preprocessor &PP = CI.getPreprocessor();
auto ModuleManager = CI.getModuleManager();
assert(ModuleManager);
// StringRef ModuleIndexPath = HSI.getModuleCachePath();
// HeaderSearch& HSI = PP.getHeaderSearchInfo();
// HSI.setModuleCachePath(TROOT::GetLibDir().Data());
std::string ModuleIndexPath = TROOT::GetLibDir().Data();
if (ModuleIndexPath.empty())
return nullptr;
// Get an existing global index. This loads it if not already loaded.
ModuleManager->resetForReload();
ModuleManager->loadGlobalIndex();
GlobalModuleIndex *GlobalIndex = ModuleManager->getGlobalIndex();

// For finding modules needing to be imported for fixit messages,
// we need to make the global index cover all modules, so we do that here.
if (!GlobalIndex && !HaveFullGlobalModuleIndex) {
ModuleMap &MMap = PP.getHeaderSearchInfo().getModuleMap();
bool RecreateIndex = false;
for (ModuleMap::module_iterator I = MMap.module_begin(), E = MMap.module_end(); I != E; ++I) {
Module *TheModule = I->second;
// We do want the index only of the prebuilt modules
std::string ModuleName = GetModuleNameAsString(TheModule, PP);
if (ModuleName.empty())
continue;
LoadModule(ModuleName, interp);
RecreateIndex = true;
}
if (RecreateIndex) {
GlobalModuleIndex::writeIndex(CI.getFileManager(), CI.getPCHContainerReader(), ModuleIndexPath);
ModuleManager->resetForReload();
ModuleManager->loadGlobalIndex();
GlobalIndex = ModuleManager->getGlobalIndex();
}
HaveFullGlobalModuleIndex = true;
}
return GlobalIndex;
}

static void RegisterCommonCxxModules(cling::Interpreter &clingInterp)
{
if (!clingInterp.getCI()->getLangOpts().Modules)
return;
Expand All @@ -1115,35 +1158,22 @@ static void RegisterCxxModules(cling::Interpreter &clingInterp)
"Core",
"RIO"};

// FIXME: Reducing those will let us be less dependent on rootmap files
static constexpr std::array<const char *, 3> ExcludeModules = {
{"Rtools", "RSQLite", "RInterface"}};

LoadModules(CoreModules, clingInterp);

// FIXME: Reducing those will let us be less dependent on rootmap files
// static constexpr std::array<const char *, 3> ExcludeModules = {
// {"Rtools", "RSQLite", "RInterface"}};

// Take this branch only from ROOT because we don't need to preload modules in rootcling
if (!IsFromRootCling()) {
// Dynamically get all the modules and load them if they are not in core modules
clang::CompilerInstance &CI = *clingInterp.getCI();
clang::ModuleMap &moduleMap = CI.getPreprocessor().getHeaderSearchInfo().getModuleMap();
clang::Preprocessor &PP = CI.getPreprocessor();
std::vector<std::string> ModulesPreloaded;
for (auto I = moduleMap.module_begin(), E = moduleMap.module_end(); I != E; ++I) {
clang::Module *M = I->second;
assert(M);

std::string ModuleName = GetModuleNameAsString(M, PP);
if (!ModuleName.empty() &&
std::find(CoreModules.begin(), CoreModules.end(), ModuleName) == CoreModules.end() &&
std::find(ExcludeModules.begin(), ExcludeModules.end(), ModuleName) ==
ExcludeModules.end()) {
if (M->IsSystem && !M->IsMissingRequirement)
LoadModule(ModuleName, clingInterp);
else if (!M->IsSystem && !M->IsMissingRequirement)
ModulesPreloaded.push_back(ModuleName);
}
}
LoadModules(ModulesPreloaded, clingInterp);
std::vector<std::string> CommonModules = {"MathCore"};
LoadModules(CommonModules, clingInterp);

// These modules should not be preloaded but they fix issues.
std::vector<std::string> FIXMEModules = {"Gpad"};
LoadModules(FIXMEModules, clingInterp);

loadGlobalModuleIndex(SourceLocation(), clingInterp);
}

// Check that the gROOT macro was exported by any core module.
Expand Down Expand Up @@ -1196,51 +1226,6 @@ static void RegisterPreIncludedHeaders(cling::Interpreter &clingInterp)
clingInterp.declare(PreIncludes);
}

static bool HaveFullGlobalModuleIndex = false;
static GlobalModuleIndex *loadGlobalModuleIndex(cling::Interpreter &interp, SourceLocation TriggerLoc)
{
CompilerInstance &CI = *interp.getCI();
Preprocessor &PP = CI.getPreprocessor();
auto ModuleManager = CI.getModuleManager();
assert(ModuleManager);
// StringRef ModuleIndexPath = HSI.getModuleCachePath();
// HeaderSearch& HSI = PP.getHeaderSearchInfo();
// HSI.setModuleCachePath(TROOT::GetLibDir().Data());
std::string ModuleIndexPath = TROOT::GetLibDir().Data();
if (ModuleIndexPath.empty())
return nullptr;
// Get an existing global index. This loads it if not already loaded.
ModuleManager->resetForReload();
ModuleManager->loadGlobalIndex();
GlobalModuleIndex *GlobalIndex = ModuleManager->getGlobalIndex();
if (!GlobalIndex && CI.hasFileManager()) {
}

// For finding modules needing to be imported for fixit messages,
// we need to make the global index cover all modules, so we do that here.
if (!GlobalIndex && !HaveFullGlobalModuleIndex) {
ModuleMap &MMap = PP.getHeaderSearchInfo().getModuleMap();
bool RecreateIndex = false;
for (ModuleMap::module_iterator I = MMap.module_begin(), E = MMap.module_end(); I != E; ++I) {
Module *TheModule = I->second;
// We do want the index only of the prebuilt modules
std::string ModuleName = GetModuleNameAsString(TheModule, PP);
if (ModuleName.empty())
continue;
LoadModule(ModuleName, interp);
RecreateIndex = true;
}
if (RecreateIndex) {
GlobalModuleIndex::writeIndex(CI.getFileManager(), CI.getPCHContainerReader(), ModuleIndexPath);
ModuleManager->resetForReload();
ModuleManager->loadGlobalIndex();
GlobalIndex = ModuleManager->getGlobalIndex();
}
HaveFullGlobalModuleIndex = true;
}
return GlobalIndex;
}

////////////////////////////////////////////////////////////////////////////////
/// Initialize the cling interpreter interface.
/// \param argv - array of arguments passed to the cling::Interpreter constructor
Expand Down Expand Up @@ -1449,34 +1434,7 @@ TCling::TCling(const char *name, const char *title, const char* const argv[])
static llvm::raw_fd_ostream fMPOuts (STDOUT_FILENO, /*ShouldClose*/false);
fMetaProcessor = llvm::make_unique<cling::MetaProcessor>(*fInterpreter, fMPOuts);

if (fInterpreter->getCI()->getLangOpts().Modules) {
// Setup core C++ modules if we have any to setup.

// Load libc and stl first.
#ifdef R__MACOSX
LoadModules({"Darwin", "std"}, *fInterpreter);
#else
LoadModules({"libc", "stl"}, *fInterpreter);
#endif

if (!fromRootCling)
loadGlobalModuleIndex(*fInterpreter, SourceLocation());

// C99 decided that it's a very good idea to name a macro `I` (the letter I).
// This seems to screw up nearly all the template code out there as `I` is
// common template parameter name and iterator variable name.
// Let's follow the GCC recommendation and undefine `I` in case any of the
// core modules have defined it:
// https://www.gnu.org/software/libc/manual/html_node/Complex-Numbers.html
fInterpreter->declare("#ifdef I\n #undef I\n #endif\n");

// These macros are from loading R related modules, which conflict with
// user's code.
fInterpreter->declare("#ifdef PI\n #undef PI\n #endif\n");
fInterpreter->declare("#ifdef ERROR\n #undef ERROR\n #endif\n");
}

// RegisterCxxModules(*fInterpreter);
RegisterCommonCxxModules(*fInterpreter);
RegisterPreIncludedHeaders(*fInterpreter);

// We are now ready (enough is loaded) to init the list of opaque typedefs.
Expand Down