[cxxmodules] Preload more modules

CMakeLists.txt

@@ -351,7 +351,11 @@ endif()
 
 if(runtime_cxxmodules)
   # Dummy target that does nothing, we don't need a PCH for modules.
-  add_custom_target(onepcm)
+  # However, we require that TMVA/Graf/Treeplayer are built here as
+  # we will always load them to workaround the fact that we can't
+  # load the decls in them via rootmap files (as they are inside
+  # namespaces which isn't supported).
+  add_custom_target(onepcm DEPENDS Net MathCore TreeViewer Hist Gpad TreePlayer EG RGL TMVA TMVAGui Graf GenVector)
 else()
   add_custom_command(OUTPUT etc/allDict.cxx.pch
                     COMMAND ${CMAKE_COMMAND} -E env ROOTIGNOREPREFIX=1 ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/etc/dictpch/makepch.py etc/allDict.cxx.pch ${__allIncludes} -I${CMAKE_BINARY_DIR}/include

cmake/modules/RootNewMacros.cmake

@@ -366,8 +366,11 @@ function(ROOT_GENERATE_DICTIONARY dictionary)
     endforeach()
   endif()
 
-  if(runtime_cxxmodules AND ARG_MODULE)
-    set(newargs -cxxmodule ${newargs})
+  set(runtime_cxxmodules_env)
+  if(runtime_cxxmodules AND ARG_MODULE AND NOT ARG_MULTIDICT)
+    # FIXME: Once modules work better, we should use some other value like "1"
+    # to disable the module-build remarks from clang.
+    set(runtime_cxxmodules_env "ROOT_MODULES=DEBUG")
   endif()
 
   #---what rootcling command to use--------------------------

core/metacling/src/TCling.cxx

@@ -113,6 +113,7 @@ clang/LLVM technology.
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/Process.h"
+#include "llvm/Support/Program.h"
 
 #include <algorithm>
 #include <iostream>
@@ -1114,6 +1115,23 @@ static bool IsFromRootCling() {
   return foundSymbol;
 }
 
+static void loadModulePath(HeaderSearch& hdrSearch, const char* inputPath) {
+  if (inputPath) {
+     StringRef path = inputPath;
+     SmallVector<StringRef, 32> paths;
+     path.split(paths, ":");
+
+     for (StringRef path : paths) {
+      SmallString<128> ModuleMapFilePath = path;
+      llvm::sys::path::append(ModuleMapFilePath, "module.modulemap");
+
+      if (auto file = hdrSearch.getFileMgr().getFile(ModuleMapFilePath)) {
+         hdrSearch.loadModuleMapFile(file, false, FileID());
+      }
+    }
+  }
+}
+
 ////////////////////////////////////////////////////////////////////////////////
 /// Initialize the cling interpreter interface.
 
@@ -1129,8 +1147,6 @@ TCling::TCling(const char *name, const char *title)
 #ifdef R__USE_CXXMODULES
    useCxxModules = true;
 #endif
-   if (useCxxModules)
-     fHeaderParsingOnDemand = false;
 
    llvm::install_fatal_error_handler(&exceptionErrorHandler);
 
@@ -1253,6 +1269,19 @@ TCling::TCling(const char *name, const char *title)
    fMetaProcessor = new cling::MetaProcessor(*fInterpreter, fMPOuts);
 
    if (fInterpreter->getCI()->getLangOpts().Modules) {
+      // HeaderSearch& hdrSearch = fInterpreter->getCI()->getPreprocessor().getHeaderSearchInfo();
+      // hdrSearch.loadTopLevelSystemModules();
+      // loadModulePath(hdrSearch, gSystem->GetDynamicPath());
+      // fInterpreter->getCI()->getHeaderSearchOpts().AddPrebuiltModulePath(".");
+
+      auto &HdrSearchOpts = fInterpreter->getCI()->getHeaderSearchOpts();
+      llvm::StringRef DynPath = gSystem->GetDynamicPath();
+      while (!DynPath.empty()) {
+         std::pair<StringRef, StringRef> Split = DynPath.split(llvm::sys::EnvPathSeparator);
+         HdrSearchOpts.AddPrebuiltModulePath(Split.first);
+         DynPath = Split.second;
+      }
+      HdrSearchOpts.AddPrebuiltModulePath(".");
       // Setup core C++ modules if we have any to setup.
 
       // Load libc and stl first.
@@ -1363,6 +1392,11 @@ TCling::~TCling()
 void TCling::Initialize()
 {
    fClingCallbacks->Initialize();
+   if (fInterpreter->getCI()->getLangOpts().Modules && !IsFromRootCling()) {
+      // Load modules that we can't automatically load via rootmap files as they
+      // contain decls in namespaces which aren't supported.
+      LoadModules({"TMVA", "EG", "RGL", "TMVAGui", "Gpad", "GenVector", "Hist", "MathCore", "Net", "TreePlayer", "TreeViewer", "Graf"}, *fInterpreter);
+   }
 }
 
 ////////////////////////////////////////////////////////////////////////////////
@@ -1764,7 +1798,24 @@ void TCling::RegisterModule(const char* modulename,
       } // if (dyLibName)
    } // if (!lateRegistration)
 
-   if (hasHeaderParsingOnDemand && fwdDeclsCode){
+   clang::Sema &TheSema = fInterpreter->getSema();
+   bool ModuleWasSuccessfullyLoaded = false;
+   if (hasCxxModule) {
+      std::string ModuleName = llvm::StringRef(modulename).substr(3).str();
+      // FIXME: We should only complain for modules which we know to exist. For example, we should not complain about
+      // modules such as GenVector32 because it needs to fall back to GenVector.
+      ModuleWasSuccessfullyLoaded = LoadModule(ModuleName, *fInterpreter, /*Complain=*/ false);
+      if (!ModuleWasSuccessfullyLoaded) {
+         // Only report if we found the module in the modulemap.
+         clang::Preprocessor &PP = TheSema.getPreprocessor();
+         clang::HeaderSearch &headerSearch = PP.getHeaderSearchInfo();
+         clang::ModuleMap &moduleMap = headerSearch.getModuleMap();
+         if (moduleMap.findModule(ModuleName))
+            Info("TCling::RegisterModule", "Module %s in modulemap failed to load.", ModuleName.c_str());
+      }
+   }
+
+   if (!ModuleWasSuccessfullyLoaded && hasHeaderParsingOnDemand && fwdDeclsCode){
       // We now parse the forward declarations. All the classes are then modified
       // in order for them to have an external lexical storage.
       std::string fwdDeclsCodeLessEnums;
@@ -1901,24 +1952,6 @@ void TCling::RegisterModule(const char* modulename,
    if (fClingCallbacks)
      oldValue = SetClassAutoloading(false);
 
-   clang::Sema &TheSema = fInterpreter->getSema();
-
-   bool ModuleWasSuccessfullyLoaded = false;
-   if (hasCxxModule) {
-      std::string ModuleName = llvm::StringRef(modulename).substr(3).str();
-      // FIXME: We should only complain for modules which we know to exist. For example, we should not complain about
-      // modules such as GenVector32 because it needs to fall back to GenVector.
-      ModuleWasSuccessfullyLoaded = LoadModule(ModuleName, *fInterpreter, /*Complain=*/ false);
-      if (!ModuleWasSuccessfullyLoaded) {
-         // Only report if we found the module in the modulemap.
-         clang::Preprocessor &PP = TheSema.getPreprocessor();
-         clang::HeaderSearch &headerSearch = PP.getHeaderSearchInfo();
-         clang::ModuleMap &moduleMap = headerSearch.getModuleMap();
-         if (moduleMap.findModule(ModuleName))
-            Info("TCling::RegisterModule", "Module %s in modulemap failed to load.", ModuleName.c_str());
-      }
-   }
-
    { // scope within which diagnostics are de-activated
    // For now we disable diagnostics because we saw them already at
    // dictionary generation time. That won't be an issue with the PCMs.
@@ -5124,6 +5157,20 @@ Int_t TCling::LoadLibraryMap(const char* rootmapfile)
          }
          if (!skip) {
             void* dirp = gSystem->OpenDirectory(d);
+
+            // Load the modulemap from the dir and add it to the prebuilt module
+            // path.
+            // FIXME: This is ROOT quality code, refactor me.
+            fInterpreter->getCI()->getHeaderSearchOpts().AddPrebuiltModulePath(d.Data());
+            auto& hdrSearch = fInterpreter->getCI()->getPreprocessor().getHeaderSearchInfo();
+            SmallString<128> ModuleMapFilePath = StringRef(d.Data());
+            llvm::sys::path::append(ModuleMapFilePath, "module.modulemap");
+
+            if (auto file = hdrSearch.getFileMgr().getFile(ModuleMapFilePath)) {
+               hdrSearch.loadModuleMapFile(file, false, FileID());
+            }
+
+
             if (dirp) {
                if (gDebug > 3) {
                   Info("LoadLibraryMap", "%s", d.Data());

interpreter/llvm/src/tools/clang/include/clang/Basic/SourceManager.h

@@ -625,11 +625,66 @@ class SourceManager : public RefCountedBase<SourceManager> {
   /// expansion.
   SmallVector<SrcMgr::SLocEntry, 0> LocalSLocEntryTable;
 
+  template<typename T, unsigned ChunkSize>
+  class SparseVector {
+    typedef std::array<T, ChunkSize> Chunk;
+    std::vector<Chunk*> Chunks;
+    std::size_t realSize = 0;
+    std::size_t allocatedChunks = 0;
+
+    Chunk *getChunk(std::size_t i) {
+      std::size_t ChunkIndex = i / ChunkSize;
+
+      Chunk* Result = Chunks.at(ChunkIndex);
+      if (Result == nullptr) {
+        Result = new Chunk();
+        Chunks[ChunkIndex] = Result;
+        allocatedChunks++;
+      }
+      return Result;
+    }
+  public:
+    SparseVector() {
+    }
+
+    typedef T value_type;
+
+    T& operator[](std::size_t i) {
+      Chunk *C = getChunk(i);
+      std::size_t Rem = i % ChunkSize;
+      return (*C)[Rem];
+    }
+
+    std::size_t size() const {
+      return realSize;
+    }
+
+    std::size_t capacity() const {
+      return allocatedChunks * ChunkSize;
+    }
+
+    void resize(std::size_t s) {
+      realSize = s;
+      Chunks.resize((s / ChunkSize) + 1);
+    }
+
+    void clear() {
+      Chunks.clear();
+      realSize = 0;
+      allocatedChunks = 0;
+    }
+
+    bool empty() const {
+      return realSize == 0;
+    }
+  };
+
   /// \brief The table of SLocEntries that are loaded from other modules.
   ///
   /// Negative FileIDs are indexes into this table. To get from ID to an index,
   /// use (-ID - 2).
-  mutable SmallVector<SrcMgr::SLocEntry, 0> LoadedSLocEntryTable;
+  mutable SparseVector<SrcMgr::SLocEntry, 4096> LoadedSLocEntryTable;
+  //mutable SmallVector<SrcMgr::SLocEntry, 0> LoadedSLocEntryTable;
 
   /// \brief The starting offset of the next local SLocEntry.
   ///