[cxxmodules] Teach cling to build modules; Build dependent modules explicitly.

cmake/modules/RootMacros.cmake

@@ -254,6 +254,64 @@ function(ROOT_GET_INSTALL_DIR result)
   SET(${result} "${shared_lib_install_dir}" PARENT_SCOPE)
 endfunction(ROOT_GET_INSTALL_DIR)
 
+#---------------------------------------------------------------------------------------------------
+#---ROOT_GENERATE_CXXMODULE( module_name DEPENDENCIES module_name1 module_name2 )
+#
+function(ROOT_GENERATE_CXXMODULE module_name headers_location)
+  CMAKE_PARSE_ARGUMENTS(ARG "" "" "DEPENDENCIES" ${ARGN})
+
+  # rootcling bare-cling -I etc/ -x c++ -I /usr/include/c++/7/ -fmodules-cache-path=lib/
+  #                      -fmodules -Xclang -emit-module -fmodule-name=std /usr/include/c++/7/module.modulemap
+  #                      -o std.pcm
+  set(root_libdir)
+  ROOT_GET_LIBRARY_OUTPUT_DIR(root_libdir)
+
+  # Get all available modulemaps which ROOT will use.
+  get_property(root_modulemaps GLOBAL PROPERTY ROOT_MODULEMAPS)
+
+  set(real_modulemap_file ${headers_location}/module.modulemap)
+
+  if (NOT ${real_modulemap_file} IN_LIST root_modulemaps)
+    # The modulemap may be virtual (automatically mounted by rootcling)
+    set(virtual_modulemap_location "${CMAKE_BINARY_DIR}/etc/cling")
+    if (${module_name} MATCHES "Cling_Runtime")
+      # Cling_Runtime and Cling_Runtime_Extra are special we copy their
+      # module.modulemap.build as a module.modulemap in etc/cling.
+      set(virtual_modulemap_location "${virtual_modulemap_location}/module.modulemap")
+    else()
+      set(virtual_modulemap_location "${virtual_modulemap_location}/${module_name}.modulemap")
+    endif()
+
+    if (${virtual_modulemap_location} IN_LIST root_modulemaps)
+      set(real_modulemap_file ${virtual_modulemap_location})
+    elseif(NOT EXISTS ${real_modulemap_file})
+      message(FATAL_ERROR "Neither '${headers_location}/module.modulemap' nor"
+                          " '${real_modulemap_file}' exists for module '${module_name}'!")
+    endif()
+  endif()
+
+  set(module_file_location ${root_libdir}/${module_name}.pcm)
+  set(file_dependencies ${real_modulemap_file})
+  foreach(dep ${ARG_DEPENDENCIES})
+    # Turn the target to a file.pcm. This allows us to add it as a file dependency
+    # in the DEPENDS clause of add_custom_command. This is the only way I found
+    # to trigger rebuild of the full chain of dependencies.
+    # Eg: a.pcm <- b.pcm <- c.pcm; rm a.pcm; ninja c.pcm; rebuilds a.pcm and b.pcm
+    list(APPEND file_dependencies ${root_libdir}/${dep}.pcm)
+  endforeach()
+
+  add_custom_command(OUTPUT ${module_file_location}
+    COMMAND $<TARGET_FILE:rootcling_stage1>
+                     bare-cling -xc++ -I${CMAKE_BINARY_DIR}/etc -I${headers_location}
+                     -fmodules -Xclang -emit-module -fmodule-name=${module_name}
+                     -fmodules-cache-path=${root_libdir} -o ${module_file_location}
+                     ${headers_location}/module.modulemap
+                     DEPENDS ${file_dependencies})
+
+  add_custom_target(${module_name} ALL DEPENDS ${module_file_location} ${file_dependencies} ${ARG_DEPENDENCIES})
+
+endfunction(ROOT_GENERATE_CXXMODULE)
+
 #---------------------------------------------------------------------------------------------------
 #---ROOT_GENERATE_DICTIONARY( dictionary headerfiles NODEPHEADERS ghdr1 ghdr2 ...
 #                                                    MODULE module DEPENDENCIES dep1 dep2

core/CMakeLists.txt

@@ -155,6 +155,42 @@ add_subdirectory(rootcling_stage1)
 
 #-------------------------------------------------------------------------------
 ROOT_LINKER_LIBRARY(Core $<TARGET_OBJECTS:BaseTROOT> ${objectlibs} BUILTINS LZMA)
+set(extra_cxxmodules_dependencies)
+if(runtime_cxxmodules)
+  # We load the builtin modulemap at rootcling bare-cling init.
+  # FIXME: What happens when the <resource_dir>/include/* changes?
+  #ROOT_GENERATE_CXXMODULE(_Builtin_stddef_max_align_t)
+  #ROOT_GENERATE_CXXMODULE(_Builtin_intrinsics)
+  set (libc)
+  if (APPLE)
+    set(libc Darwin)
+  else()
+    set(libc libc)
+  endif()
+
+  get_property(c_headers_location GLOBAL PROPERTY CLING_C_HEADERS_LOCATION)
+  ROOT_GENERATE_CXXMODULE(${libc} ${c_headers_location})
+
+  get_property(cxx_headers_location GLOBAL PROPERTY CLING_CXX_HEADERS_LOCATION)
+  # Assume the first component is where the public headers are.
+  string(FIND ${cxx_headers_location} ":" idx)
+  string(SUBSTRING ${cxx_headers_location} 0 ${idx} cxx_headers_location)
+  ROOT_GENERATE_CXXMODULE(std ${cxx_headers_location} DEPENDENCIES ${libc})
+
+  ROOT_GENERATE_CXXMODULE(Cling_Runtime ${CMAKE_BINARY_DIR}/etc/cling DEPENDENCIES std)
+  ROOT_GENERATE_CXXMODULE(Cling_Runtime_Extra ${CMAKE_BINARY_DIR}/etc/cling DEPENDENCIES Cling_Runtime)
+
+  if(cuda)
+    ROOT_GENERATE_CXXMODULE(cuda ${CUDA_INCLUDE_DIRS})
+  endif(cuda)
+
+  ROOT_GENERATE_CXXMODULE(ROOT_Config ${CMAKE_BINARY_DIR}/include DEPENDENCIES Cling_Runtime_Extra)
+  ROOT_GENERATE_CXXMODULE(ROOT_Rtypes ${CMAKE_BINARY_DIR}/include DEPENDENCIES ROOT_Config)
+  ROOT_GENERATE_CXXMODULE(ROOT_Foundation_C ${CMAKE_BINARY_DIR}/include DEPENDENCIES ROOT_Rtypes)
+  ROOT_GENERATE_CXXMODULE(ROOT_Foundation_Stage1_NoRTTI ${CMAKE_BINARY_DIR}/include DEPENDENCIES ROOT_Foundation_C)
+
+  set(extra_cxxmodules_dependencies ${libc} std Cling_Runtime Cling_Runtime_Extra ROOT_Config ROOT_Rtypes ROOT_Foundation_Stage1_NoRTTI)
+endif(runtime_cxxmodules)
 
 if (libcxx AND NOT APPLE)
   # In case we use libcxx and glibc together there is a mismatch of the
@@ -217,6 +253,8 @@ ROOT_GENERATE_DICTIONARY(G__Core
     -writeEmptyRootPCM
   LINKDEF
     base/inc/LinkDef.h
+  DEPENDENCIES
+    ${extra_cxxmodules_dependencies}
 )
 
 target_include_directories(G__Core PRIVATE ${CMAKE_SOURCE_DIR}/core/clingutils/inc)

core/clingutils/CMakeLists.txt

@@ -125,6 +125,8 @@ if (runtime_cxxmodules)
   endif()
 endif(runtime_cxxmodules)
 
+set(root_etc_cling_modulemaps)
+
 foreach(file ${custom_modulemaps}
         Interpreter/DynamicExprInfo.h
         Interpreter/DynamicLookupRuntimeUniverse.h
@@ -137,7 +139,10 @@ foreach(file ${custom_modulemaps}
   set(dest_file ${file})
   if (${file} STREQUAL "module.modulemap.build")
     set(dest_file "module.modulemap")
-  elseif(NOT ${file} MATCHES ".*modulemap")
+  endif()
+  if(${file} MATCHES ".*modulemap")
+    list(APPEND root_etc_cling_modulemaps ${CMAKE_BINARY_DIR}/etc/cling/${dest_file})
+  else()
     # We do not want our modulemap to be considered part of the PCH.
     set_property(GLOBAL APPEND PROPERTY CLINGETCPCH etc/cling/${dest_file})
   endif()
@@ -165,6 +170,7 @@ foreach(file wchar.h bits/stat.h bits/time.h)
 endforeach()
 
 set(stamp_file ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/LLVMRES.stamp)
+get_property(root_modulemaps GLOBAL PROPERTY ROOT_MODULEMAPS)
 if(MSVC)
   add_custom_command(OUTPUT ${stamp_file}
         COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/etc/cling/lib/clang/${CLANG_RESOURCE_DIR_VERSION}/include
@@ -174,6 +180,7 @@ if(MSVC)
         ${CMAKE_BINARY_DIR}/etc/cling/lib/clang/${CLANG_RESOURCE_DIR_VERSION}/include		
         COMMAND ${CMAKE_COMMAND} -E touch ${stamp_file}
         DEPENDS ${files_to_copy}
+        BYPRODUCTS ${root_etc_cling_modulemaps}
         COMMENT "Copying LLVM resource and header files")
 else()
   add_custom_command(OUTPUT ${stamp_file}
@@ -193,8 +200,14 @@ else()
         ${copy_commands}
         COMMAND ${CMAKE_COMMAND} -E touch ${stamp_file}
         DEPENDS ${files_to_copy}
+        BYPRODUCTS ${root_etc_cling_modulemaps}
         COMMENT "Copying LLVM resource and header files")
 endif()
+
+# Register the main modulemap file.
+list(APPEND root_etc_cling_modulemaps ${CMAKE_BINARY_DIR}/include/module.modulemap)
+set_property(GLOBAL APPEND PROPERTY ROOT_MODULEMAPS ${root_etc_cling_modulemaps})
+
 add_custom_target(LLVMRES DEPENDS ${stamp_file} CLING)
 # CLING is a shorthand for CLING_LIBRARIES and some other clang-specific
 # dependencies which ensure the correct order of building. Then the cling header

core/dictgen/src/rootcling_impl.cxx

@@ -3938,7 +3938,9 @@ static llvm::StringRef GetModuleNameFromRdictName(llvm::StringRef rdictName)
 {
    // Try to get the module name in the modulemap based on the filepath.
    llvm::StringRef moduleName = llvm::sys::path::filename(rdictName);
-   moduleName.consume_front("lib");
+   // FIXME: Rename the libc module and delete this special case.
+   if (!rdictName.startswith("libc"))
+      moduleName.consume_front("lib");
    moduleName.consume_back(".pcm");
    moduleName.consume_back("_rdict");
    return moduleName;
@@ -3992,23 +3994,37 @@ int RootClingMain(int argc,
 
    llvm::cl::ParseCommandLineOptions(argc, argv, "rootcling");
 
-   std::string llvmResourceDir = std::string(gDriverConfig->fTROOT__GetEtcDir()) + "/cling";
+   llvm::SmallString<512> llvmDir(gDriverConfig->fTROOT__GetEtcDir());
+   llvm::sys::path::append(llvmDir, "cling");
+   llvm::SmallString<512> resourcePathInc(llvmDir);
+   llvm::sys::path::append(resourcePathInc, "lib", "clang", "5.0.0");
+   assert(llvm::sys::fs::exists(resourcePathInc));
+   std::string llvmResourceDir = resourcePathInc.str().str();
    if (gBareClingSubcommand) {
       std::vector<const char *> clingArgsC;
       clingArgsC.push_back(executableFileName);
       // Help cling finds its runtime (RuntimeUniverse.h and such).
+      //clingArgsC.push_back("-I");
+      //clingArgsC.push_back(gDriverConfig->fTROOT__GetEtcDir());
+
+      llvm::sys::path::append(resourcePathInc, "include");
       clingArgsC.push_back("-I");
-      clingArgsC.push_back(gDriverConfig->fTROOT__GetEtcDir());
+      clingArgsC.push_back(resourcePathInc.c_str());
+      // Add the builtin modulemap for clang. See -fbuiltin-module-map
+      llvm::SmallString<512> builtinModuleMap("-fmodule-map-file=");
+      builtinModuleMap.append(resourcePathInc);
+      llvm::sys::path::append(builtinModuleMap, "module.modulemap");
+      clingArgsC.push_back(resourcePathInc.c_str());
 
-      //clingArgsC.push_back("-resource-dir");
-      //clingArgsC.push_back(llvmResourceDir.c_str());
+      //clingArgsC.push_back("-nobuiltininc");
+      clingArgsC.push_back("-noruntime");
 
       for (const std::string& Opt : gOptBareClingSink)
          clingArgsC.push_back(Opt.c_str());
 
       auto interp = llvm::make_unique<cling::Interpreter>(clingArgsC.size(),
                                                           &clingArgsC[0],
-                                                          llvmResourceDir.c_str());
+                                                          llvmDir.c_str());
       // FIXME: Diagnose when we have misspelled a flag. Currently we show no
       // diagnostic and report exit as success.
       return interp->getDiagnostics().hasFatalErrorOccurred();
@@ -4218,40 +4234,11 @@ int RootClingMain(int argc,
 
       // Specify the module name that we can lookup the module in the modulemap.
       outputFile = llvm::sys::path::stem(gOptSharedLibFileName).str();
-      // Try to get the module name in the modulemap based on the filepath.
       moduleName = GetModuleNameFromRdictName(outputFile);
-
       clingArgsInterpreter.push_back("-fmodule-name");
       clingArgsInterpreter.push_back(moduleName.str());
 
       std::string moduleCachePath = llvm::sys::path::parent_path(gOptSharedLibFileName).str();
-      // FIXME: This is a horrible workaround to fix the incremental builds.
-      // The enumerated modules are built by clang impicitly based on #include of
-      // a header which is contained within that module. The build system has
-      // no way to track dependencies on them and trigger a rebuild.
-      // A possible solution can be to disable completely the implicit build of
-      // modules and each module to be built by rootcling. We need to teach
-      // rootcling how to build modules with no IO support.
-      if (moduleName == "Core") {
-         assert(gDriverConfig->fBuildingROOTStage1);
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "_Builtin_intrinsics.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "_Builtin_stddef_max_align_t.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "Cling_Runtime.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "Cling_Runtime_Extra.pcm").str().c_str());
-#ifdef R__MACOSX
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "Darwin.pcm").str().c_str());
-#else
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "libc.pcm").str().c_str());
-#endif
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "std.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "cuda.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "boost.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "tinyxml2.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "ROOT_Config.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "ROOT_Rtypes.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "ROOT_Foundation_C.pcm").str().c_str());
-         remove((moduleCachePath + llvm::sys::path::get_separator() + "ROOT_Foundation_Stage1_NoRTTI.pcm").str().c_str());
-      }
 
       // Set the C++ modules output directory to the directory where we generate
       // the shared library.
@@ -4296,7 +4283,7 @@ int RootClingMain(int argc,
 #endif
 
       owningInterpPtr.reset(new cling::Interpreter(clingArgsC.size(), &clingArgsC[0],
-                                                   llvmResourceDir.c_str()));
+                                                   llvmDir.c_str()));
       interpPtr = owningInterpPtr.get();
       // Force generation of _Builtin_intrinsics by rootcling_stage1.
       // The rest of the modules are implicitly generated by cling when including
@@ -4358,7 +4345,7 @@ int RootClingMain(int argc,
    if (gOptCxxModule) {
       for (llvm::StringRef DepMod : gOptModuleDependencies) {
          if (DepMod.endswith("_rdict.pcm")) {
-            ROOT::TMetaUtils::Warning(0, "'%s' value is deprecated. Please use [<fullpat>]%s.pcm\n",
+            ROOT::TMetaUtils::Warning(0, "'%s' value is deprecated. Please use [<fullpath>]%s.pcm\n",
                                       DepMod.data(),
                                       GetModuleNameFromRdictName(DepMod).str().data());
          }
@@ -4693,7 +4680,7 @@ int RootClingMain(int argc,
       clingArgs.push_back("-Ietc/cling/cint"); // For multiset and multimap
 
       if (!ldefr.Parse(selectionRules, interpPragmaSource, clingArgs,
-                       llvmResourceDir.c_str())) {
+                       llvmDir.c_str())) {
          ROOT::TMetaUtils::Error(0, "Parsing Linkdef file %s\n", linkdefFilename.c_str());
          rootclingRetCode += 1;
       } else {

core/metacling/src/TCling.cxx

@@ -1455,7 +1455,7 @@ TCling::TCling(const char *name, const char *title, const char* const argv[])
         extraArgs && *extraArgs; ++extraArgs) {
       if (!strcmp(*extraArgs, "-resource-dir")) {
          // Take the next arg as the llvm resource directory.
-         llvmResourceDir = *(++extraArgs);
+         //llvmResourceDir = *(++extraArgs);
       } else {
          interpArgs.push_back(*extraArgs);
       }

interpreter/cling/include/cling/libc.modulemap

@@ -43,6 +43,7 @@ module "libc" [system] [extern_c] [no_undeclared_includes] {
     header "signal.h"
   }
   module "stddef.h" {
+    requires !header_existence
     export *
     textual header "stddef.h"
   }