Prepare for LLVM 11

src/libprofiler_builtins/build.rs

@@ -24,6 +24,12 @@ fn main() {
         "InstrProfilingUtil.c",
         "InstrProfilingValue.c",
         "InstrProfilingWriter.c",
+        // This file was renamed in LLVM 10.
+        "InstrProfilingRuntime.cc",
+        "InstrProfilingRuntime.cpp",
+        // These files were added in LLVM 11.
+        "InstrProfilingInternal.c",
+        "InstrProfilingBiasVar.c",
     ];
 
     if target.contains("msvc") {
@@ -69,14 +75,12 @@ fn main() {
 
     let src_root = root.join("lib").join("profile");
     for src in profile_sources {
-        cfg.file(src_root.join(src));
+        let path = src_root.join(src);
+        if path.exists() {
+            cfg.file(path);
+        }
     }
 
-    // The file was renamed in LLVM 10.
-    let old_runtime_path = src_root.join("InstrProfilingRuntime.cc");
-    let new_runtime_path = src_root.join("InstrProfilingRuntime.cpp");
-    cfg.file(if old_runtime_path.exists() { old_runtime_path } else { new_runtime_path });
-
     cfg.include(root.join("include"));
     cfg.warnings(false);
     cfg.compile("profiler-rt");

src/librustc_codegen_llvm/back/lto.rs

@@ -797,6 +797,7 @@ pub unsafe fn optimize_thin_module(
         kind: ModuleKind::Regular,
     };
     {
+        let target = &*module.module_llvm.tm;
         let llmod = module.module_llvm.llmod();
         save_temp_bitcode(&cgcx, &module, "thin-lto-input");
 
@@ -833,7 +834,7 @@ pub unsafe fn optimize_thin_module(
         {
             let _timer =
                 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name());
-            if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod) {
+            if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) {
                 let msg = "failed to prepare thin LTO module";
                 return Err(write::llvm_err(&diag_handler, msg));
             }
@@ -865,7 +866,7 @@ pub unsafe fn optimize_thin_module(
         {
             let _timer =
                 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name());
-            if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod) {
+            if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) {
                 let msg = "failed to prepare thin LTO module";
                 return Err(write::llvm_err(&diag_handler, msg));
             }

src/librustc_codegen_llvm/llvm/ffi.rs

@@ -233,6 +233,8 @@ pub enum TypeKind {
     Metadata = 14,
     X86_MMX = 15,
     Token = 16,
+    ScalableVector = 17,
+    BFloat = 18,
 }
 
 impl TypeKind {
@@ -255,6 +257,8 @@ impl TypeKind {
             TypeKind::Metadata => rustc_codegen_ssa::common::TypeKind::Metadata,
             TypeKind::X86_MMX => rustc_codegen_ssa::common::TypeKind::X86_MMX,
             TypeKind::Token => rustc_codegen_ssa::common::TypeKind::Token,
+            TypeKind::ScalableVector => rustc_codegen_ssa::common::TypeKind::ScalableVector,
+            TypeKind::BFloat => rustc_codegen_ssa::common::TypeKind::BFloat,
         }
     }
 }
@@ -2141,10 +2145,18 @@ extern "C" {
         PreservedSymbols: *const *const c_char,
         PreservedSymbolsLen: c_uint,
     ) -> Option<&'static mut ThinLTOData>;
-    pub fn LLVMRustPrepareThinLTORename(Data: &ThinLTOData, Module: &Module) -> bool;
+    pub fn LLVMRustPrepareThinLTORename(
+        Data: &ThinLTOData,
+        Module: &Module,
+        Target: &TargetMachine,
+    ) -> bool;
     pub fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool;
     pub fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool;
-    pub fn LLVMRustPrepareThinLTOImport(Data: &ThinLTOData, Module: &Module) -> bool;
+    pub fn LLVMRustPrepareThinLTOImport(
+        Data: &ThinLTOData,
+        Module: &Module,
+        Target: &TargetMachine,
+    ) -> bool;
     pub fn LLVMRustGetThinLTOModuleImports(
         Data: *const ThinLTOData,
         ModuleNameCallback: ThinLTOModuleNameCallback,

src/librustc_codegen_ssa/common.rs

@@ -98,6 +98,8 @@ pub enum TypeKind {
     Metadata,
     X86_MMX,
     Token,
+    ScalableVector,
+    BFloat,
 }
 
 // FIXME(mw): Anything that is produced via DepGraph::with_task() must implement

src/rustllvm/PassWrapper.cpp

@@ -49,8 +49,10 @@ typedef struct LLVMOpaqueTargetMachine *LLVMTargetMachineRef;
 
 DEFINE_STDCXX_CONVERSION_FUNCTIONS(Pass, LLVMPassRef)
 DEFINE_STDCXX_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef)
+#if LLVM_VERSION_LT(11, 0)
 DEFINE_STDCXX_CONVERSION_FUNCTIONS(PassManagerBuilder,
                                    LLVMPassManagerBuilderRef)
+#endif
 
 extern "C" void LLVMInitializePasses() {
   PassRegistry &Registry = *PassRegistry::getPassRegistry();
@@ -343,17 +345,17 @@ enum class LLVMRustPassBuilderOptLevel {
 static PassBuilder::OptimizationLevel fromRust(LLVMRustPassBuilderOptLevel Level) {
   switch (Level) {
   case LLVMRustPassBuilderOptLevel::O0:
-    return PassBuilder::O0;
+    return PassBuilder::OptimizationLevel::O0;
   case LLVMRustPassBuilderOptLevel::O1:
-    return PassBuilder::O1;
+    return PassBuilder::OptimizationLevel::O1;
   case LLVMRustPassBuilderOptLevel::O2:
-    return PassBuilder::O2;
+    return PassBuilder::OptimizationLevel::O2;
   case LLVMRustPassBuilderOptLevel::O3:
-    return PassBuilder::O3;
+    return PassBuilder::OptimizationLevel::O3;
   case LLVMRustPassBuilderOptLevel::Os:
-    return PassBuilder::Os;
+    return PassBuilder::OptimizationLevel::Os;
   case LLVMRustPassBuilderOptLevel::Oz:
-    return PassBuilder::Oz;
+    return PassBuilder::OptimizationLevel::Oz;
   default:
     report_fatal_error("Bad PassBuilderOptLevel.");
   }
@@ -796,8 +798,13 @@ LLVMRustOptimizeWithNewPassManager(
   // We manually collect pipeline callbacks so we can apply them at O0, where the
   // PassBuilder does not create a pipeline.
   std::vector<std::function<void(ModulePassManager &)>> PipelineStartEPCallbacks;
+#if LLVM_VERSION_GE(11, 0)
+  std::vector<std::function<void(ModulePassManager &, PassBuilder::OptimizationLevel)>>
+      OptimizerLastEPCallbacks;
+#else
   std::vector<std::function<void(FunctionPassManager &, PassBuilder::OptimizationLevel)>>
       OptimizerLastEPCallbacks;
+#endif
 
   if (VerifyIR) {
     PipelineStartEPCallbacks.push_back([VerifyIR](ModulePassManager &MPM) {
@@ -811,6 +818,14 @@ LLVMRustOptimizeWithNewPassManager(
           SanitizerOptions->SanitizeMemoryTrackOrigins,
           SanitizerOptions->SanitizeMemoryRecover,
           /*CompileKernel=*/false);
+#if LLVM_VERSION_GE(11, 0)
+      OptimizerLastEPCallbacks.push_back(
+        [Options](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
+          MPM.addPass(MemorySanitizerPass(Options));
+          MPM.addPass(createModuleToFunctionPassAdaptor(MemorySanitizerPass(Options)));
+        }
+      );
+#else
 #if LLVM_VERSION_GE(10, 0)
       PipelineStartEPCallbacks.push_back([Options](ModulePassManager &MPM) {
         MPM.addPass(MemorySanitizerPass(Options));
@@ -821,9 +836,18 @@ LLVMRustOptimizeWithNewPassManager(
           FPM.addPass(MemorySanitizerPass(Options));
         }
       );
+#endif
     }
 
     if (SanitizerOptions->SanitizeThread) {
+#if LLVM_VERSION_GE(11, 0)
+      OptimizerLastEPCallbacks.push_back(
+        [](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
+          MPM.addPass(ThreadSanitizerPass());
+          MPM.addPass(createModuleToFunctionPassAdaptor(ThreadSanitizerPass()));
+        }
+      );
+#else
 #if LLVM_VERSION_GE(10, 0)
       PipelineStartEPCallbacks.push_back([](ModulePassManager &MPM) {
         MPM.addPass(ThreadSanitizerPass());
@@ -834,9 +858,22 @@ LLVMRustOptimizeWithNewPassManager(
           FPM.addPass(ThreadSanitizerPass());
         }
       );
+#endif
     }
 
     if (SanitizerOptions->SanitizeAddress) {
+#if LLVM_VERSION_GE(11, 0)
+      OptimizerLastEPCallbacks.push_back(
+        [SanitizerOptions](ModulePassManager &MPM, PassBuilder::OptimizationLevel Level) {
+          MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
+          MPM.addPass(ModuleAddressSanitizerPass(
+              /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
+          MPM.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(
+              /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover,
+              /*UseAfterScope=*/true)));
+        }
+      );
+#else
       PipelineStartEPCallbacks.push_back([&](ModulePassManager &MPM) {
         MPM.addPass(RequireAnalysisPass<ASanGlobalsMetadataAnalysis, Module>());
       });
@@ -853,21 +890,27 @@ LLVMRustOptimizeWithNewPassManager(
               /*CompileKernel=*/false, SanitizerOptions->SanitizeAddressRecover));
         }
       );
+#endif
     }
   }
 
   ModulePassManager MPM(DebugPassManager);
   if (!NoPrepopulatePasses) {
-    if (OptLevel == PassBuilder::O0) {
+    if (OptLevel == PassBuilder::OptimizationLevel::O0) {
       for (const auto &C : PipelineStartEPCallbacks)
         C(MPM);
 
+#if LLVM_VERSION_GE(11, 0)
+      for (const auto &C : OptimizerLastEPCallbacks)
+        C(MPM, OptLevel);
+#else
       if (!OptimizerLastEPCallbacks.empty()) {
         FunctionPassManager FPM(DebugPassManager);
         for (const auto &C : OptimizerLastEPCallbacks)
           C(FPM, OptLevel);
         MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
       }
+#endif
 
       MPM.addPass(AlwaysInlinerPass(EmitLifetimeMarkers));
 
@@ -892,12 +935,17 @@ LLVMRustOptimizeWithNewPassManager(
         break;
       case LLVMRustOptStage::PreLinkThinLTO:
         MPM = PB.buildThinLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
+#if LLVM_VERSION_GE(11, 0)
+        for (const auto &C : OptimizerLastEPCallbacks)
+          C(MPM, OptLevel);
+#else
         if (!OptimizerLastEPCallbacks.empty()) {
           FunctionPassManager FPM(DebugPassManager);
           for (const auto &C : OptimizerLastEPCallbacks)
             C(FPM, OptLevel);
           MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
         }
+#endif
         break;
       case LLVMRustOptStage::PreLinkFatLTO:
         MPM = PB.buildLTOPreLinkDefaultPipeline(OptLevel, DebugPassManager);
@@ -994,10 +1042,10 @@ class RustAssemblyAnnotationWriter : public AssemblyAnnotationWriter {
     const Value *Value;
     if (const CallInst *CI = dyn_cast<CallInst>(I)) {
       Name = "call";
-      Value = CI->getCalledValue();
+      Value = CI->getCalledOperand();
     } else if (const InvokeInst* II = dyn_cast<InvokeInst>(I)) {
       Name = "invoke";
-      Value = II->getCalledValue();
+      Value = II->getCalledOperand();
     } else {
       // Could demangle more operations, e. g.
       // `store %place, @function`.
@@ -1335,10 +1383,33 @@ LLVMRustFreeThinLTOData(LLVMRustThinLTOData *Data) {
 // `ProcessThinLTOModule` function. Here they're split up into separate steps
 // so rustc can save off the intermediate bytecode between each step.
 
+#if LLVM_VERSION_GE(11, 0)
+static bool
+clearDSOLocalOnDeclarations(Module &Mod, TargetMachine &TM) {
+  // When linking an ELF shared object, dso_local should be dropped. We
+  // conservatively do this for -fpic.
+  bool ClearDSOLocalOnDeclarations =
+      TM.getTargetTriple().isOSBinFormatELF() &&
+      TM.getRelocationModel() != Reloc::Static &&
+      Mod.getPIELevel() == PIELevel::Default;
+  return ClearDSOLocalOnDeclarations;
+}
+#endif
+
 extern "C" bool
-LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
+LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
+                             LLVMTargetMachineRef TM) {
   Module &Mod = *unwrap(M);
-  if (renameModuleForThinLTO(Mod, Data->Index)) {
+  TargetMachine &Target = *unwrap(TM);
+
+#if LLVM_VERSION_GE(11, 0)
+  bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
+  bool error = renameModuleForThinLTO(Mod, Data->Index, ClearDSOLocal);
+#else
+  bool error = renameModuleForThinLTO(Mod, Data->Index);
+#endif
+
+  if (error) {
     LLVMRustSetLastError("renameModuleForThinLTO failed");
     return false;
   }
@@ -1362,8 +1433,10 @@ LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef
 }
 
 extern "C" bool
-LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
+LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M,
+                             LLVMTargetMachineRef TM) {
   Module &Mod = *unwrap(M);
+  TargetMachine &Target = *unwrap(TM);
 
   const auto &ImportList = Data->ImportLists.lookup(Mod.getModuleIdentifier());
   auto Loader = [&](StringRef Identifier) {
@@ -1399,7 +1472,12 @@ LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M) {
 
     return MOrErr;
   };
+#if LLVM_VERSION_GE(11, 0)
+  bool ClearDSOLocal = clearDSOLocalOnDeclarations(Mod, Target);
+  FunctionImporter Importer(Data->Index, Loader, ClearDSOLocal);
+#else
   FunctionImporter Importer(Data->Index, Loader);
+#endif
   Expected<bool> Result = Importer.importFunctions(Mod, ImportList);
   if (!Result) {
     LLVMRustSetLastError(toString(Result.takeError()).c_str());
@@ -1558,22 +1636,11 @@ LLVMRustThinLTOPatchDICompileUnit(LLVMModuleRef Mod, DICompileUnit *Unit) {
   }
 
   // Use LLVM's built-in `DebugInfoFinder` to find a bunch of debuginfo and
-  // process it recursively. Note that we specifically iterate over instructions
-  // to ensure we feed everything into it.
+  // process it recursively. Note that we used to specifically iterate over
+  // instructions to ensure we feed everything into it, but `processModule`
+  // started doing this the same way in LLVM 7 (commit d769eb36ab2b8).
   DebugInfoFinder Finder;
   Finder.processModule(*M);
-  for (Function &F : M->functions()) {
-    for (auto &FI : F) {
-      for (Instruction &BI : FI) {
-        if (auto Loc = BI.getDebugLoc())
-          Finder.processLocation(*M, Loc);
-        if (auto DVI = dyn_cast<DbgValueInst>(&BI))
-          Finder.processValue(*M, DVI);
-        if (auto DDI = dyn_cast<DbgDeclareInst>(&BI))
-          Finder.processDeclare(*M, DDI);
-      }
-    }
-  }
 
   // After we've found all our debuginfo, rewrite all subprograms to point to
   // the same `DICompileUnit`.