Upgrade building LLVM-from-source to version 18.1.3

This newer version addresses a miscompile affecting `example_cloth.py`
when running it on the CPU.

The default build still uses pre-compiled LLVM 15 static libraries
fetched from packman, for now, but when we're building LLVM from source
we can use the more recent 18.1.3 version.

Note that while this still works with GCC 7.4, the static libraries
built with that version of GCC have an ABI incompatibility with at least
GCC 9 and 10. This affects tests such as `test_copy.py`. Use of
`-fabi-version=11` does not address this issue. This isn't a problem for
from-source builds as long as these static libraries are not attempted
to be used with newer versions of GCC (i.e. not stored in packman).

Author: c0d1f1ed

build_llvm.py

@@ -58,7 +58,7 @@ def build_from_source_for_arch(args, arch, llvm_source):
         print(f"Cloning LLVM project from {repo_url}...")
 
         shallow_clone = True  # https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/
-        version = "15.0.7"
+        version = "18.1.3"
         if shallow_clone:
             repo = Repo.clone_from(
                 repo_url, to_path=llvm_source, single_branch=True, branch=f"llvmorg-{version}", depth=1
@@ -71,9 +71,11 @@ def build_from_source_for_arch(args, arch, llvm_source):
 
     # CMake supports Debug, Release, RelWithDebInfo, and MinSizeRel builds
     if args.mode == "release":
+        msvc_runtime = "MultiThreaded"
         # prefer smaller size over aggressive speed
         cmake_build_type = "MinSizeRel"
     else:
+        msvc_runtime = "MultiThreadedDebug"
         # When args.mode == "debug" we build a Debug version of warp.dll but
         # we generally don't want warp-clang.dll to be a slow Debug version.
         if args.debug_llvm:
@@ -114,10 +116,7 @@ def build_from_source_for_arch(args, arch, llvm_source):
         "-B", build_path,
         "-G", "Ninja",
         "-D", f"CMAKE_BUILD_TYPE={cmake_build_type}",
-        "-D", "LLVM_USE_CRT_RELEASE=MT",
-        "-D", "LLVM_USE_CRT_MINSIZEREL=MT",
-        "-D", "LLVM_USE_CRT_DEBUG=MTd",
-        "-D", "LLVM_USE_CRT_RELWITHDEBINFO=MTd",
+        "-D", f"CMAKE_MSVC_RUNTIME_LIBRARY={msvc_runtime}",
         "-D", f"LLVM_TARGETS_TO_BUILD={target_backend};NVPTX",
         "-D", "LLVM_ENABLE_PROJECTS=clang",
         "-D", "LLVM_ENABLE_ZLIB=FALSE",
@@ -329,6 +328,7 @@ def build_warp_clang_for_arch(args, lib_name, arch):
 
         if os.name == "nt":
             libs.append("Version.lib")
+            libs.append("Ws2_32.lib")
             libs.append(f'/LIBPATH:"{libpath}"')
         else:
             libs = [f"-l{lib[3:-2]}" for lib in libs if os.path.splitext(lib)[1] == ".a"]

warp/native/clang/clang.cpp

@@ -11,6 +11,11 @@
 #include <clang/Frontend/CompilerInstance.h>
 #include <clang/Basic/DiagnosticOptions.h>
 #include <clang/Frontend/TextDiagnosticPrinter.h>
+#if LLVM_VERSION_MAJOR >= 18
+#include <llvm/Frontend/Debug/Options.h>
+#else
+#include <llvm/Support/CodeGen.h>
+#endif
 #include <clang/CodeGen/CodeGenAction.h>
 #include <clang/Basic/TargetInfo.h>
 #include <clang/Lex/PreprocessorOptions.h>
@@ -21,7 +26,6 @@
 #include <llvm/ExecutionEngine/GenericValue.h>
 #include <llvm/Target/TargetMachine.h>
 #include <llvm/MC/TargetRegistry.h>
-#include <llvm/Support/Host.h>
 #include <llvm/PassRegistry.h>
 #include <llvm/InitializePasses.h>
 #include <llvm/IR/LegacyPassManager.h>
@@ -114,7 +118,11 @@ static std::unique_ptr<llvm::Module> cpp_to_llvm(const std::string& input_file,
 
     if(debug)
     {
+        #if LLVM_VERSION_MAJOR >= 18
+        compiler_invocation.getCodeGenOpts().setDebugInfo(llvm::codegenoptions::FullDebugInfo);
+        #else
         compiler_invocation.getCodeGenOpts().setDebugInfo(clang::codegenoptions::FullDebugInfo);
+        #endif
     }
 
     // Map code to a MemoryBuffer
@@ -174,7 +182,11 @@ static std::unique_ptr<llvm::Module> cuda_to_llvm(const std::string& input_file,
 
     if(debug)
     {
+        #if LLVM_VERSION_MAJOR >= 18
+        compiler_invocation.getCodeGenOpts().setDebugInfo(llvm::codegenoptions::FullDebugInfo);
+        #else
         compiler_invocation.getCodeGenOpts().setDebugInfo(clang::codegenoptions::FullDebugInfo);
+        #endif
     }
 
     // Map code to a MemoryBuffer
@@ -234,7 +246,11 @@ WP_API int compile_cpp(const char* cpp_src, const char *input_file, const char*
     llvm::raw_fd_ostream output(output_file, error_code, llvm::sys::fs::OF_None);
 
     llvm::legacy::PassManager pass_manager;
+    #if LLVM_VERSION_MAJOR >= 18
+    llvm::CodeGenFileType file_type = llvm::CodeGenFileType::ObjectFile;
+    #else
     llvm::CodeGenFileType file_type = llvm::CGFT_ObjectFile;
+    #endif
     target_machine->addPassesToEmitFile(pass_manager, output, nullptr, file_type);
 
     pass_manager.run(*module);
@@ -287,7 +303,11 @@ WP_API int compile_cuda(const char* cpp_src, const char *input_file, const char*
     llvm::raw_fd_ostream output(output_file, error_code, llvm::sys::fs::OF_None);
 
     llvm::legacy::PassManager pass_manager;
+    #if LLVM_VERSION_MAJOR >= 18
+    llvm::CodeGenFileType file_type = llvm::CodeGenFileType::AssemblyFile;
+    #else
     llvm::CodeGenFileType file_type = llvm::CGFT_AssemblyFile;
+    #endif
     target_machine->addPassesToEmitFile(pass_manager, output, nullptr, file_type);
 
     pass_manager.run(*module);
@@ -352,10 +372,17 @@ WP_API int load_obj(const char* object_file, const char* module_name)
         #endif
 
         const auto flags = llvm::JITSymbolFlags::Exported | llvm::JITSymbolFlags::Absolute;
+        #if LLVM_VERSION_MAJOR >= 18
+        #define SYMBOL(sym) { jit->getExecutionSession().intern(MANGLING_PREFIX #sym), { llvm::orc::ExecutorAddr::fromPtr(&::sym), flags} }
+        #define SYMBOL_T(sym, T) { jit->getExecutionSession().intern(MANGLING_PREFIX #sym), { llvm::orc::ExecutorAddr::fromPtr(static_cast<T>(&::sym)), flags} }
+
+        auto error = dll->define(llvm::orc::absoluteSymbols(llvm::orc::SymbolMap({
+        #else
         #define SYMBOL(sym) { jit->getExecutionSession().intern(MANGLING_PREFIX #sym), { llvm::pointerToJITTargetAddress(&::sym), flags} }
         #define SYMBOL_T(sym, T) { jit->getExecutionSession().intern(MANGLING_PREFIX #sym), { llvm::pointerToJITTargetAddress(static_cast<T>(&::sym)), flags} }
 
         auto error = dll->define(llvm::orc::absoluteSymbols({
+        #endif
             SYMBOL(printf), SYMBOL(puts), SYMBOL(putchar),
             SYMBOL_T(abs, int(*)(int)), SYMBOL(llabs),
             SYMBOL(fmodf), SYMBOL_T(fmod, double(*)(double, double)),
@@ -397,7 +424,11 @@ WP_API int load_obj(const char* object_file, const char* module_name)
         #else
             SYMBOL(sincosf), SYMBOL_T(sincos, void(*)(double,double*,double*)),
         #endif
+        #if LLVM_VERSION_MAJOR >= 18
+        })));
+        #else
         }));
+        #endif
 
         if(error)
         {