[CIR] Add support for nontemporal loads and stores

clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h

@@ -160,13 +160,15 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   }
 
   cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr,
-                         bool isVolatile = false, uint64_t alignment = 0) {
+                         bool isVolatile = false, bool isNontemporal = false,
+                         uint64_t alignment = 0) {
     mlir::IntegerAttr intAttr;
     if (alignment)
       intAttr = mlir::IntegerAttr::get(
           mlir::IntegerType::get(ptr.getContext(), 64), alignment);
 
     return create<cir::LoadOp>(loc, ptr, /*isDeref=*/false, isVolatile,
+                               isNontemporal,
                                /*alignment=*/intAttr,
                                /*mem_order=*/
                                cir::MemOrderAttr{},
@@ -175,7 +177,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
 
   mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr,
                                 uint64_t alignment) {
-    return createLoad(loc, ptr, /*isVolatile=*/false, alignment);
+    return createLoad(loc, ptr, /*isVolatile=*/false, /*isNontemporal=*/false,
+                      alignment);
   }
 
   mlir::Value createNot(mlir::Value value) {
@@ -350,13 +353,14 @@ class CIRBaseBuilderTy : public mlir::OpBuilder {
   }
 
   cir::StoreOp createStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
-                           bool _volatile = false,
+                           bool isVolatile = false, bool isNontemporal = false,
                            ::mlir::IntegerAttr align = {},
                            cir::MemOrderAttr order = {}) {
     if (mlir::cast<cir::PointerType>(dst.getType()).getPointee() !=
         val.getType())
       dst = createPtrBitcast(dst, val.getType());
-    return create<cir::StoreOp>(loc, val, dst, _volatile, align, order,
+    return create<cir::StoreOp>(loc, val, dst, isVolatile, isNontemporal, align,
+                                order,
                                 /*tbaa=*/cir::TBAAAttr{});
   }
 

clang/include/clang/CIR/Dialect/IR/CIROps.td

@@ -587,6 +587,7 @@ def LoadOp : CIR_Op<"load", [
   let arguments = (ins Arg<CIR_PointerType, "the address to load from",
                            [MemRead]>:$addr, UnitAttr:$isDeref,
                        UnitAttr:$is_volatile,
+                       UnitAttr:$is_nontemporal,
                        OptionalAttr<I64Attr>:$alignment,
                        OptionalAttr<MemOrder>:$mem_order,
                        OptionalAttr<CIR_AnyTBAAAttr>:$tbaa
@@ -596,6 +597,7 @@ def LoadOp : CIR_Op<"load", [
   let assemblyFormat = [{
     (`deref` $isDeref^)?
     (`volatile` $is_volatile^)?
+    (`nontemporal` $is_nontemporal^)?
     (`align` `(` $alignment^ `)`)?
     (`atomic` `(` $mem_order^ `)`)?
     $addr `:` qualified(type($addr)) `,` type($result) attr-dict
@@ -656,12 +658,14 @@ def StoreOp : CIR_Op<"store", [
                        Arg<CIR_PointerType, "the address to store the value",
                            [MemWrite]>:$addr,
                        UnitAttr:$is_volatile,
+                       UnitAttr:$is_nontemporal,
                        OptionalAttr<I64Attr>:$alignment,
                        OptionalAttr<MemOrder>:$mem_order,
                        OptionalAttr<CIR_AnyTBAAAttr>:$tbaa);
 
   let assemblyFormat = [{
     (`volatile` $is_volatile^)?
+    (`nontemporal` $is_nontemporal^)?
     (`align` `(` $alignment^ `)`)?
     (`atomic` `(` $mem_order^ `)`)?
     $value `,` $addr attr-dict `:` type($value) `,` qualified(type($addr))

clang/lib/CIR/CodeGen/CIRGenAtomic.cpp

@@ -619,6 +619,7 @@ static void emitAtomicOp(CIRGenFunction &CGF, AtomicExpr *E, Address Dest,
     // FIXME(cir): add scope information.
     assert(!cir::MissingFeatures::syncScopeID());
     builder.createStore(loc, loadVal1, Ptr, E->isVolatile(),
+                        /*isNontemporal=*/false,
                         /*alignment=*/mlir::IntegerAttr{}, orderAttr);
     return;
   }

clang/lib/CIR/CodeGen/CIRGenBuilder.h

@@ -859,32 +859,35 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   }
 
   cir::LoadOp createLoad(mlir::Location loc, Address addr,
-                         bool isVolatile = false) {
+                         bool isVolatile = false, bool isNontemporal = false) {
     auto ptrTy = mlir::dyn_cast<cir::PointerType>(addr.getPointer().getType());
     if (addr.getElementType() != ptrTy.getPointee())
       addr = addr.withPointer(
           createPtrBitcast(addr.getPointer(), addr.getElementType()));
 
     return create<cir::LoadOp>(
         loc, addr.getElementType(), addr.getPointer(), /*isDeref=*/false,
-        /*is_volatile=*/isVolatile, /*alignment=*/mlir::IntegerAttr{},
+        /*is_volatile=*/isVolatile, /*is_nontemporal=*/isNontemporal,
+        /*alignment=*/mlir::IntegerAttr{},
         /*mem_order=*/cir::MemOrderAttr{}, /*tbaa=*/cir::TBAAAttr{});
   }
 
   cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty,
                                 mlir::Value ptr, llvm::MaybeAlign align,
-                                bool isVolatile) {
+                                bool isVolatile, bool isNontemporal) {
     if (ty != mlir::cast<cir::PointerType>(ptr.getType()).getPointee())
       ptr = createPtrBitcast(ptr, ty);
     uint64_t alignment = align ? align->value() : 0;
-    return CIRBaseBuilderTy::createLoad(loc, ptr, isVolatile, alignment);
+    return CIRBaseBuilderTy::createLoad(loc, ptr, isVolatile, isNontemporal,
+                                        alignment);
   }
 
   cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty,
                                 mlir::Value ptr, llvm::MaybeAlign align) {
     // TODO: make sure callsites shouldn't be really passing volatile.
     assert(!cir::MissingFeatures::volatileLoadOrStore());
-    return createAlignedLoad(loc, ty, ptr, align, /*isVolatile=*/false);
+    return createAlignedLoad(loc, ty, ptr, align, /*isVolatile=*/false,
+                             /*isNontemporal=*/false);
   }
 
   cir::LoadOp
@@ -894,11 +897,11 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   }
 
   cir::StoreOp createStore(mlir::Location loc, mlir::Value val, Address dst,
-                           bool _volatile = false,
+                           bool isVolatile = false, bool isNontemporal = false,
                            ::mlir::IntegerAttr align = {},
                            cir::MemOrderAttr order = {}) {
-    return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), _volatile,
-                                         align, order);
+    return CIRBaseBuilderTy::createStore(loc, val, dst.getPointer(), isVolatile,
+                                         isNontemporal, align, order);
   }
 
   cir::StoreOp createFlagStore(mlir::Location loc, bool val, mlir::Value dst) {
@@ -937,16 +940,17 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy {
   cir::StoreOp
   createAlignedStore(mlir::Location loc, mlir::Value val, mlir::Value dst,
                      clang::CharUnits align = clang::CharUnits::One(),
-                     bool _volatile = false, cir::MemOrderAttr order = {}) {
+                     bool isVolatile = false, bool isNontemporal = false,
+                     cir::MemOrderAttr order = {}) {
     llvm::MaybeAlign mayAlign = align.getAsAlign();
     mlir::IntegerAttr alignAttr;
     if (mayAlign) {
       uint64_t alignment = mayAlign ? mayAlign->value() : 0;
       alignAttr = mlir::IntegerAttr::get(
           mlir::IntegerType::get(dst.getContext(), 64), alignment);
     }
-    return CIRBaseBuilderTy::createStore(loc, val, dst, _volatile, alignAttr,
-                                         order);
+    return CIRBaseBuilderTy::createStore(loc, val, dst, isVolatile,
+                                         isNontemporal, alignAttr, order);
   }
 
   // Convert byte offset to sequence of high-level indices suitable for

clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp

@@ -306,6 +306,24 @@ static mlir::Value makeBinaryAtomicValue(
   return emitFromInt(cgf, rmwi->getResult(0), typ, valueType);
 }
 
+static void emitNontemporalStore(CIRGenFunction &cgf, const CallExpr *expr) {
+  mlir::Value val = cgf.emitScalarExpr(expr->getArg(0));
+  Address addr = cgf.emitPointerWithAlignment(expr->getArg(1));
+
+  val = cgf.emitToMemory(val, expr->getArg(0)->getType());
+  LValue lv = cgf.makeAddrLValue(addr, expr->getArg(0)->getType());
+  lv.setNontemporal(true);
+  cgf.emitStoreOfScalar(val, lv, false);
+}
+
+static mlir::Value emitNontemporalLoad(CIRGenFunction &cgf,
+                                       const CallExpr *expr) {
+  Address addr = cgf.emitPointerWithAlignment(expr->getArg(0));
+  LValue lv = cgf.makeAddrLValue(addr, expr->getType());
+  lv.setNontemporal(true);
+  return cgf.emitLoadOfScalar(lv, expr->getExprLoc());
+}
+
 static RValue emitBinaryAtomic(CIRGenFunction &CGF, cir::AtomicFetchKind kind,
                                const CallExpr *E) {
   return RValue::get(makeBinaryAtomicValue(CGF, kind, E));
@@ -1907,10 +1925,13 @@ RValue CIRGenFunction::emitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
 
   case Builtin::BI__sync_synchronize:
     llvm_unreachable("BI__sync_synchronize NYI");
+
   case Builtin::BI__builtin_nontemporal_load:
-    llvm_unreachable("BI__builtin_nontemporal_load NYI");
+    return RValue::get(emitNontemporalLoad(*this, E));
   case Builtin::BI__builtin_nontemporal_store:
-    llvm_unreachable("BI__builtin_nontemporal_store NYI");
+    emitNontemporalStore(*this, E);
+    return RValue::get(nullptr);
+
   case Builtin::BI__c11_atomic_is_lock_free:
     llvm_unreachable("BI__c11_atomic_is_lock_free NYI");
   case Builtin::BI__atomic_is_lock_free:

clang/lib/CIR/CodeGen/CIRGenExpr.cpp

@@ -662,11 +662,8 @@ void CIRGenFunction::emitStoreOfScalar(mlir::Value value, Address addr,
   }
 
   assert(currSrcLoc && "must pass in source location");
-  auto storeOp = builder.createStore(*currSrcLoc, value, addr, isVolatile);
-
-  if (isNontemporal) {
-    llvm_unreachable("NYI");
-  }
+  auto storeOp =
+      builder.createStore(*currSrcLoc, value, addr, isVolatile, isNontemporal);
 
   CGM.decorateOperationWithTBAA(storeOp, tbaaInfo);
 }
@@ -2962,11 +2959,9 @@ mlir::Value CIRGenFunction::emitLoadOfScalar(Address addr, bool isVolatile,
     Ptr = builder.create<cir::CastOp>(loc, ElemPtrTy, cir::CastKind::bitcast,
                                       Ptr);
   }
-  auto loadOp = builder.CIRBaseBuilderTy::createLoad(loc, Ptr, isVolatile);
+  auto loadOp =
+      builder.CIRBaseBuilderTy::createLoad(loc, Ptr, isVolatile, isNontemporal);
 
-  if (isNontemporal) {
-    llvm_unreachable("NYI");
-  }
   CGM.decorateOperationWithTBAA(loadOp, tbaaInfo);
 
   assert(!cir::MissingFeatures::emitScalarRangeCheck() && "NYI");

clang/lib/CIR/CodeGen/CIRGenFunction.h

@@ -2041,7 +2041,7 @@ class CIRGenFunction : public CIRGenTypeCache {
       builder.restoreInsertionPoint(OutermostConditional->getInsertPoint());
       builder.createStore(
           value.getLoc(), value, addr,
-          /*volatile*/ false,
+          /*isVolatile=*/false, /*isNontemporal=*/false,
           mlir::IntegerAttr::get(
               mlir::IntegerType::get(value.getContext(), 64),
               (uint64_t)addr.getAlignment().getAsAlign().value()));

clang/lib/CIR/CodeGen/CIRGenValue.h

@@ -235,6 +235,7 @@ class LValue {
   void setNonGC(bool Value) { NonGC = Value; }
 
   bool isNontemporal() const { return Nontemporal; }
+  void setNontemporal(bool value) { Nontemporal = value; }
 
   bool isObjCWeak() const {
     return Quals.getObjCGCAttr() == clang::Qualifiers::Weak;

clang/lib/CIR/Dialect/IR/CIRMemorySlot.cpp

@@ -149,9 +149,10 @@ DeletionKind cir::CopyOp::removeBlockingUses(
     OpBuilder &builder, Value reachingDefinition,
     const DataLayout &dataLayout) {
   if (loadsFrom(slot))
-    builder.create<cir::StoreOp>(getLoc(), reachingDefinition, getDst(), false,
-                                 mlir::IntegerAttr{}, cir::MemOrderAttr(),
-                                 cir::TBAAAttr{});
+    builder.create<cir::StoreOp>(getLoc(), reachingDefinition, getDst(),
+                                 /*is_volatile=*/false,
+                                 /*is_nontemporal=*/false, mlir::IntegerAttr{},
+                                 cir::MemOrderAttr(), cir::TBAAAttr{});
   return DeletionKind::Delete;
 }
 

clang/lib/CIR/Dialect/Transforms/TargetLowering/ItaniumCXXABI.cpp

@@ -384,6 +384,7 @@ void ItaniumCXXABI::lowerGetMethod(
         op.getLoc(), vtablePtrPtrTy, cir::CastKind::bitcast, loweredObjectPtr);
     mlir::Value vtablePtr = rewriter.create<cir::LoadOp>(
         op.getLoc(), vtablePtrPtr, /*isDeref=*/false, /*isVolatile=*/false,
+        /*isNontemporal=*/false,
         /*alignment=*/mlir::IntegerAttr(), /*mem_order=*/cir::MemOrderAttr(),
         /*tbaa=*/mlir::ArrayAttr());
 
@@ -418,6 +419,7 @@ void ItaniumCXXABI::lowerGetMethod(
             op.getLoc(), vfpPtrTy, cir::CastKind::bitcast, vfpAddr);
         funcPtr = rewriter.create<cir::LoadOp>(
             op.getLoc(), vfpPtr, /*isDeref=*/false, /*isVolatile=*/false,
+            /*isNontemporal=*/false,
             /*alignment=*/mlir::IntegerAttr(),
             /*mem_order=*/cir::MemOrderAttr(),
             /*tbaa=*/mlir::ArrayAttr());

clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp

@@ -1680,7 +1680,7 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
   // TODO: nontemporal, syncscope.
   auto newLoad = rewriter.create<mlir::LLVM::LoadOp>(
       op->getLoc(), llvmTy, adaptor.getAddr(), /* alignment */ alignment,
-      op.getIsVolatile(), /* nontemporal */ false,
+      op.getIsVolatile(), /* nontemporal */ op.getIsNontemporal(),
       /* invariant */ false, /* invariantGroup */ invariant, ordering);
 
   // Convert adapted result to its original type if needed.
@@ -1722,7 +1722,8 @@ mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
   // TODO: nontemporal, syncscope.
   auto storeOp = rewriter.create<mlir::LLVM::StoreOp>(
       op->getLoc(), value, adaptor.getAddr(), alignment, op.getIsVolatile(),
-      /* nontemporal */ false, /* invariantGroup */ invariant, ordering);
+      /* nontemporal */ op.getIsNontemporal(), /* invariantGroup */ invariant,
+      ordering);
   rewriter.replaceOp(op, storeOp);
   if (auto tbaa = op.getTbaaAttr()) {
     storeOp.setTBAATags(lowerCIRTBAAAttr(tbaa, rewriter, lowerMod));

clang/test/CIR/CodeGen/builtin-nontemporal.cpp

@@ -0,0 +1,24 @@
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t.cir
+// RUN: FileCheck --check-prefix=CIR --input-file=%t.cir %s
+// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-llvm %s -o %t.ll
+// RUN: FileCheck --check-prefix=LLVM --input-file=%t.ll %s
+
+int nontemporal_load(const int *ptr) {
+  return __builtin_nontemporal_load(ptr);
+}
+
+// CIR-LABEL: @_Z16nontemporal_loadPKi
+// CIR: %{{.+}} = cir.load nontemporal %{{.+}} : !cir.ptr<!s32i>, !s32i
+
+// LLVM-LABEL: @_Z16nontemporal_loadPKi
+// LLVM: %{{.+}} = load i32, ptr %{{.+}}, align 4, !nontemporal !1
+
+void nontemporal_store(int *ptr, int value) {
+  __builtin_nontemporal_store(value, ptr);
+}
+
+// CIR-LABEL: @_Z17nontemporal_storePii
+// CIR: cir.store nontemporal %{{.+}}, %{{.+}} : !s32i, !cir.ptr<!s32i>
+
+// LLVM-LABEL: @_Z17nontemporal_storePii
+// LLVM: store i32 %{{.+}}, ptr %{{.+}}, align 4, !nontemporal !1