Support external operand arrays in GenTreeMultiOp

src/coreclr/jit/compiler.h

@@ -3167,6 +3167,12 @@ class Compiler
                                                  CorInfoType    simdBaseJitType,
                                                  unsigned       simdSize,
                                                  bool           isSimdAsHWIntrinsic = false);
+    GenTreeHWIntrinsic* gtNewSimdHWIntrinsicNode(var_types              type,
+                                                 IntrinsicNodeBuilder&& nodeBuilder,
+                                                 NamedIntrinsic         hwIntrinsicID,
+                                                 CorInfoType            simdBaseJitType,
+                                                 unsigned               simdSize,
+                                                 bool                   isSimdAsHWIntrinsic = false);
 
     GenTreeHWIntrinsic* gtNewSimdAsHWIntrinsicNode(var_types      type,
                                                    NamedIntrinsic hwIntrinsicID,

src/coreclr/jit/gentree.cpp

@@ -8013,16 +8013,16 @@ GenTree* Compiler::gtCloneExpr(
 #ifdef FEATURE_SIMD
         case GT_SIMD:
             copy = new (this, GT_SIMD)
-                GenTreeSIMD(tree->TypeGet(), getAllocator(CMK_ASTNode), tree->AsSIMD()->GetOperandArray(),
-                            tree->AsSIMD()->GetOperandCount(), tree->AsSIMD()->GetSIMDIntrinsicId(),
-                            tree->AsSIMD()->GetSimdBaseJitType(), tree->AsSIMD()->GetSimdSize());
+                GenTreeSIMD(tree->TypeGet(), IntrinsicNodeBuilder(getAllocator(CMK_ASTNode), tree->AsSIMD()),
+                            tree->AsSIMD()->GetSIMDIntrinsicId(), tree->AsSIMD()->GetSimdBaseJitType(),
+                            tree->AsSIMD()->GetSimdSize());
             goto CLONE_MULTIOP_OPERANDS;
 #endif
 #ifdef FEATURE_HW_INTRINSICS
         case GT_HWINTRINSIC:
             copy = new (this, GT_HWINTRINSIC)
-                GenTreeHWIntrinsic(tree->TypeGet(), getAllocator(CMK_ASTNode), tree->AsHWIntrinsic()->GetOperandArray(),
-                                   tree->AsHWIntrinsic()->GetOperandCount(), tree->AsHWIntrinsic()->GetHWIntrinsicId(),
+                GenTreeHWIntrinsic(tree->TypeGet(), IntrinsicNodeBuilder(getAllocator(CMK_ASTNode), tree->AsMultiOp()),
+                                   tree->AsHWIntrinsic()->GetHWIntrinsicId(),
                                    tree->AsHWIntrinsic()->GetSimdBaseJitType(), tree->AsHWIntrinsic()->GetSimdSize(),
                                    tree->AsHWIntrinsic()->IsSimdAsHWIntrinsic());
             copy->AsHWIntrinsic()->SetAuxiliaryJitType(tree->AsHWIntrinsic()->GetAuxiliaryJitType());
@@ -18708,14 +18708,8 @@ void GenTreeMultiOp::ResetOperandArray(size_t    newOperandCount,
     return true;
 }
 
-void GenTreeMultiOp::InitializeOperands(CompAllocator allocator,
-                                        GenTree**     operands,
-                                        size_t        operandCount,
-                                        GenTree**     inlineOperands,
-                                        size_t        inlineOperandCount)
+void GenTreeMultiOp::InitializeOperands(GenTree** operands, size_t operandCount)
 {
-    m_operands = (operandCount <= inlineOperandCount) ? inlineOperands : allocator.allocate<GenTree*>(operandCount);
-
     for (size_t i = 0; i < operandCount; i++)
     {
         m_operands[i] = operands[i];
@@ -18935,13 +18929,31 @@ GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types      type,
                                                        unsigned       simdSize,
                                                        bool           isSimdAsHWIntrinsic)
 {
+    IntrinsicNodeBuilder nodeBuilder(getAllocator(CMK_ASTNode), operandCount);
     for (size_t i = 0; i < operandCount; i++)
     {
+        nodeBuilder.AddOperand(i, operands[i]);
         SetOpLclRelatedToSIMDIntrinsic(operands[i]);
     }
 
-    return new (this, GT_HWINTRINSIC) GenTreeHWIntrinsic(type, getAllocator(CMK_ASTNode), operands, operandCount,
-                                                         hwIntrinsicID, simdBaseJitType, simdSize, isSimdAsHWIntrinsic);
+    return new (this, GT_HWINTRINSIC) GenTreeHWIntrinsic(type, std::move(nodeBuilder), hwIntrinsicID, simdBaseJitType,
+                                                         simdSize, isSimdAsHWIntrinsic);
+}
+
+GenTreeHWIntrinsic* Compiler::gtNewSimdHWIntrinsicNode(var_types              type,
+                                                       IntrinsicNodeBuilder&& nodeBuilder,
+                                                       NamedIntrinsic         hwIntrinsicID,
+                                                       CorInfoType            simdBaseJitType,
+                                                       unsigned               simdSize,
+                                                       bool                   isSimdAsHWIntrinsic)
+{
+    for (size_t i = 0; i < nodeBuilder.GetOperandCount(); i++)
+    {
+        SetOpLclRelatedToSIMDIntrinsic(nodeBuilder.GetOperand(i));
+    }
+
+    return new (this, GT_HWINTRINSIC) GenTreeHWIntrinsic(type, std::move(nodeBuilder), hwIntrinsicID, simdBaseJitType,
+                                                         simdSize, isSimdAsHWIntrinsic);
 }
 
 GenTree* Compiler::gtNewSimdAbsNode(

src/coreclr/jit/gentree.h

@@ -5131,21 +5131,25 @@ struct GenTreeMultiOp : public GenTree
     {
         const size_t OperandCount = sizeof...(Operands);
 
+        m_operands = (OperandCount <= InlineOperandCount) ? inlineOperands : allocator.allocate<GenTree*>(OperandCount);
+
         // "OperandCount + 1" so that it works well when OperandCount is 0.
         GenTree* operandsArray[OperandCount + 1]{operands...};
-        InitializeOperands(allocator, operandsArray, OperandCount, inlineOperands, InlineOperandCount);
+        InitializeOperands(operandsArray, OperandCount);
     }
 
+    // Note that this constructor takes the owndership of the "operands" array.
     template <unsigned InlineOperandCount>
-    GenTreeMultiOp(genTreeOps    oper,
-                   var_types     type,
-                   CompAllocator allocator,
-                   GenTree**     operands,
-                   size_t        operandCount,
+    GenTreeMultiOp(genTreeOps oper,
+                   var_types  type,
+                   GenTree**  operands,
+                   size_t     operandCount,
                    GenTree* (&inlineOperands)[InlineOperandCount] DEBUGARG(bool largeNode))
         : GenTree(oper, type DEBUGARG(largeNode))
     {
-        InitializeOperands(allocator, operands, operandCount, inlineOperands, InlineOperandCount);
+        m_operands = (operandCount <= InlineOperandCount) ? inlineOperands : operands;
+
+        InitializeOperands(operands, operandCount);
     }
 
 public:
@@ -5203,11 +5207,7 @@ struct GenTreeMultiOp : public GenTree
     static bool OperandsAreEqual(GenTreeMultiOp* op1, GenTreeMultiOp* op2);
 
 private:
-    void InitializeOperands(CompAllocator allocator,
-                            GenTree**     operands,
-                            size_t        operandCount,
-                            GenTree**     inlineOperands,
-                            size_t        inlineOperandCount);
+    void InitializeOperands(GenTree** operands, size_t operandCount);
 
     void SetOperandCount(size_t newOperandCount)
     {
@@ -5216,6 +5216,72 @@ struct GenTreeMultiOp : public GenTree
     }
 };
 
+// Helper class used to implement the constructor of GenTreeJitIntrinsic which
+// transfers the ownership of the passed-in array to the underlying MultiOp node.
+class IntrinsicNodeBuilder final
+{
+    friend struct GenTreeJitIntrinsic;
+
+    GenTree** m_operands;
+    size_t    m_operandCount;
+    GenTree*  m_inlineOperands[2];
+
+public:
+    IntrinsicNodeBuilder(CompAllocator allocator, size_t operandCount) : m_operandCount(operandCount)
+    {
+        m_operands = (operandCount <= ArrLen(m_inlineOperands)) ? m_inlineOperands
+                                                                : allocator.allocate<GenTree*>(operandCount);
+#ifdef DEBUG
+        for (size_t i = 0; i < operandCount; i++)
+        {
+            m_operands[i] = nullptr;
+        }
+#endif // DEBUG
+    }
+
+    IntrinsicNodeBuilder(CompAllocator allocator, GenTreeMultiOp* source) : m_operandCount(source->GetOperandCount())
+    {
+        m_operands = (m_operandCount <= ArrLen(m_inlineOperands)) ? m_inlineOperands
+                                                                  : allocator.allocate<GenTree*>(m_operandCount);
+        for (size_t i = 0; i < m_operandCount; i++)
+        {
+            m_operands[i] = source->Op(i + 1);
+        }
+    }
+
+    void AddOperand(size_t index, GenTree* operand)
+    {
+        assert(index < m_operandCount);
+        assert(m_operands[index] == nullptr);
+        m_operands[index] = operand;
+    }
+
+    GenTree* GetOperand(size_t index) const
+    {
+        assert(index < m_operandCount);
+        assert(m_operands[index] != nullptr);
+        return m_operands[index];
+    }
+
+    size_t GetOperandCount() const
+    {
+        return m_operandCount;
+    }
+
+private:
+    GenTree** GetBuiltOperands()
+    {
+#ifdef DEBUG
+        for (size_t i = 0; i < m_operandCount; i++)
+        {
+            assert(m_operands[i] != nullptr);
+        }
+#endif // DEBUG
+
+        return m_operands;
+    }
+};
+
 struct GenTreeJitIntrinsic : public GenTreeMultiOp
 {
 protected:
@@ -5321,14 +5387,17 @@ struct GenTreeJitIntrinsic : public GenTreeMultiOp
 #endif
 
 protected:
-    GenTreeJitIntrinsic(genTreeOps    oper,
-                        var_types     type,
-                        CompAllocator allocator,
-                        GenTree**     operands,
-                        size_t        operandCount,
-                        CorInfoType   simdBaseJitType,
-                        unsigned      simdSize)
-        : GenTreeMultiOp(oper, type, allocator, operands, operandCount, gtInlineOperands DEBUGARG(false))
+    GenTreeJitIntrinsic(genTreeOps             oper,
+                        var_types              type,
+                        IntrinsicNodeBuilder&& nodeBuilder,
+                        CorInfoType            simdBaseJitType,
+                        unsigned               simdSize)
+        : GenTreeMultiOp(oper,
+                         type,
+                         nodeBuilder.GetBuiltOperands(),
+                         nodeBuilder.GetOperandCount(),
+                         gtInlineOperands
+                         DEBUGARG(false))
         , gtLayoutNum(0)
         , gtAuxiliaryJitType(CORINFO_TYPE_UNDEF)
         , gtOtherReg(REG_NA)
@@ -5352,14 +5421,12 @@ struct GenTreeJitIntrinsic : public GenTreeMultiOp
 /* gtSIMD   -- SIMD intrinsic   (possibly-binary op [NULL op2 is allowed] with additional fields) */
 struct GenTreeSIMD : public GenTreeJitIntrinsic
 {
-    GenTreeSIMD(var_types       type,
-                CompAllocator   allocator,
-                GenTree**       operands,
-                size_t          operandCount,
-                SIMDIntrinsicID simdIntrinsicID,
-                CorInfoType     simdBaseJitType,
-                unsigned        simdSize)
-        : GenTreeJitIntrinsic(GT_SIMD, type, allocator, operands, operandCount, simdBaseJitType, simdSize)
+    GenTreeSIMD(var_types              type,
+                IntrinsicNodeBuilder&& nodeBuilder,
+                SIMDIntrinsicID        simdIntrinsicID,
+                CorInfoType            simdBaseJitType,
+                unsigned               simdSize)
+        : GenTreeJitIntrinsic(GT_SIMD, type, std::move(nodeBuilder), simdBaseJitType, simdSize)
     {
         gtSIMDIntrinsicID = simdIntrinsicID;
     }
@@ -5408,15 +5475,13 @@ struct GenTreeSIMD : public GenTreeJitIntrinsic
 #ifdef FEATURE_HW_INTRINSICS
 struct GenTreeHWIntrinsic : public GenTreeJitIntrinsic
 {
-    GenTreeHWIntrinsic(var_types      type,
-                       CompAllocator  allocator,
-                       GenTree**      operands,
-                       size_t         operandCount,
-                       NamedIntrinsic hwIntrinsicID,
-                       CorInfoType    simdBaseJitType,
-                       unsigned       simdSize,
-                       bool           isSimdAsHWIntrinsic)
-        : GenTreeJitIntrinsic(GT_HWINTRINSIC, type, allocator, operands, operandCount, simdBaseJitType, simdSize)
+    GenTreeHWIntrinsic(var_types              type,
+                       IntrinsicNodeBuilder&& nodeBuilder,
+                       NamedIntrinsic         hwIntrinsicID,
+                       CorInfoType            simdBaseJitType,
+                       unsigned               simdSize,
+                       bool                   isSimdAsHWIntrinsic)
+        : GenTreeJitIntrinsic(GT_HWINTRINSIC, type, std::move(nodeBuilder), simdBaseJitType, simdSize)
     {
         SetHWIntrinsicId(hwIntrinsicID);
 

src/coreclr/jit/hwintrinsicarm64.cpp

@@ -482,15 +482,14 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic        intrinsic,
             // We shouldn't handle this as an intrinsic if the
             // respective ISAs have been disabled by the user.
 
-            GenTree* args[32];
-            assert(sig->numArgs <= ArrLen(args));
+            IntrinsicNodeBuilder nodeBuilder(getAllocator(CMK_ASTNode), sig->numArgs);
 
             for (int i = sig->numArgs - 1; i >= 0; i--)
             {
-                args[i] = impPopStack().val;
+                nodeBuilder.AddOperand(i, impPopStack().val);
             }
 
-            retNode = gtNewSimdHWIntrinsicNode(retType, args, sig->numArgs, intrinsic, simdBaseJitType, simdSize);
+            retNode = gtNewSimdHWIntrinsicNode(retType, std::move(nodeBuilder), intrinsic, simdBaseJitType, simdSize);
             break;
         }
 

src/coreclr/jit/hwintrinsicxarch.cpp

@@ -858,15 +858,14 @@ GenTree* Compiler::impBaseIntrinsic(NamedIntrinsic        intrinsic,
             }
 #endif // TARGET_X86
 
-            GenTree* args[32];
-            assert(sig->numArgs <= ArrLen(args));
+            IntrinsicNodeBuilder nodeBuilder(getAllocator(CMK_ASTNode), sig->numArgs);
 
             for (int i = sig->numArgs - 1; i >= 0; i--)
             {
-                args[i] = impPopStack().val;
+                nodeBuilder.AddOperand(i, impPopStack().val);
             }
 
-            retNode = gtNewSimdHWIntrinsicNode(retType, args, sig->numArgs, intrinsic, simdBaseJitType, simdSize);
+            retNode = gtNewSimdHWIntrinsicNode(retType, std::move(nodeBuilder), intrinsic, simdBaseJitType, simdSize);
             break;
         }
 

src/coreclr/jit/simd.cpp

@@ -1985,13 +1985,13 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
             // SIMDIntrinsicInitN
             //    op2 - list of initializer values stitched into a list
             //    op1 - byref of vector
-            GenTree* args[4]{};
-            bool     initFromFirstArgIndir = false;
+            IntrinsicNodeBuilder nodeBuilder(getAllocator(CMK_ASTNode), argCount - 1);
+            bool                 initFromFirstArgIndir = false;
 
             if (simdIntrinsicID == SIMDIntrinsicInit)
             {
-                op2     = impSIMDPopStack(simdBaseType);
-                args[0] = op2;
+                op2 = impSIMDPopStack(simdBaseType);
+                nodeBuilder.AddOperand(0, op2);
             }
             else
             {
@@ -2001,7 +2001,6 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
                 unsigned initCount    = argCount - 1;
                 unsigned elementCount = getSIMDVectorLength(size, simdBaseType);
                 noway_assert(initCount == elementCount);
-                assert(initCount <= ArrLen(args));
 
                 // Build an array with the N values.
                 // We must maintain left-to-right order of the args, but we will pop
@@ -2026,7 +2025,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
                     }
 
                     assert(genActualType(arg) == genActualType(simdBaseType));
-                    args[initCount - i - 1] = arg;
+                    nodeBuilder.AddOperand(initCount - i - 1, arg);
                 }
 
                 if (areArgsContiguous && simdBaseType == TYP_FLOAT)
@@ -2035,7 +2034,7 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
                     // we intialize the vector from first argument address, only when
                     // the simdBaseType is TYP_FLOAT and the arguments are located contiguously in memory
                     initFromFirstArgIndir = true;
-                    GenTree*  op2Address  = createAddressNodeForSIMDInit(args[0], size);
+                    GenTree*  op2Address  = createAddressNodeForSIMDInit(nodeBuilder.GetOperand(0), size);
                     var_types simdType    = getSIMDTypeForSize(size);
                     op2                   = gtNewOperNode(GT_IND, simdType, op2Address);
                 }
@@ -2111,8 +2110,8 @@ GenTree* Compiler::impSIMDIntrinsic(OPCODE                opcode,
                 }
                 else
                 {
-                    simdTree = new (this, GT_SIMD) GenTreeSIMD(simdType, getAllocator(CMK_ASTNode), args, argCount - 1,
-                                                               simdIntrinsicID, simdBaseJitType, size);
+                    simdTree = new (this, GT_SIMD) GenTreeSIMD(simdType, std::move(nodeBuilder), simdIntrinsicID,
+                                                               simdBaseJitType, size);
                 }
             }