Merge pull request #5465 from ChipKerchner/addRVVVectorizedFP16Packing

Add vectorized packing for FP16 and BF16 for RISC-V.  Reactivate vector packing for FP64 transposed

Author: martin-frbg

kernel/riscv64/KERNEL.RISCV64_ZVL128B

@@ -120,13 +120,21 @@ endif
 
 DGEMMKERNEL    =  dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N)_zvl128b.c
 DGEMMONCOPY    =  ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c
+ifneq ($(filter $(DGEMM_UNROLL_N),4 8 16),)
+DGEMMOTCOPY    =  gemm_tcopy_$(DGEMM_UNROLL_N)_rvv.c
+else
 DGEMMOTCOPY    =  ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c
+endif
 DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)
 DGEMMOTCOPYOBJ =  dgemm_otcopy$(TSUFFIX).$(SUFFIX)
 
 ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N))
 DGEMMINCOPY    =  ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c
+ifneq ($(filter $(DGEMM_UNROLL_M),4 8 16),)
+DGEMMITCOPY    =  gemm_tcopy_$(DGEMM_UNROLL_M)_rvv.c
+else
 DGEMMITCOPY    =  ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c
+endif
 DGEMMINCOPYOBJ =  dgemm_incopy$(TSUFFIX).$(SUFFIX)
 DGEMMITCOPYOBJ =  dgemm_itcopy$(TSUFFIX).$(SUFFIX)
 endif
@@ -258,8 +266,13 @@ endif
 
 ifeq ($(BUILD_HFLOAT16), 1)
 SHGEMMKERNEL    =  shgemm_kernel_$(SHGEMM_UNROLL_M)x$(SHGEMM_UNROLL_N)_zvl128b.c
+ifneq ($(filter $(SHGEMM_UNROLL_N),8 16),)
+SHGEMMONCOPY    =  gemm_ncopy_$(SHGEMM_UNROLL_N)fp_rvv.c
+SHGEMMOTCOPY    =  gemm_tcopy_$(SHGEMM_UNROLL_N)fp_rvv.c
+else
 SHGEMMONCOPY    =  ../generic/gemm_ncopy_$(SHGEMM_UNROLL_N).c
 SHGEMMOTCOPY    =  ../generic/gemm_tcopy_$(SHGEMM_UNROLL_N).c
+endif
 SHGEMMONCOPYOBJ =  shgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SHGEMMOTCOPYOBJ =  shgemm_otcopy$(TSUFFIX).$(SUFFIX)
 ifndef SHGEMM_BETA
@@ -269,8 +282,13 @@ endif
 
 ifeq ($(BUILD_BFLOAT16), 1)
 SBGEMMKERNEL    =  sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_zvl128b.c
+ifneq ($(filter $(SBGEMM_UNROLL_N),8 16),)
+SBGEMMONCOPY    =  gemm_ncopy_$(SBGEMM_UNROLL_N)fp_rvv.c
+SBGEMMOTCOPY    =  gemm_tcopy_$(SBGEMM_UNROLL_N)fp_rvv.c
+else
 SBGEMMONCOPY    =  ../generic/gemm_ncopy_$(SBGEMM_UNROLL_N).c
 SBGEMMOTCOPY    =  ../generic/gemm_tcopy_$(SBGEMM_UNROLL_N).c
+endif
 SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)
 ifndef SBGEMM_BETA

kernel/riscv64/KERNEL.RISCV64_ZVL256B

@@ -119,12 +119,20 @@ endif
 
 DGEMMKERNEL    =  dgemm_kernel_$(DGEMM_UNROLL_M)x$(DGEMM_UNROLL_N)_zvl256b.c
 DGEMMONCOPY    =  ../generic/gemm_ncopy_$(DGEMM_UNROLL_N).c
+ifneq ($(filter $(DGEMM_UNROLL_N),4 8 16),)
+DGEMMOTCOPY    =  gemm_tcopy_$(DGEMM_UNROLL_N)_rvv.c
+else
 DGEMMOTCOPY    =  ../generic/gemm_tcopy_$(DGEMM_UNROLL_N).c
+endif
 DGEMMONCOPYOBJ =  dgemm_oncopy$(TSUFFIX).$(SUFFIX)
 DGEMMOTCOPYOBJ =  dgemm_otcopy$(TSUFFIX).$(SUFFIX)
 ifneq ($(DGEMM_UNROLL_M), $(DGEMM_UNROLL_N))
 DGEMMINCOPY    =  ../generic/gemm_ncopy_$(DGEMM_UNROLL_M).c
+ifneq ($(filter $(DGEMM_UNROLL_M),4 8 16),)
+DGEMMITCOPY    =  gemm_tcopy_$(DGEMM_UNROLL_M)_rvv.c
+else
 DGEMMITCOPY    =  ../generic/gemm_tcopy_$(DGEMM_UNROLL_M).c
+endif
 DGEMMINCOPYOBJ =  dgemm_incopy$(TSUFFIX).$(SUFFIX)
 DGEMMITCOPYOBJ =  dgemm_itcopy$(TSUFFIX).$(SUFFIX)
 endif
@@ -226,13 +234,23 @@ SOMATCOPY_CT = omatcopy_ct_rvv.c
 ifeq ($(BUILD_HFLOAT16), 1)
 SHGEMMKERNEL    =  shgemm_kernel_$(SHGEMM_UNROLL_M)x$(SHGEMM_UNROLL_N)_zvl256b.c
 ifneq ($(SHGEMM_UNROLL_M), $(SHGEMM_UNROLL_N))
+ifneq ($(filter $(SHGEMM_UNROLL_M),8 16),)
+SHGEMMINCOPY    =  gemm_ncopy_$(SHGEMM_UNROLL_M)fp_rvv.c
+SHGEMMITCOPY    =  gemm_tcopy_$(SHGEMM_UNROLL_M)fp_rvv.c
+else
 SHGEMMINCOPY    =  ../generic/gemm_ncopy_$(SHGEMM_UNROLL_M).c
 SHGEMMITCOPY    =  ../generic/gemm_tcopy_$(SHGEMM_UNROLL_M).c
+endif
 SHGEMMINCOPYOBJ =  shgemm_incopy$(TSUFFIX).$(SUFFIX)
 SHGEMMITCOPYOBJ =  shgemm_itcopy$(TSUFFIX).$(SUFFIX)
 endif
+ifneq ($(filter $(SHGEMM_UNROLL_N),8 16),)
+SHGEMMONCOPY    =  gemm_ncopy_$(SHGEMM_UNROLL_N)fp_rvv.c
+SHGEMMOTCOPY    =  gemm_tcopy_$(SHGEMM_UNROLL_N)fp_rvv.c
+else
 SHGEMMONCOPY    =  ../generic/gemm_ncopy_$(SHGEMM_UNROLL_N).c
 SHGEMMOTCOPY    =  ../generic/gemm_tcopy_$(SHGEMM_UNROLL_N).c
+endif
 SHGEMMONCOPYOBJ =  shgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SHGEMMOTCOPYOBJ =  shgemm_otcopy$(TSUFFIX).$(SUFFIX)
 ifndef SHGEMM_BETA
@@ -243,13 +261,23 @@ endif
 ifeq ($(BUILD_BFLOAT16), 1)
 SBGEMMKERNEL    =  sbgemm_kernel_$(SBGEMM_UNROLL_M)x$(SBGEMM_UNROLL_N)_zvl256b.c
 ifneq ($(SBGEMM_UNROLL_M), $(SBGEMM_UNROLL_N))
+ifneq ($(filter $(SBGEMM_UNROLL_M),8 16),)
+SBGEMMINCOPY    =  gemm_ncopy_$(SBGEMM_UNROLL_M)fp_rvv.c
+SBGEMMITCOPY    =  gemm_tcopy_$(SBGEMM_UNROLL_M)fp_rvv.c
+else
 SBGEMMINCOPY    =  ../generic/gemm_ncopy_$(SBGEMM_UNROLL_M).c
 SBGEMMITCOPY    =  ../generic/gemm_tcopy_$(SBGEMM_UNROLL_M).c
+endif
 SBGEMMINCOPYOBJ =  sbgemm_incopy$(TSUFFIX).$(SUFFIX)
 SBGEMMITCOPYOBJ =  sbgemm_itcopy$(TSUFFIX).$(SUFFIX)
 endif
+ifneq ($(filter $(SBGEMM_UNROLL_N),8 16),)
+SBGEMMONCOPY    =  gemm_ncopy_$(SBGEMM_UNROLL_N)fp_rvv.c
+SBGEMMOTCOPY    =  gemm_tcopy_$(SBGEMM_UNROLL_N)fp_rvv.c
+else
 SBGEMMONCOPY    =  ../generic/gemm_ncopy_$(SBGEMM_UNROLL_N).c
 SBGEMMOTCOPY    =  ../generic/gemm_tcopy_$(SBGEMM_UNROLL_N).c
+endif
 SBGEMMONCOPYOBJ =  sbgemm_oncopy$(TSUFFIX).$(SUFFIX)
 SBGEMMOTCOPYOBJ =  sbgemm_otcopy$(TSUFFIX).$(SUFFIX)
 ifndef SBGEMM_BETA