Skip to content

Commit aa81328

Browse files
TamarChristinaArmTamarChristinaArm
authored
Add VectorTableList and TableVectorExtension intrinsics (#35600)
1 parent 0764d8c commit aa81328

25 files changed

+4776
-20
lines changed

‎src/coreclr/src/jit/emit.h‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1222,9 +1222,11 @@ class emitter
12221222

12231223
#define PERFSCORE_THROUGHPUT_ILLEGAL -1024.0f
12241224

1225-
#define PERFSCORE_THROUGHPUT_4X 0.25f // Fastest - Quad issue
1226-
#define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Faster - Three issue
1227-
#define PERFSCORE_THROUGHPUT_2X 0.5f // Faster - Dual issue
1225+
#define PERFSCORE_THROUGHPUT_6X (1.0f / 6.0f) // Hextuple issue
1226+
#define PERFSCORE_THROUGHPUT_5X 0.20f // Pentuple issue
1227+
#define PERFSCORE_THROUGHPUT_4X 0.25f // Quad issue
1228+
#define PERFSCORE_THROUGHPUT_3X (1.0f / 3.0f) // Three issue
1229+
#define PERFSCORE_THROUGHPUT_2X 0.5f // Dual issue
12281230

12291231
#define PERFSCORE_THROUGHPUT_1C 1.0f // Single Issue
12301232

‎src/coreclr/src/jit/emitarm64.cpp‎

Lines changed: 102 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -817,6 +817,22 @@ void emitter::emitInsSanityCheck(instrDesc* id)
817817
break;
818818

819819
case IF_DV_3C: // DV_3C .Q.........mmmmm ......nnnnnddddd Vd Vn Vm (vector)
820+
switch (id->idIns())
821+
{
822+
case INS_tbl:
823+
case INS_tbl_2regs:
824+
case INS_tbl_3regs:
825+
case INS_tbl_4regs:
826+
case INS_tbx:
827+
case INS_tbx_2regs:
828+
case INS_tbx_3regs:
829+
case INS_tbx_4regs:
830+
elemsize = optGetElemsize(id->idInsOpt());
831+
assert(elemsize == EA_1BYTE);
832+
break;
833+
default:
834+
break;
835+
}
820836
assert(isValidVectorDatasize(id->idOpSize()));
821837
assert(isValidArrangement(id->idOpSize(), id->idInsOpt()));
822838
assert(isVectorRegister(id->idReg1()));
@@ -3213,15 +3229,16 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt)
32133229
}
32143230

32153231
//------------------------------------------------------------------------
3216-
// insGetLoadStoreRegisterListSize: Returns a size of the register list a given instruction operates on.
3232+
// insGetRegisterListSize: Returns a size of the register list a given instruction operates on.
32173233
//
32183234
// Arguments:
3219-
// ins - A Load/Store Vector instruction (e.g. ld1 (2 registers), ld1r, st1).
3235+
// ins - An instruction which uses a register list
3236+
// (e.g. ld1 (2 registers), ld1r, st1, tbl, tbx).
32203237
//
32213238
// Return value:
32223239
// A number of consecutive SIMD and floating-point registers the instruction loads to/store from.
32233240
//
3224-
/*static*/ unsigned emitter::insGetLoadStoreRegisterListSize(instruction ins)
3241+
/*static*/ unsigned emitter::insGetRegisterListSize(instruction ins)
32253242
{
32263243
unsigned registerListSize = 0;
32273244

@@ -3230,6 +3247,8 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt)
32303247
case INS_ld1:
32313248
case INS_ld1r:
32323249
case INS_st1:
3250+
case INS_tbl:
3251+
case INS_tbx:
32333252
registerListSize = 1;
32343253
break;
32353254

@@ -3238,6 +3257,8 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt)
32383257
case INS_ld2r:
32393258
case INS_st1_2regs:
32403259
case INS_st2:
3260+
case INS_tbl_2regs:
3261+
case INS_tbx_2regs:
32413262
registerListSize = 2;
32423263
break;
32433264

@@ -3246,6 +3267,8 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt)
32463267
case INS_ld3r:
32473268
case INS_st1_3regs:
32483269
case INS_st3:
3270+
case INS_tbl_3regs:
3271+
case INS_tbx_3regs:
32493272
registerListSize = 3;
32503273
break;
32513274

@@ -3254,6 +3277,8 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt)
32543277
case INS_ld4r:
32553278
case INS_st1_4regs:
32563279
case INS_st4:
3280+
case INS_tbl_4regs:
3281+
case INS_tbx_4regs:
32573282
registerListSize = 4;
32583283
break;
32593284

@@ -5192,7 +5217,7 @@ void emitter::emitIns_R_R_I(
51925217

51935218
if (insOptsAnyArrangement(opt))
51945219
{
5195-
registerListSize = insGetLoadStoreRegisterListSize(ins);
5220+
registerListSize = insGetRegisterListSize(ins);
51965221
assert(isValidVectorDatasize(size));
51975222
assert(isValidArrangement(size, opt));
51985223
assert((size * registerListSize) == imm);
@@ -5226,7 +5251,7 @@ void emitter::emitIns_R_R_I(
52265251
assert(isValidArrangement(size, opt));
52275252

52285253
elemsize = optGetElemsize(opt);
5229-
registerListSize = insGetLoadStoreRegisterListSize(ins);
5254+
registerListSize = insGetRegisterListSize(ins);
52305255
assert((elemsize * registerListSize) == imm);
52315256

52325257
// Load single structure and replicate post-indexed by an immediate
@@ -5676,6 +5701,14 @@ void emitter::emitIns_R_R_R(
56765701
case INS_eor:
56775702
case INS_orr:
56785703
case INS_orn:
5704+
case INS_tbl:
5705+
case INS_tbl_2regs:
5706+
case INS_tbl_3regs:
5707+
case INS_tbl_4regs:
5708+
case INS_tbx:
5709+
case INS_tbx_2regs:
5710+
case INS_tbx_3regs:
5711+
case INS_tbx_4regs:
56795712
if (isVectorRegister(reg1))
56805713
{
56815714
assert(isValidVectorDatasize(size));
@@ -6612,7 +6645,7 @@ void emitter::emitIns_R_R_I_I(
66126645
assert(isValidVectorElemsize(elemsize));
66136646
assert(isValidVectorIndex(EA_16BYTE, elemsize, imm1));
66146647

6615-
registerListSize = insGetLoadStoreRegisterListSize(ins);
6648+
registerListSize = insGetRegisterListSize(ins);
66166649
assert((elemsize * registerListSize) == (unsigned)imm2);
66176650
assert(insOptsPostIndex(opt));
66186651

@@ -11884,7 +11917,7 @@ void emitter::emitDispIns(
1188411917

1188511918
case IF_LS_2D: // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn
1188611919
case IF_LS_2E: // LS_2E .Q.............. ....ssnnnnnttttt Vt Rn
11887-
registerListSize = insGetLoadStoreRegisterListSize(id->idIns());
11920+
registerListSize = insGetRegisterListSize(id->idIns());
1188811921
emitDispVectorRegList(id->idReg1(), registerListSize, id->idInsOpt(), true);
1188911922

1189011923
if (fmt == IF_LS_2D)
@@ -11903,7 +11936,7 @@ void emitter::emitDispIns(
1190311936

1190411937
case IF_LS_2F: // LS_2F .Q.............. xx.Sssnnnnnttttt Vt[] Rn
1190511938
case IF_LS_2G: // LS_2G .Q.............. xx.Sssnnnnnttttt Vt[] Rn
11906-
registerListSize = insGetLoadStoreRegisterListSize(id->idIns());
11939+
registerListSize = insGetRegisterListSize(id->idIns());
1190711940
elemsize = id->idOpSize();
1190811941
emitDispVectorElemList(id->idReg1(), registerListSize, elemsize, id->idSmallCns(), true);
1190911942

@@ -11967,7 +12000,7 @@ void emitter::emitDispIns(
1196712000

1196812001
case IF_LS_3F: // LS_3F .Q.........mmmmm ....ssnnnnnttttt Vt Rn Rm
1196912002
case IF_LS_3G: // LS_3G .Q.........mmmmm ...Sssnnnnnttttt Vt[] Rn Rm
11970-
registerListSize = insGetLoadStoreRegisterListSize(id->idIns());
12003+
registerListSize = insGetRegisterListSize(id->idIns());
1197112004

1197212005
if (fmt == IF_LS_3F)
1197312006
{
@@ -12468,9 +12501,25 @@ void emitter::emitDispIns(
1246812501

1246912502
case IF_DV_3C: // DV_3C .Q.........mmmmm ......nnnnnddddd Vd Vn Vm (vector)
1247012503
emitDispVectorReg(id->idReg1(), id->idInsOpt(), true);
12471-
if (ins != INS_mov)
12504+
switch (ins)
1247212505
{
12473-
emitDispVectorReg(id->idReg2(), id->idInsOpt(), true);
12506+
case INS_tbl:
12507+
case INS_tbl_2regs:
12508+
case INS_tbl_3regs:
12509+
case INS_tbl_4regs:
12510+
case INS_tbx:
12511+
case INS_tbx_2regs:
12512+
case INS_tbx_3regs:
12513+
case INS_tbx_4regs:
12514+
registerListSize = insGetRegisterListSize(ins);
12515+
elemsize = id->idOpSize();
12516+
emitDispVectorRegList(id->idReg2(), registerListSize, id->idInsOpt(), true);
12517+
break;
12518+
case INS_mov:
12519+
break;
12520+
default:
12521+
emitDispVectorReg(id->idReg2(), id->idInsOpt(), true);
12522+
break;
1247412523
}
1247512524
emitDispVectorReg(id->idReg3(), id->idInsOpt(), false);
1247612525
break;
@@ -14145,9 +14194,48 @@ emitter::insExecutionCharacteristics emitter::getInsExecutionCharacteristics(ins
1414514194
}
1414614195
break;
1414714196

14148-
case IF_DV_3C: // mov,and, bic, eor, mov,mvn, orn, bsl, bit, bif (vector)
14149-
result.insThroughput = PERFSCORE_THROUGHPUT_2X;
14150-
result.insLatency = PERFSCORE_LATENCY_1C;
14197+
case IF_DV_3C: // mov,and, bic, eor, mov,mvn, orn, bsl, bit, bif,
14198+
// tbl, tbx (vector)
14199+
switch (ins)
14200+
{
14201+
case INS_tbl:
14202+
result.insThroughput = PERFSCORE_THROUGHPUT_2X;
14203+
result.insLatency = PERFSCORE_LATENCY_1C;
14204+
break;
14205+
case INS_tbl_2regs:
14206+
result.insThroughput = PERFSCORE_THROUGHPUT_3X;
14207+
result.insLatency = PERFSCORE_LATENCY_2C;
14208+
break;
14209+
case INS_tbl_3regs:
14210+
result.insThroughput = PERFSCORE_THROUGHPUT_4X;
14211+
result.insLatency = PERFSCORE_LATENCY_3C;
14212+
break;
14213+
case INS_tbl_4regs:
14214+
result.insThroughput = PERFSCORE_THROUGHPUT_3X;
14215+
result.insLatency = PERFSCORE_LATENCY_4C;
14216+
break;
14217+
case INS_tbx:
14218+
result.insThroughput = PERFSCORE_THROUGHPUT_3X;
14219+
result.insLatency = PERFSCORE_LATENCY_2C;
14220+
break;
14221+
case INS_tbx_2regs:
14222+
result.insThroughput = PERFSCORE_THROUGHPUT_4X;
14223+
result.insLatency = PERFSCORE_LATENCY_3C;
14224+
break;
14225+
case INS_tbx_3regs:
14226+
result.insThroughput = PERFSCORE_THROUGHPUT_5X;
14227+
result.insLatency = PERFSCORE_LATENCY_4C;
14228+
break;
14229+
case INS_tbx_4regs:
14230+
result.insThroughput = PERFSCORE_THROUGHPUT_6X;
14231+
result.insLatency = PERFSCORE_LATENCY_5C;
14232+
break;
14233+
default:
14234+
// All other instructions
14235+
result.insThroughput = PERFSCORE_THROUGHPUT_2X;
14236+
result.insLatency = PERFSCORE_LATENCY_1C;
14237+
break;
14238+
}
1415114239
break;
1415214240

1415314241
case IF_DV_2E: // mov, dup (scalar)

‎src/coreclr/src/jit/emitarm64.h‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -453,9 +453,9 @@ static emitAttr optGetSrcsize(insOpts conversion);
453453
// for an element of size 'elemsize' in a vector register of size 'datasize'
454454
static bool isValidVectorIndex(emitAttr datasize, emitAttr elemsize, ssize_t index);
455455

456-
// For a given Load/Store Vector instruction 'ins' returns a number of consecutive SIMD registers
457-
// the instruction loads to/store from.
458-
static unsigned insGetLoadStoreRegisterListSize(instruction ins);
456+
// For a given instruction 'ins' which contains a register lists returns a
457+
// number of consecutive SIMD registers the instruction loads to/store from.
458+
static unsigned insGetRegisterListSize(instruction ins);
459459

460460
/************************************************************************/
461461
/* Public inline informational methods */

‎src/coreclr/src/jit/hwintrinsiclistarm64.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,8 @@ HARDWARE_INTRINSIC(AdvSimd, SqrtScalar,
118118
HARDWARE_INTRINSIC(AdvSimd, Store, -1, 2, {INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1, INS_st1}, HW_Category_MemoryStore, HW_Flag_SpecialCodeGen|HW_Flag_BaseTypeFromSecondArg)
119119
HARDWARE_INTRINSIC(AdvSimd, Subtract, -1, 2, {INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_sub, INS_fsub, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
120120
HARDWARE_INTRINSIC(AdvSimd, SubtractScalar, 8, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sub, INS_sub, INS_fsub, INS_fsub}, HW_Category_SIMDScalar, HW_Flag_NoFlag)
121+
HARDWARE_INTRINSIC(AdvSimd, VectorTableLookup, 8, 2, {INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
122+
HARDWARE_INTRINSIC(AdvSimd, VectorTableLookupExtension, 8, 3, {INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_HasRMWSemantics)
121123
HARDWARE_INTRINSIC(AdvSimd, Xor, -1, 2, {INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor, INS_eor}, HW_Category_SimpleSIMD, HW_Flag_Commutative)
122124

123125
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
@@ -195,6 +197,8 @@ HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeEven, -
195197
HARDWARE_INTRINSIC(AdvSimd_Arm64, TransposeOdd, -1, 2, {INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2, INS_trn2}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
196198
HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipEven, -1, 2, {INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1, INS_uzp1}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
197199
HARDWARE_INTRINSIC(AdvSimd_Arm64, UnzipOdd, -1, 2, {INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2, INS_uzp2}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
200+
HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookup, 16, 2, {INS_tbl, INS_tbl, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
201+
HARDWARE_INTRINSIC(AdvSimd_Arm64, VectorTableLookupExtension, 16, 3, {INS_tbx, INS_tbx, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SimpleSIMD, HW_Flag_HasRMWSemantics)
198202
HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipHigh, -1, 2, {INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2, INS_zip2}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
199203
HARDWARE_INTRINSIC(AdvSimd_Arm64, ZipLow, -1, 2, {INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1, INS_zip1}, HW_Category_SimpleSIMD, HW_Flag_NoFlag)
200204

‎src/coreclr/src/jit/instrsarm64.h‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1832,6 +1832,31 @@ INST1(uxtl, "uxtl", 0, 0, IF_DV_2O, 0x2F00A400)
18321832

18331833
INST1(uxtl2, "uxtl2", 0, 0, IF_DV_2O, 0x6F00A400)
18341834
// uxtl2 Vd,Vn DV_2O 011011110iiiiiii 101001nnnnnddddd 6F00 A400 Vd,Vn (shift - vector)
1835+
1836+
INST1(tbl, "tbl", 0, 0, IF_DV_3C, 0x0E000000)
1837+
// tbl Vd,{Vn},Vm DV_3C 0Q001110000mmmmm 000000nnnnnddddd 0E00 0000 Vd,Vn,Vm (vector)
1838+
1839+
INST1(tbl_2regs, "tbl", 0, 0, IF_DV_3C, 0x0E002000)
1840+
// tbl Vd,{Vn,Vn+1},Vm DV_3C 0Q001110000mmmmm 001000nnnnnddddd 0E00 2000 Vd,Vn,Vm (vector)
1841+
1842+
INST1(tbl_3regs, "tbl", 0, 0, IF_DV_3C, 0x0E004000)
1843+
// tbl Vd,{Vn,Vn+1,Vn+2},Vm DV_3C 0Q001110000mmmmm 010000nnnnnddddd 0E00 4000 Vd,Vn,Vm (vector)
1844+
1845+
INST1(tbl_4regs, "tbl", 0, 0, IF_DV_3C, 0x0E006000)
1846+
// tbl Vd,{Vn,Vn+1,Vn+2,Vn+3},Vm DV_3C 0Q001110000mmmmm 011000nnnnnddddd 0E00 6000 Vd,Vn,Vm (vector)
1847+
1848+
INST1(tbx, "tbx", 0, 0, IF_DV_3C, 0x0E001000)
1849+
// tbx Vd,{Vn},Vm DV_3C 0Q001110000mmmmm 000100nnnnnddddd 0E00 1000 Vd,Vn,Vm (vector)
1850+
1851+
INST1(tbx_2regs, "tbx", 0, 0, IF_DV_3C, 0x0E003000)
1852+
// tbx Vd,{Vn,Vn+1},Vm DV_3C 0Q001110000mmmmm 001100nnnnnddddd 0E00 3000 Vd,Vn,Vm (vector)
1853+
1854+
INST1(tbx_3regs, "tbx", 0, 0, IF_DV_3C, 0x0E005000)
1855+
// tbx Vd,{Vn,Vn+1,Vn+2},Vm DV_3C 0Q001110000mmmmm 010100nnnnnddddd 0E00 5000 Vd,Vn,Vm (vector)
1856+
1857+
INST1(tbx_4regs, "tbx", 0, 0, IF_DV_3C, 0x0E007000)
1858+
// tbx Vd,{Vn,Vn+1,Vn+2,Vn+3},Vm DV_3C 0Q001110000mmmmm 011100nnnnnddddd 0E00 7000 Vd,Vn,Vm (vector)
1859+
18351860
// clang-format on
18361861

18371862
/*****************************************************************************/

‎src/coreclr/tests/src/JIT/HardwareIntrinsics/Arm/AdvSimd.Arm64/AdvSimd.Arm64_r.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,10 @@
224224
<Compile Include="TransposeOdd.Vector128.UInt16.cs" />
225225
<Compile Include="TransposeOdd.Vector128.UInt32.cs" />
226226
<Compile Include="TransposeOdd.Vector128.UInt64.cs" />
227+
<Compile Include="VectorTableLookup.Vector128.Byte.cs" />
228+
<Compile Include="VectorTableLookup.Vector128.SByte.cs" />
229+
<Compile Include="VectorTableLookupExtension.Vector128.Byte.cs" />
230+
<Compile Include="VectorTableLookupExtension.Vector128.SByte.cs" />
227231
<Compile Include="UnzipEven.Vector64.Byte.cs" />
228232
<Compile Include="UnzipEven.Vector64.Int16.cs" />
229233
<Compile Include="UnzipEven.Vector64.Int32.cs" />

‎src/coreclr/tests/src/JIT/HardwareIntrinsics/Arm/AdvSimd.Arm64/AdvSimd.Arm64_ro.csproj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,10 @@
224224
<Compile Include="TransposeOdd.Vector128.UInt16.cs" />
225225
<Compile Include="TransposeOdd.Vector128.UInt32.cs" />
226226
<Compile Include="TransposeOdd.Vector128.UInt64.cs" />
227+
<Compile Include="VectorTableLookup.Vector128.Byte.cs" />
228+
<Compile Include="VectorTableLookup.Vector128.SByte.cs" />
229+
<Compile Include="VectorTableLookupExtension.Vector128.Byte.cs" />
230+
<Compile Include="VectorTableLookupExtension.Vector128.SByte.cs" />
227231
<Compile Include="UnzipEven.Vector64.Byte.cs" />
228232
<Compile Include="UnzipEven.Vector64.Int16.cs" />
229233
<Compile Include="UnzipEven.Vector64.Int32.cs" />

0 commit comments

Comments
 (0)