Skip to content

Avx512 upper regs macro #79423

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
anthonycanino wants to merge 7 commits into from
Closed

Avx512 upper regs macro #79423

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
40807a3
Change regMask_enum and regMaskTP to unsigned __int64_t on AMD64.
anthonycanino Oct 27, 2022
90f7d44
(WIP) Add upper 16 SIMD registers to allocator.
anthonycanino Oct 28, 2022
86b56dd
Refactor code to use `const instrDesc *` instead of `instruction`.
anthonycanino Oct 31, 2022
9768d11
Add EVEX high SIMD reg bits to emitter paths.
anthonycanino Nov 3, 2022
b2d4da6
Limit high SIMD reg to compatible intrinsics lsra build.
anthonycanino Nov 14, 2022
e3cd148
Allow `compiler` to be passed into the CALLEE TRASH macros.
anthonycanino Nov 30, 2022
7bc1399
Fixing some more of the macros.
anthonycanino Dec 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/coreclr/jit/codegenarm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1642,7 +1642,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
);
}

regSet.verifyRegistersUsed(RBM_CALLEE_TRASH);
regSet.verifyRegistersUsed(RBM_CALLEE_TRASH(compiler));
}

#ifdef PROFILING_SUPPORTED
Expand Down Expand Up @@ -1909,7 +1909,7 @@ void CodeGen::genAllocLclFrame(unsigned frameSize, regNumber initReg, bool* pIni
void CodeGen::genPushFltRegs(regMaskTP regMask)
{
assert(regMask != 0); // Don't call uness we have some registers to push
assert((regMask & RBM_ALLFLOAT) == regMask); // Only floasting point registers should be in regMask
assert((regMask & RBM_ALLFLOAT(compiler)) == regMask); // Only floasting point registers should be in regMask

regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask));
int slots = genCountBits(regMask);
Expand All @@ -1928,7 +1928,7 @@ void CodeGen::genPushFltRegs(regMaskTP regMask)
void CodeGen::genPopFltRegs(regMaskTP regMask)
{
assert(regMask != 0); // Don't call uness we have some registers to pop
assert((regMask & RBM_ALLFLOAT) == regMask); // Only floasting point registers should be in regMask
assert((regMask & RBM_ALLFLOAT(compiler)) == regMask); // Only floasting point registers should be in regMask

regNumber lowReg = genRegNumFromMask(genFindLowestBit(regMask));
int slots = genCountBits(regMask);
Expand Down Expand Up @@ -2135,7 +2135,7 @@ void CodeGen::genPopCalleeSavedRegisters(bool jmpEpilog)
assert(compiler->compGeneratingEpilog);

regMaskTP maskPopRegs = regSet.rsGetModifiedRegsMask() & RBM_CALLEE_SAVED;
regMaskTP maskPopRegsFloat = maskPopRegs & RBM_ALLFLOAT;
regMaskTP maskPopRegsFloat = maskPopRegs & RBM_ALLFLOAT(compiler);
regMaskTP maskPopRegsInt = maskPopRegs & ~maskPopRegsFloat;

// First, pop float registers
Expand Down Expand Up @@ -2295,7 +2295,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block)

compiler->unwindBegProlog();

regMaskTP maskPushRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT;
regMaskTP maskPushRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT(compiler);
regMaskTP maskPushRegsInt = genFuncletInfo.fiSaveRegs & ~maskPushRegsFloat;

regMaskTP maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPushRegsFloat);
Expand Down Expand Up @@ -2391,7 +2391,7 @@ void CodeGen::genFuncletEpilog()
/* The saved regs info saves the LR register. We need to pop the PC register to return */
assert(genFuncletInfo.fiSaveRegs & RBM_LR);

regMaskTP maskPopRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT;
regMaskTP maskPopRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT(compiler);
regMaskTP maskPopRegsInt = genFuncletInfo.fiSaveRegs & ~maskPopRegsFloat;

regMaskTP maskStackAlloc = genStackAllocRegisterMask(genFuncletInfo.fiSpDelta, maskPopRegsFloat);
Expand Down
12 changes: 6 additions & 6 deletions src/coreclr/jit/codegenarm64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,7 @@ void CodeGen::genSaveCalleeSavedRegistersHelp(regMaskTP regsToSaveMask, int lowe

// Save integer registers at higher addresses than floating-point registers.

regMaskTP maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT;
regMaskTP maskSaveRegsFloat = regsToSaveMask & RBM_ALLFLOAT(compiler);
regMaskTP maskSaveRegsInt = regsToSaveMask & ~maskSaveRegsFloat;

if (maskSaveRegsFloat != RBM_NONE)
Expand Down Expand Up @@ -1027,7 +1027,7 @@ void CodeGen::genRestoreCalleeSavedRegistersHelp(regMaskTP regsToRestoreMask, in

// Save integer registers at higher addresses than floating-point registers.

regMaskTP maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT;
regMaskTP maskRestoreRegsFloat = regsToRestoreMask & RBM_ALLFLOAT(compiler);
regMaskTP maskRestoreRegsInt = regsToRestoreMask & ~maskRestoreRegsFloat;

// Restore in the opposite order of saving.
Expand Down Expand Up @@ -1352,7 +1352,7 @@ void CodeGen::genFuncletProlog(BasicBlock* block)

compiler->unwindBegProlog();

regMaskTP maskSaveRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT;
regMaskTP maskSaveRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT(compiler);
regMaskTP maskSaveRegsInt = genFuncletInfo.fiSaveRegs & ~maskSaveRegsFloat;

// Funclets must always save LR and FP, since when we have funclets we must have an FP frame.
Expand Down Expand Up @@ -1556,7 +1556,7 @@ void CodeGen::genFuncletEpilog()
unwindStarted = true;
}

regMaskTP maskRestoreRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT;
regMaskTP maskRestoreRegsFloat = genFuncletInfo.fiSaveRegs & RBM_ALLFLOAT(compiler);
regMaskTP maskRestoreRegsInt = genFuncletInfo.fiSaveRegs & ~maskRestoreRegsFloat;

// Funclets must always save LR and FP, since when we have funclets we must have an FP frame.
Expand Down Expand Up @@ -5256,7 +5256,7 @@ void CodeGen::genSIMDIntrinsicInitN(GenTreeSIMD* simdNode)
{
// Note that we cannot use targetReg before consuming all float source operands.
// Therefore use an internal temp register
vectorReg = simdNode->GetSingleTempReg(RBM_ALLFLOAT);
vectorReg = simdNode->GetSingleTempReg(RBM_ALLFLOAT(compiler));
}

// We will first consume the list items in execution (left to right) order,
Expand Down Expand Up @@ -5655,7 +5655,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed)

genEmitHelperCall(CORINFO_HELP_PROF_FCN_ENTER, 0, EA_UNKNOWN);

if ((genRegMask(initReg) & RBM_PROFILER_ENTER_TRASH) != RBM_NONE)
if ((genRegMask(initReg) & RBM_PROFILER_ENTER_TRASH(compiler)) != RBM_NONE)
{
*pInitRegZeroed = false;
}
Expand Down
12 changes: 6 additions & 6 deletions src/coreclr/jit/codegenarmarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2634,7 +2634,7 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node)

if (shouldUse16ByteWideInstrs)
{
const regNumber simdReg = node->GetSingleTempReg(RBM_ALLFLOAT);
const regNumber simdReg = node->GetSingleTempReg(RBM_ALLFLOAT(compiler));

const int initValue = (src->AsIntCon()->IconValue() & 0xFF);
emit->emitIns_R_I(INS_movi, EA_16BYTE, simdReg, initValue, INS_OPTS_16B);
Expand Down Expand Up @@ -2960,8 +2960,8 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node)

if (shouldUse16ByteWideInstrs)
{
const regNumber simdReg1 = node->ExtractTempReg(RBM_ALLFLOAT);
const regNumber simdReg2 = node->GetSingleTempReg(RBM_ALLFLOAT);
const regNumber simdReg1 = node->ExtractTempReg(RBM_ALLFLOAT(compiler));
const regNumber simdReg2 = node->GetSingleTempReg(RBM_ALLFLOAT(compiler));

helper.Unroll(FP_REGSIZE_BYTES, intReg1, simdReg1, simdReg2, srcReg, dstReg, GetEmitter());
}
Expand Down Expand Up @@ -3190,7 +3190,7 @@ void CodeGen::genCall(GenTreeCall* call)
// We should not have GC pointers in killed registers live around the call.
// GC info for arg registers were cleared when consuming arg nodes above
// and LSRA should ensure it for other trashed registers.
regMaskTP killMask = RBM_CALLEE_TRASH;
regMaskTP killMask = RBM_CALLEE_TRASH(compiler);
if (call->IsHelperCall())
{
CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd);
Expand Down Expand Up @@ -4693,7 +4693,7 @@ void CodeGen::genPushCalleeSavedRegisters()
#endif // DEBUG

#if defined(TARGET_ARM)
regMaskTP maskPushRegsFloat = rsPushRegs & RBM_ALLFLOAT;
regMaskTP maskPushRegsFloat = rsPushRegs & RBM_ALLFLOAT(compiler);
regMaskTP maskPushRegsInt = rsPushRegs & ~maskPushRegsFloat;

maskPushRegsInt |= genStackAllocRegisterMask(compiler->compLclFrameSize, maskPushRegsFloat);
Expand Down Expand Up @@ -4800,7 +4800,7 @@ void CodeGen::genPushCalleeSavedRegisters()

int offset; // This will be the starting place for saving the callee-saved registers, in increasing order.

regMaskTP maskSaveRegsFloat = rsPushRegs & RBM_ALLFLOAT;
regMaskTP maskSaveRegsFloat = rsPushRegs & RBM_ALLFLOAT(compiler);
regMaskTP maskSaveRegsInt = rsPushRegs & ~maskSaveRegsFloat;

#ifdef DEBUG
Expand Down
28 changes: 14 additions & 14 deletions src/coreclr/jit/codegencommon.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,19 +590,19 @@ regMaskTP Compiler::compHelperCallKillSet(CorInfoHelpFunc helper)
{
case CORINFO_HELP_ASSIGN_REF:
case CORINFO_HELP_CHECKED_ASSIGN_REF:
return RBM_CALLEE_TRASH_WRITEBARRIER;
return RBM_CALLEE_TRASH_WRITEBARRIER(this);

case CORINFO_HELP_ASSIGN_BYREF:
return RBM_CALLEE_TRASH_WRITEBARRIER_BYREF;
return RBM_CALLEE_TRASH_WRITEBARRIER_BYREF(this);

case CORINFO_HELP_PROF_FCN_ENTER:
return RBM_PROFILER_ENTER_TRASH;
return RBM_PROFILER_ENTER_TRASH(this);

case CORINFO_HELP_PROF_FCN_LEAVE:
return RBM_PROFILER_LEAVE_TRASH;
return RBM_PROFILER_LEAVE_TRASH(this);

case CORINFO_HELP_PROF_FCN_TAILCALL:
return RBM_PROFILER_TAILCALL_TRASH;
return RBM_PROFILER_TAILCALL_TRASH(this);

#ifdef TARGET_X86
case CORINFO_HELP_ASSIGN_REF_EAX:
Expand All @@ -622,16 +622,16 @@ regMaskTP Compiler::compHelperCallKillSet(CorInfoHelpFunc helper)
#endif

case CORINFO_HELP_STOP_FOR_GC:
return RBM_STOP_FOR_GC_TRASH;
return RBM_STOP_FOR_GC_TRASH(this);

case CORINFO_HELP_INIT_PINVOKE_FRAME:
return RBM_INIT_PINVOKE_FRAME_TRASH;
return RBM_INIT_PINVOKE_FRAME_TRASH(this);

case CORINFO_HELP_VALIDATE_INDIRECT_CALL:
return RBM_VALIDATE_INDIRECT_CALL_TRASH;

default:
return RBM_CALLEE_TRASH;
return RBM_CALLEE_TRASH(this);
}
}

Expand Down Expand Up @@ -3590,18 +3590,18 @@ void CodeGen::genFnPrologCalleeRegArgs(regNumber xtraReg, bool* pXtraRegClobbere

regMaskTP fpAvailMask;

fpAvailMask = RBM_FLT_CALLEE_TRASH & ~regArgMaskLive;
fpAvailMask = RBM_FLT_CALLEE_TRASH(compiler) & ~regArgMaskLive;
if (GlobalJitOptions::compFeatureHfa)
{
fpAvailMask &= RBM_ALLDOUBLE;
fpAvailMask &= RBM_ALLDOUBLE(compiler);
}

if (fpAvailMask == RBM_NONE)
{
fpAvailMask = RBM_ALLFLOAT & ~regArgMaskLive;
fpAvailMask = RBM_ALLFLOAT(compiler) & ~regArgMaskLive;
if (GlobalJitOptions::compFeatureHfa)
{
fpAvailMask &= RBM_ALLDOUBLE;
fpAvailMask &= RBM_ALLDOUBLE(compiler);
}
}

Expand Down Expand Up @@ -5246,7 +5246,7 @@ void CodeGen::genFinalizeFrame()
// We always save FP.
noway_assert(isFramePointerUsed());
#if defined(TARGET_AMD64) || defined(TARGET_ARM64)
regMaskTP okRegs = (RBM_CALLEE_TRASH | RBM_FPBASE | RBM_ENC_CALLEE_SAVED);
regMaskTP okRegs = (RBM_CALLEE_TRASH(compiler) | RBM_FPBASE | RBM_ENC_CALLEE_SAVED);
if (RBM_ENC_CALLEE_SAVED != 0)
{
regSet.rsSetRegsModified(RBM_ENC_CALLEE_SAVED);
Expand Down Expand Up @@ -5304,7 +5304,7 @@ void CodeGen::genFinalizeFrame()

#if defined(TARGET_ARM)
// TODO-ARM64-Bug?: enable some variant of this for FP on ARM64?
regMaskTP maskPushRegsFloat = maskCalleeRegsPushed & RBM_ALLFLOAT;
regMaskTP maskPushRegsFloat = maskCalleeRegsPushed & RBM_ALLFLOAT(compiler);
regMaskTP maskPushRegsInt = maskCalleeRegsPushed & ~maskPushRegsFloat;

if ((maskPushRegsFloat != RBM_NONE) ||
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/jit/codegenlinear.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2332,8 +2332,8 @@ void CodeGen::genEmitCallIndir(int callType,

// These should have been put in volatile registers to ensure they do not
// get overridden by epilog sequence during tailcall.
noway_assert(!isJump || (iReg == REG_NA) || ((RBM_CALLEE_TRASH & genRegMask(iReg)) != 0));
noway_assert(!isJump || (xReg == REG_NA) || ((RBM_CALLEE_TRASH & genRegMask(xReg)) != 0));
noway_assert(!isJump || (iReg == REG_NA) || ((RBM_CALLEE_TRASH(compiler) & genRegMask(iReg)) != 0));
noway_assert(!isJump || (xReg == REG_NA) || ((RBM_CALLEE_TRASH(compiler) & genRegMask(xReg)) != 0));

GetEmitter()->emitIns_Call(emitter::EmitCallType(callType),
methHnd,
Expand Down
20 changes: 10 additions & 10 deletions src/coreclr/jit/codegenxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2869,7 +2869,7 @@ void CodeGen::genCodeForInitBlkUnroll(GenTreeBlk* node)

if (willUseSimdMov)
{
regNumber srcXmmReg = node->GetSingleTempReg(RBM_ALLFLOAT);
regNumber srcXmmReg = node->GetSingleTempReg(RBM_ALLFLOAT(compiler));

unsigned regSize = (size >= YMM_REGSIZE_BYTES) && compiler->compOpportunisticallyDependsOn(InstructionSet_AVX)
? YMM_REGSIZE_BYTES
Expand Down Expand Up @@ -3137,7 +3137,7 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node)

if (size >= XMM_REGSIZE_BYTES)
{
regNumber tempReg = node->GetSingleTempReg(RBM_ALLFLOAT);
regNumber tempReg = node->GetSingleTempReg(RBM_ALLFLOAT(compiler));

instruction simdMov = simdUnalignedMovIns();

Expand Down Expand Up @@ -3459,7 +3459,7 @@ void CodeGen::genStructPutArgUnroll(GenTreePutArgStk* putArgNode)
if (loadSize >= XMM_REGSIZE_BYTES)
#endif
{
xmmTmpReg = putArgNode->GetSingleTempReg(RBM_ALLFLOAT);
xmmTmpReg = putArgNode->GetSingleTempReg(RBM_ALLFLOAT(compiler));
}
if ((loadSize % XMM_REGSIZE_BYTES) != 0)
{
Expand Down Expand Up @@ -5655,7 +5655,7 @@ void CodeGen::genCall(GenTreeCall* call)
// We should not have GC pointers in killed registers live around the call.
// GC info for arg registers were cleared when consuming arg nodes above
// and LSRA should ensure it for other trashed registers.
regMaskTP killMask = RBM_CALLEE_TRASH;
regMaskTP killMask = RBM_CALLEE_TRASH(compiler);
if (call->IsHelperCall())
{
CorInfoHelpFunc helpFunc = compiler->eeGetHelperNum(call->gtCallMethHnd);
Expand Down Expand Up @@ -7910,13 +7910,13 @@ void CodeGen::genPutArgStkFieldList(GenTreePutArgStk* putArgStk)
intTmpReg = putArgStk->GetSingleTempReg(RBM_ALLINT);
assert(genIsValidIntReg(intTmpReg));
}
if ((rsvdRegs & RBM_ALLFLOAT) != 0)
if ((rsvdRegs & RBM_ALLFLOAT(compiler)) != 0)
{
simdTmpReg = putArgStk->GetSingleTempReg(RBM_ALLFLOAT);
simdTmpReg = putArgStk->GetSingleTempReg(RBM_ALLFLOAT(compiler));
assert(genIsValidFloatReg(simdTmpReg));
}
assert(genCountBits(rsvdRegs) == (unsigned)((intTmpReg == REG_NA) ? 0 : 1) + ((simdTmpReg == REG_NA) ? 0 : 1));
}
}

for (GenTreeFieldList::Use& use : fieldList->Uses())
{
Expand Down Expand Up @@ -9100,7 +9100,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed)
}

// If initReg is one of RBM_CALLEE_TRASH, then it needs to be zero'ed before using.
if ((RBM_CALLEE_TRASH & genRegMask(initReg)) != 0)
if ((RBM_CALLEE_TRASH(compiler) & genRegMask(initReg)) != 0)
{
*pInitRegZeroed = false;
}
Expand Down Expand Up @@ -9137,7 +9137,7 @@ void CodeGen::genProfilingEnterCallback(regNumber initReg, bool* pInitRegZeroed)
genEmitHelperCall(CORINFO_HELP_PROF_FCN_ENTER, 0, EA_UNKNOWN, REG_DEFAULT_PROFILER_CALL_TARGET);

// If initReg is one of RBM_CALLEE_TRASH, then it needs to be zero'ed before using.
if ((RBM_CALLEE_TRASH & genRegMask(initReg)) != 0)
if ((RBM_CALLEE_TRASH(compiler) & genRegMask(initReg)) != 0)
{
*pInitRegZeroed = false;
}
Expand Down Expand Up @@ -9178,7 +9178,7 @@ void CodeGen::genProfilingLeaveCallback(unsigned helper)
if (compiler->lvaKeepAliveAndReportThis() && compiler->lvaGetDesc(compiler->info.compThisArg)->lvIsInReg())
{
regMaskTP thisPtrMask = genRegMask(compiler->lvaGetDesc(compiler->info.compThisArg)->GetRegNum());
noway_assert((RBM_PROFILER_LEAVE_TRASH & thisPtrMask) == 0);
noway_assert((RBM_PROFILER_LEAVE_TRASH(compiler) & thisPtrMask) == 0);
}

// At this point return value is computed and stored in RAX or XMM0.
Expand Down
1 change: 1 addition & 0 deletions src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9055,6 +9055,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
// Returns:
// `true` if user requests EVEX encoding and it's safe, `false` if not.
//
public:
bool DoJitStressEvexEncoding() const
{
#if defined(TARGET_XARCH) && defined(DEBUG)
Expand Down
Loading