Skip to content

Faster unsigned division by constants #52893

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tannergooding merged 13 commits into from
May 18, 2021
Merged

Faster unsigned division by constants #52893

Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
ee6385f
Faster unsigned division by constants
pentp Mar 13, 2021
cfc7e34
Fix Arm build and add some tests.
pentp Mar 14, 2021
299b6e1
Improve register allocation
pentp Mar 14, 2021
eacfd46
Fix ARM64 codegen
pentp Mar 14, 2021
8b0fc7d
Fix MULHI flags
pentp Mar 15, 2021
c50f0fe
Remove ARM32 codegen
pentp Mar 15, 2021
f595fe6
Widen 32bit UDIV to 64bit MULHI when possible. Improve register alloc…
pentp Mar 16, 2021
58c5564
Always widen 32bit UDIV to 64bit MUL/MULHI
pentp Mar 16, 2021
b6f4b54
Cleanup
pentp Mar 16, 2021
41c2c06
Final optimization
pentp Mar 17, 2021
ad3f5ff
Fix typo
pentp Mar 17, 2021
176562d
Rebase and use inst_Mov instead of inst_RV_RV(INS_mov)
pentp May 18, 2021
935f098
Fix formatting (1 space)
pentp May 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Always widen 32bit UDIV to 64bit MUL/MULHI
  • Loading branch information
@pentp
pentp committed May 18, 2021
commit 58c55645259ef1890f7864ce62098f391297a40d
67 changes: 42 additions & 25 deletions src/coreclr/jit/lower.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5170,26 +5170,35 @@ bool Lowering::LowerUnsignedDivOrMod(GenTreeOp* divMod)
#if defined(TARGET_XARCH) || defined(TARGET_ARM64)
if (!comp->opts.MinOpts() && (divisorValue >= 3))
{
size_t magic;
bool increment;
int preShift;
int postShift;
var_types mulType = type;
size_t magic;
bool increment;
int preShift;
int postShift;
bool simpleMul = false;

if (type == TYP_INT)
{
magic =
MagicDivide::GetUnsigned32Magic(static_cast<uint32_t>(divisorValue), &increment, &preShift, &postShift);

#ifdef TARGET_64BIT
// avoid inc_saturate/multiple shifts by widening the multiplication to 32x64
if (increment || preShift && postShift)
// avoid inc_saturate/multiple shifts by widening to 32x64 MULHI
if (increment || (preShift
#ifdef TARGET_XARCH
// IMUL reg,reg,imm32 can't be used if magic<0 because of sign-extension
&& static_cast<int32_t>(magic) < 0
#endif
))
{
mulType = TYP_LONG;
divisor->gtType = TYP_LONG;
magic = MagicDivide::GetUnsigned64Magic(static_cast<uint64_t>(divisorValue), &increment, &preShift,
&postShift, 32);
}
// otherwise just widen to regular multiplication
else
{
postShift += 32;
simpleMul = true;
}
#endif
}
else
Expand Down Expand Up @@ -5231,22 +5240,23 @@ bool Lowering::LowerUnsignedDivOrMod(GenTreeOp* divMod)
BlockRange().InsertBefore(divMod, preShiftBy, adjustedDividend);
firstNode = preShiftBy;
}
else if (mulType != type)
else if (type != TYP_I_IMPL)
{
adjustedDividend = comp->gtNewCastNode(mulType, adjustedDividend, true, TYP_ULONG);
adjustedDividend = comp->gtNewCastNode(TYP_I_IMPL, adjustedDividend, true, TYP_U_IMPL);
BlockRange().InsertBefore(divMod, adjustedDividend);
firstNode = adjustedDividend;
}

#ifdef TARGET_XARCH
// force input transformation to RAX because the following MULHI will kill RDX:RAX anyway and LSRA often causes
// reduntant copies otherwise
if (firstNode)
if (firstNode && !simpleMul)
adjustedDividend->SetRegNum(REG_RAX);
#endif

divisor->gtType = TYP_I_IMPL;
divisor->AsIntCon()->SetIconValue(magic);
if (isDiv && !postShift && mulType == type)
if (isDiv && !postShift && type == TYP_I_IMPL)
{
divMod->SetOper(GT_MULHI);
divMod->gtOp1 = adjustedDividend;
Expand All @@ -5258,7 +5268,15 @@ bool Lowering::LowerUnsignedDivOrMod(GenTreeOp* divMod)
// The existing node will later be transformed into a GT_RSZ/GT_SUB that
// computes the final result. This way don't need to find and change the use
// of the existing node.
GenTree* mulhi = comp->gtNewOperNode(GT_MULHI, mulType, adjustedDividend, divisor);
GenTree* mulhi;
if (simpleMul)
{
mulhi = comp->gtNewOperNode(GT_MUL, TYP_I_IMPL, adjustedDividend, divisor);
}
else
{
mulhi = comp->gtNewOperNode(GT_MULHI, TYP_I_IMPL, adjustedDividend, divisor);
}
mulhi->gtFlags |= GTF_UNSIGNED;
BlockRange().InsertBefore(divMod, mulhi);
if (!firstNode)
Expand All @@ -5269,27 +5287,18 @@ bool Lowering::LowerUnsignedDivOrMod(GenTreeOp* divMod)
GenTree* shiftBy = comp->gtNewIconNode(postShift, TYP_INT);
BlockRange().InsertBefore(divMod, shiftBy);

if (isDiv)
if (isDiv && type == TYP_I_IMPL)
{
divMod->SetOper(GT_RSZ);
divMod->gtOp1 = mulhi;
divMod->gtOp2 = shiftBy;
}
else
{
mulhi = comp->gtNewOperNode(GT_RSZ, type, mulhi, shiftBy);
mulhi = comp->gtNewOperNode(GT_RSZ, TYP_I_IMPL, mulhi, shiftBy);
BlockRange().InsertBefore(divMod, mulhi);
}
}
else if (isDiv)
{
assert(mulType != type);
divMod->SetOper(GT_CAST);
divMod->gtOp1 = mulhi;
divMod->gtOp2 = nullptr;
divMod->gtFlags |= GTF_UNSIGNED;
divMod->AsCast()->gtCastType = TYP_UINT;
}

if (!isDiv)
{
Expand All @@ -5304,6 +5313,14 @@ bool Lowering::LowerUnsignedDivOrMod(GenTreeOp* divMod)

BlockRange().InsertBefore(divMod, divisor, mul, dividend);
}
else if (type != TYP_I_IMPL)
{
divMod->SetOper(GT_CAST);
divMod->gtOp1 = mulhi;
divMod->gtOp2 = nullptr;
divMod->gtFlags |= GTF_UNSIGNED;
divMod->AsCast()->gtCastType = TYP_UINT;
}
}

if (firstNode)
Expand Down