diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 352d1b2acfcd7b..913951568fd1bd 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -1023,9 +1023,9 @@ void emitter::emitBegFN(bool hasFramePtr emitIGbuffSize = 0; #if FEATURE_LOOP_ALIGN - emitLastAlignedIgNum = 0; - emitLastInnerLoopStartIgNum = 0; - emitLastInnerLoopEndIgNum = 0; + emitLastAlignedIgNum = 0; + emitLastLoopStart = 0; + emitLastLoopEnd = 0; #endif /* Record stack frame info (the temp size is just an estimate) */ @@ -4820,58 +4820,112 @@ unsigned emitter::getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG // if currIG has back-edge to dstIG. // // Notes: -// If the current loop encloses a loop that is already marked as align, then remove -// the alignment flag present on IG before dstIG. +// Despite we align only inner most loop, we might see intersected loops because of control flow +// re-arrangement like adding a split edge in LSRA. +// +// If there is an intersection of current loop with last loop that is already marked as align, +// then *do not align* one of the loop that completely encloses the other one. Or if they both intersect, +// then *do not align* either of them because since the flow is complicated enough that aligning one of them +// will not improve the performance. // void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) { - insGroup* dstIG = (insGroup*)loopTopBlock->bbEmitCookie; + insGroup* dstIG = (insGroup*)loopTopBlock->bbEmitCookie; + bool alignCurrentLoop = true; + bool alignLastLoop = true; // With (dstIG != nullptr), ensure that only back edges are tracked. // If there is forward jump, dstIG is not yet generated. // // We don't rely on (block->bbJumpDest->bbNum <= block->bbNum) because the basic // block numbering is not guaranteed to be sequential. - if ((dstIG != nullptr) && (dstIG->igNum <= emitCurIG->igNum)) { unsigned currLoopStart = dstIG->igNum; unsigned currLoopEnd = emitCurIG->igNum; // Only mark back-edge if current loop starts after the last inner loop ended. - if (emitLastInnerLoopEndIgNum < currLoopStart) + if (emitLastLoopEnd < currLoopStart) { emitCurIG->igLoopBackEdge = dstIG; JITDUMP("** IG%02u jumps back to IG%02u forming a loop.\n", currLoopEnd, currLoopStart); - emitLastInnerLoopStartIgNum = currLoopStart; - emitLastInnerLoopEndIgNum = currLoopEnd; + emitLastLoopStart = currLoopStart; + emitLastLoopEnd = currLoopEnd; + } + else if (currLoopStart == emitLastLoopStart) + { + // Note: If current and last loop starts at same point, + // retain the alignment flag of the smaller loop. + // | + // .---->|<----. + // last | | | + // loop | | | current + // .---->| | loop + // | | + // |-----. + // + } + else if ((currLoopStart < emitLastLoopStart) && (emitLastLoopEnd < currLoopEnd)) + { + // if current loop completely encloses last loop, + // then current loop should not be aligned. + alignCurrentLoop = false; + } + else if ((emitLastLoopStart < currLoopStart) && (currLoopEnd < emitLastLoopEnd)) + { + // if last loop completely encloses current loop, + // then last loop should not be aligned. + alignLastLoop = false; + } + else + { + // The loops intersect and should not align either of the loops + alignLastLoop = false; + alignCurrentLoop = false; } - // Otherwise, mark the dstIG->prevIG as no alignment needed. - // - // Note: If current loop's back-edge target is same as emitLastInnerLoopStartIgNum, - // retain the alignment flag of dstIG->prevIG so the loop - // (emitLastInnerLoopStartIgNum ~ emitLastInnerLoopEndIgNum) is still aligned. - else if (emitLastInnerLoopStartIgNum != currLoopStart) - { - // Find the IG before dstIG... - instrDescAlign* alignInstr = emitAlignList; - while ((alignInstr != nullptr) && (alignInstr->idaIG->igNext != dstIG)) - { - alignInstr = alignInstr->idaNext; - } - // ...and clear the IGF_LOOP_ALIGN flag - if (alignInstr != nullptr) + if (!alignLastLoop || !alignCurrentLoop) + { + instrDescAlign* alignInstr = emitAlignList; + bool markedLastLoop = alignLastLoop; + bool markedCurrLoop = alignCurrentLoop; + while ((alignInstr != nullptr)) { - assert(alignInstr->idaIG->igNext == dstIG); - alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; + // Find the IG before current loop and clear the IGF_LOOP_ALIGN flag + if (!alignCurrentLoop && (alignInstr->idaIG->igNext == dstIG)) + { + assert(!markedCurrLoop); + alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; + markedCurrLoop = true; + JITDUMP("** Skip alignment for current loop IG%02u ~ IG%02u because it encloses an aligned loop " + "IG%02u ~ IG%02u.\n", + currLoopStart, currLoopEnd, emitLastLoopStart, emitLastLoopEnd); + } + + // Find the IG before the last loop and clear the IGF_LOOP_ALIGN flag + if (!alignLastLoop && (alignInstr->idaIG->igNext != nullptr) && + (alignInstr->idaIG->igNext->igNum == emitLastLoopStart)) + { + assert(!markedLastLoop); + assert(alignInstr->idaIG->isLoopAlign()); + alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; + markedLastLoop = true; + JITDUMP("** Skip alignment for aligned loop IG%02u ~ IG%02u because it encloses the current loop " + "IG%02u ~ IG%02u.\n", + emitLastLoopStart, emitLastLoopEnd, currLoopStart, currLoopEnd); + } + + if (markedLastLoop && markedCurrLoop) + { + break; + } + + alignInstr = alignInstr->idaNext; } - JITDUMP( - "** Skip alignment for loop IG%02u ~ IG%02u, because it encloses an aligned loop IG%02u ~ IG%02u.\n", - currLoopStart, currLoopEnd, emitLastInnerLoopStartIgNum, emitLastInnerLoopEndIgNum); + assert(markedLastLoop && markedCurrLoop); } } } diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 666201fc98bf00..2748acf4637665 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -1762,12 +1762,12 @@ class emitter void emitJumpDistBind(); // Bind all the local jumps in method #if FEATURE_LOOP_ALIGN - instrDescAlign* emitCurIGAlignList; // list of align instructions in current IG - unsigned emitLastInnerLoopStartIgNum; // Start IG of last inner loop - unsigned emitLastInnerLoopEndIgNum; // End IG of last inner loop - unsigned emitLastAlignedIgNum; // last IG that has align instruction - instrDescAlign* emitAlignList; // list of local align instructions in method - instrDescAlign* emitAlignLast; // last align instruction in method + instrDescAlign* emitCurIGAlignList; // list of align instructions in current IG + unsigned emitLastLoopStart; // Start IG of last inner loop + unsigned emitLastLoopEnd; // End IG of last inner loop + unsigned emitLastAlignedIgNum; // last IG that has align instruction + instrDescAlign* emitAlignList; // list of local align instructions in method + instrDescAlign* emitAlignLast; // last align instruction in method unsigned getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG_ARG(bool isAlignAdjusted)); // Get the smallest loop size void emitLoopAlignment();