From 2565e4df04c07aac08b5d3b0eba9cd32a83cc255 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Thu, 1 Jul 2021 16:38:19 -0700 Subject: [PATCH 1/4] Handle blocks added in loop as part of split edges of LSRA If there are new blocks added by LSRA and modifies the flow of blocks that are in loop, then make sure that we do not align such loops if they intersect with last aligned loop. --- src/coreclr/jit/emit.cpp | 112 ++++++++++++++++++++++++++++++--------- src/coreclr/jit/emit.h | 4 +- 2 files changed, 88 insertions(+), 28 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 352d1b2acfcd7b..e47a8607727cac 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -1024,8 +1024,8 @@ void emitter::emitBegFN(bool hasFramePtr #if FEATURE_LOOP_ALIGN emitLastAlignedIgNum = 0; - emitLastInnerLoopStartIgNum = 0; - emitLastInnerLoopEndIgNum = 0; + emitLastLoopStart = 0; + emitLastLoopEnd = 0; #endif /* Record stack frame info (the temp size is just an estimate) */ @@ -4820,58 +4820,118 @@ unsigned emitter::getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG // if currIG has back-edge to dstIG. // // Notes: -// If the current loop encloses a loop that is already marked as align, then remove -// the alignment flag present on IG before dstIG. +// Despite we align only inner most loop, we might see intersected loops because of control flow +// re-arrangement like adding a split edge in LSRA. +// +// If there is an intersection of current loop with last loop that is already marked as align, +// then *do not align* one of the loop that completely encloses the other one. Or if they both intersect, +// then *do not align* either of them because since the flow is complicated enough that aligning one of them +// will not improve the performance. // void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) { insGroup* dstIG = (insGroup*)loopTopBlock->bbEmitCookie; + bool noAlignCurrentLoop = false; + bool noAlignLastLoop = false; // With (dstIG != nullptr), ensure that only back edges are tracked. // If there is forward jump, dstIG is not yet generated. // // We don't rely on (block->bbJumpDest->bbNum <= block->bbNum) because the basic // block numbering is not guaranteed to be sequential. - if ((dstIG != nullptr) && (dstIG->igNum <= emitCurIG->igNum)) { unsigned currLoopStart = dstIG->igNum; unsigned currLoopEnd = emitCurIG->igNum; // Only mark back-edge if current loop starts after the last inner loop ended. - if (emitLastInnerLoopEndIgNum < currLoopStart) + if (emitLastLoopEnd < currLoopStart) { emitCurIG->igLoopBackEdge = dstIG; JITDUMP("** IG%02u jumps back to IG%02u forming a loop.\n", currLoopEnd, currLoopStart); - emitLastInnerLoopStartIgNum = currLoopStart; - emitLastInnerLoopEndIgNum = currLoopEnd; + emitLastLoopStart = currLoopStart; + emitLastLoopEnd = currLoopEnd; + } + // else if current loop completely encloses last loop, + // then current loop should not be aligned. + else if ((currLoopStart <= emitLastLoopStart) && (emitLastLoopEnd < currLoopEnd)) + { + // Note: If current and last loop starts at same point, + // retain the alignment flag of the smaller loop. + // | + // .---->|<----. + // last | | | + // loop | | | current + // .---->| | loop + // | | + // |-----. + // + noAlignCurrentLoop = true; + } + + // else if last loop completely encloses current loop, + // then last loop should not be aligned. + else if ((emitLastLoopStart <= currLoopStart) && (currLoopEnd < emitLastLoopEnd)) + { + // Note: If current and last loop starts at same point, + // retain the alignment flag of the smaller loop. + // | + // .---->|<----. + // last | | | + // loop | | | current + // | |-----. loop + // | | + // .---->| + // + noAlignLastLoop = true; } - // Otherwise, mark the dstIG->prevIG as no alignment needed. - // - // Note: If current loop's back-edge target is same as emitLastInnerLoopStartIgNum, - // retain the alignment flag of dstIG->prevIG so the loop - // (emitLastInnerLoopStartIgNum ~ emitLastInnerLoopEndIgNum) is still aligned. - else if (emitLastInnerLoopStartIgNum != currLoopStart) + else + { + // The loops intersect and should not align either of the loops + noAlignLastLoop = true; + noAlignCurrentLoop = true; + } + + if (noAlignLastLoop || noAlignCurrentLoop) { - // Find the IG before dstIG... instrDescAlign* alignInstr = emitAlignList; - while ((alignInstr != nullptr) && (alignInstr->idaIG->igNext != dstIG)) + bool markedLastLoop = !noAlignLastLoop; + bool markedCurrLoop = !noAlignCurrentLoop; + while ((alignInstr != nullptr)) { - alignInstr = alignInstr->idaNext; - } + // Find the IG before current loop and clear the IGF_LOOP_ALIGN flag + if (noAlignCurrentLoop && (alignInstr->idaIG->igNext == dstIG)) + { + assert(!markedCurrLoop); + alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; + markedCurrLoop = true; + JITDUMP("** Skip alignment for loop IG%02u ~ IG%02u because it encloses an aligned loop IG%02u ~ IG%02u.\n", + currLoopStart, currLoopEnd, emitLastLoopStart, emitLastLoopEnd); + } - // ...and clear the IGF_LOOP_ALIGN flag - if (alignInstr != nullptr) - { - assert(alignInstr->idaIG->igNext == dstIG); - alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; + // Find the IG before the last loop and clear the IGF_LOOP_ALIGN flag + if (noAlignLastLoop && (alignInstr->idaIG->igNext != nullptr) && + (alignInstr->idaIG->igNext->igNum == emitLastLoopStart)) + { + assert(!markedLastLoop); + assert(alignInstr->idaIG->isLoopAlign()); + alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; + markedLastLoop = true; + JITDUMP("** Skip alignment for loop IG%02u ~ IG%02u because it encloses an aligned loop IG%02u ~ IG%02u.\n", + emitLastLoopStart, emitLastLoopEnd, currLoopStart, currLoopEnd); + } + + if (markedLastLoop && markedCurrLoop) + { + break; + } + + alignInstr = alignInstr->idaNext; } - JITDUMP( - "** Skip alignment for loop IG%02u ~ IG%02u, because it encloses an aligned loop IG%02u ~ IG%02u.\n", - currLoopStart, currLoopEnd, emitLastInnerLoopStartIgNum, emitLastInnerLoopEndIgNum); + assert(markedLastLoop && markedCurrLoop); } } } diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 666201fc98bf00..7163334d66b211 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -1763,8 +1763,8 @@ class emitter #if FEATURE_LOOP_ALIGN instrDescAlign* emitCurIGAlignList; // list of align instructions in current IG - unsigned emitLastInnerLoopStartIgNum; // Start IG of last inner loop - unsigned emitLastInnerLoopEndIgNum; // End IG of last inner loop + unsigned emitLastLoopStart; // Start IG of last inner loop + unsigned emitLastLoopEnd; // End IG of last inner loop unsigned emitLastAlignedIgNum; // last IG that has align instruction instrDescAlign* emitAlignList; // list of local align instructions in method instrDescAlign* emitAlignLast; // last align instruction in method From 5d777fa92293bbf8fc2696c8e1e141e428446c2c Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 2 Jul 2021 09:35:42 -0700 Subject: [PATCH 2/4] Retain LOOP_ALIGN flag of loops whose start are same --- src/coreclr/jit/emit.cpp | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index e47a8607727cac..1452ce388ceecc 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -4854,9 +4854,7 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) emitLastLoopStart = currLoopStart; emitLastLoopEnd = currLoopEnd; } - // else if current loop completely encloses last loop, - // then current loop should not be aligned. - else if ((currLoopStart <= emitLastLoopStart) && (emitLastLoopEnd < currLoopEnd)) + else if (currLoopStart == emitLastLoopStart) { // Note: If current and last loop starts at same point, // retain the alignment flag of the smaller loop. @@ -4868,23 +4866,18 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) // | | // |-----. // + } + // else if current loop completely encloses last loop, + // then current loop should not be aligned. + else if ((currLoopStart < emitLastLoopStart) && (emitLastLoopEnd < currLoopEnd)) + { noAlignCurrentLoop = true; } // else if last loop completely encloses current loop, // then last loop should not be aligned. - else if ((emitLastLoopStart <= currLoopStart) && (currLoopEnd < emitLastLoopEnd)) + else if ((emitLastLoopStart < currLoopStart) && (currLoopEnd < emitLastLoopEnd)) { - // Note: If current and last loop starts at same point, - // retain the alignment flag of the smaller loop. - // | - // .---->|<----. - // last | | | - // loop | | | current - // | |-----. loop - // | | - // .---->| - // noAlignLastLoop = true; } else @@ -4907,7 +4900,7 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) assert(!markedCurrLoop); alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; markedCurrLoop = true; - JITDUMP("** Skip alignment for loop IG%02u ~ IG%02u because it encloses an aligned loop IG%02u ~ IG%02u.\n", + JITDUMP("** Skip alignment for current loop IG%02u ~ IG%02u because it encloses an aligned loop IG%02u ~ IG%02u.\n", currLoopStart, currLoopEnd, emitLastLoopStart, emitLastLoopEnd); } @@ -4919,7 +4912,7 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) assert(alignInstr->idaIG->isLoopAlign()); alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; markedLastLoop = true; - JITDUMP("** Skip alignment for loop IG%02u ~ IG%02u because it encloses an aligned loop IG%02u ~ IG%02u.\n", + JITDUMP("** Skip alignment for aligned loop IG%02u ~ IG%02u because it encloses the current loop IG%02u ~ IG%02u.\n", emitLastLoopStart, emitLastLoopEnd, currLoopStart, currLoopEnd); } From aae9dcd1921c85dd533c444cea2eb404420700d5 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Fri, 2 Jul 2021 09:37:07 -0700 Subject: [PATCH 3/4] jit format --- src/coreclr/jit/emit.cpp | 31 ++++++++++++++++--------------- src/coreclr/jit/emit.h | 12 ++++++------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 1452ce388ceecc..bf0a32296cd739 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -1023,9 +1023,9 @@ void emitter::emitBegFN(bool hasFramePtr emitIGbuffSize = 0; #if FEATURE_LOOP_ALIGN - emitLastAlignedIgNum = 0; - emitLastLoopStart = 0; - emitLastLoopEnd = 0; + emitLastAlignedIgNum = 0; + emitLastLoopStart = 0; + emitLastLoopEnd = 0; #endif /* Record stack frame info (the temp size is just an estimate) */ @@ -4822,7 +4822,7 @@ unsigned emitter::getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG // Notes: // Despite we align only inner most loop, we might see intersected loops because of control flow // re-arrangement like adding a split edge in LSRA. -// +// // If there is an intersection of current loop with last loop that is already marked as align, // then *do not align* one of the loop that completely encloses the other one. Or if they both intersect, // then *do not align* either of them because since the flow is complicated enough that aligning one of them @@ -4830,9 +4830,9 @@ unsigned emitter::getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG // void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) { - insGroup* dstIG = (insGroup*)loopTopBlock->bbEmitCookie; - bool noAlignCurrentLoop = false; - bool noAlignLastLoop = false; + insGroup* dstIG = (insGroup*)loopTopBlock->bbEmitCookie; + bool noAlignCurrentLoop = false; + bool noAlignLastLoop = false; // With (dstIG != nullptr), ensure that only back edges are tracked. // If there is forward jump, dstIG is not yet generated. @@ -4867,17 +4867,16 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) // |-----. // } - // else if current loop completely encloses last loop, - // then current loop should not be aligned. else if ((currLoopStart < emitLastLoopStart) && (emitLastLoopEnd < currLoopEnd)) { + // if current loop completely encloses last loop, + // then current loop should not be aligned. noAlignCurrentLoop = true; } - - // else if last loop completely encloses current loop, - // then last loop should not be aligned. else if ((emitLastLoopStart < currLoopStart) && (currLoopEnd < emitLastLoopEnd)) { + // if last loop completely encloses current loop, + // then last loop should not be aligned. noAlignLastLoop = true; } else @@ -4889,7 +4888,7 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) if (noAlignLastLoop || noAlignCurrentLoop) { - instrDescAlign* alignInstr = emitAlignList; + instrDescAlign* alignInstr = emitAlignList; bool markedLastLoop = !noAlignLastLoop; bool markedCurrLoop = !noAlignCurrentLoop; while ((alignInstr != nullptr)) @@ -4900,7 +4899,8 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) assert(!markedCurrLoop); alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; markedCurrLoop = true; - JITDUMP("** Skip alignment for current loop IG%02u ~ IG%02u because it encloses an aligned loop IG%02u ~ IG%02u.\n", + JITDUMP("** Skip alignment for current loop IG%02u ~ IG%02u because it encloses an aligned loop " + "IG%02u ~ IG%02u.\n", currLoopStart, currLoopEnd, emitLastLoopStart, emitLastLoopEnd); } @@ -4912,7 +4912,8 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) assert(alignInstr->idaIG->isLoopAlign()); alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; markedLastLoop = true; - JITDUMP("** Skip alignment for aligned loop IG%02u ~ IG%02u because it encloses the current loop IG%02u ~ IG%02u.\n", + JITDUMP("** Skip alignment for aligned loop IG%02u ~ IG%02u because it encloses the current loop " + "IG%02u ~ IG%02u.\n", emitLastLoopStart, emitLastLoopEnd, currLoopStart, currLoopEnd); } diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 7163334d66b211..2748acf4637665 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -1762,12 +1762,12 @@ class emitter void emitJumpDistBind(); // Bind all the local jumps in method #if FEATURE_LOOP_ALIGN - instrDescAlign* emitCurIGAlignList; // list of align instructions in current IG - unsigned emitLastLoopStart; // Start IG of last inner loop - unsigned emitLastLoopEnd; // End IG of last inner loop - unsigned emitLastAlignedIgNum; // last IG that has align instruction - instrDescAlign* emitAlignList; // list of local align instructions in method - instrDescAlign* emitAlignLast; // last align instruction in method + instrDescAlign* emitCurIGAlignList; // list of align instructions in current IG + unsigned emitLastLoopStart; // Start IG of last inner loop + unsigned emitLastLoopEnd; // End IG of last inner loop + unsigned emitLastAlignedIgNum; // last IG that has align instruction + instrDescAlign* emitAlignList; // list of local align instructions in method + instrDescAlign* emitAlignLast; // last align instruction in method unsigned getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG_ARG(bool isAlignAdjusted)); // Get the smallest loop size void emitLoopAlignment(); From cd2004c9e256170540546b6b748a4204d4824902 Mon Sep 17 00:00:00 2001 From: Kunal Pathak Date: Wed, 7 Jul 2021 23:39:03 -0700 Subject: [PATCH 4/4] review feedback --- src/coreclr/jit/emit.cpp | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index bf0a32296cd739..913951568fd1bd 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -4830,9 +4830,9 @@ unsigned emitter::getLoopSize(insGroup* igLoopHeader, unsigned maxLoopSize DEBUG // void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) { - insGroup* dstIG = (insGroup*)loopTopBlock->bbEmitCookie; - bool noAlignCurrentLoop = false; - bool noAlignLastLoop = false; + insGroup* dstIG = (insGroup*)loopTopBlock->bbEmitCookie; + bool alignCurrentLoop = true; + bool alignLastLoop = true; // With (dstIG != nullptr), ensure that only back edges are tracked. // If there is forward jump, dstIG is not yet generated. @@ -4871,30 +4871,30 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) { // if current loop completely encloses last loop, // then current loop should not be aligned. - noAlignCurrentLoop = true; + alignCurrentLoop = false; } else if ((emitLastLoopStart < currLoopStart) && (currLoopEnd < emitLastLoopEnd)) { // if last loop completely encloses current loop, // then last loop should not be aligned. - noAlignLastLoop = true; + alignLastLoop = false; } else { // The loops intersect and should not align either of the loops - noAlignLastLoop = true; - noAlignCurrentLoop = true; + alignLastLoop = false; + alignCurrentLoop = false; } - if (noAlignLastLoop || noAlignCurrentLoop) + if (!alignLastLoop || !alignCurrentLoop) { instrDescAlign* alignInstr = emitAlignList; - bool markedLastLoop = !noAlignLastLoop; - bool markedCurrLoop = !noAlignCurrentLoop; + bool markedLastLoop = alignLastLoop; + bool markedCurrLoop = alignCurrentLoop; while ((alignInstr != nullptr)) { // Find the IG before current loop and clear the IGF_LOOP_ALIGN flag - if (noAlignCurrentLoop && (alignInstr->idaIG->igNext == dstIG)) + if (!alignCurrentLoop && (alignInstr->idaIG->igNext == dstIG)) { assert(!markedCurrLoop); alignInstr->idaIG->igFlags &= ~IGF_LOOP_ALIGN; @@ -4905,7 +4905,7 @@ void emitter::emitSetLoopBackEdge(BasicBlock* loopTopBlock) } // Find the IG before the last loop and clear the IGF_LOOP_ALIGN flag - if (noAlignLastLoop && (alignInstr->idaIG->igNext != nullptr) && + if (!alignLastLoop && (alignInstr->idaIG->igNext != nullptr) && (alignInstr->idaIG->igNext->igNum == emitLastLoopStart)) { assert(!markedLastLoop);