Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
d441794
update API of addMutableState
kiszk Nov 24, 2017
870d106
eliminate initialization with default value
kiszk Nov 25, 2017
24d7087
allocate a global Java array to store a lot of mutable state in a class
kiszk Nov 26, 2017
3eb5842
fix scala style error
kiszk Nov 26, 2017
074d711
fix test failure of ExpressionEncoderSuite.NestedArray
kiszk Nov 27, 2017
eafa3f8
rebase with master
kiszk Nov 30, 2017
90d15f3
address review comments
kiszk Nov 30, 2017
c456c07
add useFreshname parameter to addMutableState
kiszk Nov 30, 2017
9ca5ab3
rebase with master
kiszk Nov 30, 2017
5b36c61
fix test failures
kiszk Dec 1, 2017
fd51d75
drop to creat a loop for initialization
kiszk Dec 7, 2017
effe918
fix test failure
kiszk Dec 8, 2017
9df109c
update comments
kiszk Dec 8, 2017
634d494
address review comment
kiszk Dec 10, 2017
d3438fd
address review comment
kiszk Dec 12, 2017
f4f3754
address review comment
kiszk Dec 12, 2017
f1e1fca
address review comments except test case
kiszk Dec 13, 2017
0937ef2
rebase with master
kiszk Dec 13, 2017
4bfcc1a
Do not use compaction as possible for frequently-accessed variable
kiszk Dec 13, 2017
49119a9
exclude mutable state from argument list for ExpressionCodegn
kiszk Dec 13, 2017
24f49c5
fix test failures
kiszk Dec 14, 2017
15e967e
address review comments
kiszk Dec 14, 2017
d6c1a97
address review comments
kiszk Dec 14, 2017
a9d40e9
address review comments
kiszk Dec 14, 2017
31914c0
address review comments
kiszk Dec 15, 2017
0e45c19
address review comments
kiszk Dec 19, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
eliminate initialization with default value
  • Loading branch information
kiszk committed Dec 14, 2017
commit 870d106da7dabbade7811cc494dfabb82b1d0259
Original file line number Diff line number Diff line change
Expand Up @@ -321,12 +321,10 @@ case class RegExpReplace(subject: Expression, regexp: Expression, rep: Expressio

val matcher = ctx.freshName("matcher")

val termLastRegex = ctx.addMutableState("UTF8String", "lastRegex", v => s"$v = null;")
val termPattern = ctx.addMutableState(classNamePattern, "pattern", v => s"$v = null;")
val termLastReplacement = ctx.addMutableState("String", "lastReplacement",
v => s"$v = null;")
val termLastReplacementInUTF8 = ctx.addMutableState("UTF8String", "lastReplacementInUTF8",
v => s"$v = null;")
val termLastRegex = ctx.addMutableState("UTF8String", "lastRegex")
val termPattern = ctx.addMutableState(classNamePattern, "pattern")
val termLastReplacement = ctx.addMutableState("String", "lastReplacement")
val termLastReplacementInUTF8 = ctx.addMutableState("UTF8String", "lastReplacementInUTF8")
val termResult = ctx.addMutableState(classNameStringBuffer, "result",
v => s"$v = new $classNameStringBuffer();")

Expand Down Expand Up @@ -414,8 +412,8 @@ case class RegExpExtract(subject: Expression, regexp: Expression, idx: Expressio
val matcher = ctx.freshName("matcher")
val matchResult = ctx.freshName("matchResult")

val termLastRegex = ctx.addMutableState("UTF8String", "lastRegex", v => s"$v = null;")
val termPattern = ctx.addMutableState(classNamePattern, "pattern", v => s"$v = null;")
val termLastRegex = ctx.addMutableState("UTF8String", "lastRegex")
val termPattern = ctx.addMutableState(classNamePattern, "pattern")

val setEvNotNull = if (nullable) {
s"${ev.isNull} = false;"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -534,9 +534,9 @@ case class StringTranslate(srcExpr: Expression, matchingExpr: Expression, replac
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val classNameDict = classOf[JMap[Character, Character]].getCanonicalName

val termLastMatching = ctx.addMutableState("UTF8String", "lastMatching", v => s"$v = null;")
val termLastReplace = ctx.addMutableState("UTF8String", "lastReplace", v => s"$v = null;")
val termDict = ctx.addMutableState(classNameDict, "dict", v => s"$v = null;")
val termLastMatching = ctx.addMutableState("UTF8String", "lastMatching")
val termLastReplace = ctx.addMutableState("UTF8String", "lastReplace")
val termDict = ctx.addMutableState(classNameDict, "dict")

nullSafeCodeGen(ctx, ev, (src, matching, replace) => {
val check = if (matchingExpr.foldable && replaceExpr.foldable) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,17 +75,17 @@ private[sql] trait ColumnarBatchScan extends CodegenSupport {
// metrics
val numOutputRows = metricTerm(ctx, "numOutputRows")
val scanTimeMetric = metricTerm(ctx, "scanTime")
val scanTimeTotalNs = ctx.addMutableState(ctx.JAVA_LONG, "scanTime", v => s"$v = 0;")
val scanTimeTotalNs = ctx.addMutableState(ctx.JAVA_LONG, "scanTime")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see that the initialization is not needed since 0 is the default value, but maybe we can leave it for clarity?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see. Let us leave default value as a comment for clarity.


val columnarBatchClz = classOf[ColumnarBatch].getName
val batch = ctx.addMutableState(columnarBatchClz, "batch", v => s"$v = null;")
val batch = ctx.addMutableState(columnarBatchClz, "batch")

val idx = ctx.addMutableState(ctx.JAVA_INT, "batchIdx", v => s"$v = 0;")
val idx = ctx.addMutableState(ctx.JAVA_INT, "batchIdx")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

val columnVectorClzs = vectorTypes.getOrElse(
Seq.fill(output.indices.size)(classOf[ColumnVector].getName))
val (colVars, columnAssigns) = columnVectorClzs.zipWithIndex.map {
case (columnVectorClz, i) =>
val name = ctx.addMutableState(columnVectorClz, s"colInstance$i", v => s"$v = null;")
val name = ctx.addMutableState(columnVectorClz, s"colInstance$i")
(name, s"$name = ($columnVectorClz) $batch.column($i);")
}.unzip

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ case class HashAggregateExec(
private var bufVars: Seq[ExprCode] = _

private def doProduceWithoutKeys(ctx: CodegenContext): String = {
val initAgg = ctx.addMutableState(ctx.JAVA_BOOLEAN, "initAgg", v => s"$v = false;")
val initAgg = ctx.addMutableState(ctx.JAVA_BOOLEAN, "initAgg")
// The generated function doesn't have input row in the code context.
ctx.INPUT_ROW = null

Expand Down Expand Up @@ -565,7 +565,7 @@ case class HashAggregateExec(
}

private def doProduceWithKeys(ctx: CodegenContext): String = {
val initAgg = ctx.addMutableState(ctx.JAVA_BOOLEAN, "initAgg", v => s"$v = false;")
val initAgg = ctx.addMutableState(ctx.JAVA_BOOLEAN, "initAgg")
if (sqlContext.conf.enableTwoLevelAggMap) {
enableTwoLevelHashMap(ctx)
} else {
Expand Down Expand Up @@ -750,7 +750,7 @@ case class HashAggregateExec(

val (checkFallbackForGeneratedHashMap, checkFallbackForBytesToBytesMap, resetCounter,
incCounter) = if (testFallbackStartsAt.isDefined) {
val countTerm = ctx.addMutableState(ctx.JAVA_INT, "fallbackCounter", v => s"$v = 0;")
val countTerm = ctx.addMutableState(ctx.JAVA_INT, "fallbackCounter")
(s"$countTerm < ${testFallbackStartsAt.get._1}",
s"$countTerm < ${testFallbackStartsAt.get._2}", s"$countTerm = 0;", s"$countTerm += 1;")
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,7 @@ case class RangeExec(range: org.apache.spark.sql.catalyst.plans.logical.Range)
protected override def doProduce(ctx: CodegenContext): String = {
val numOutput = metricTerm(ctx, "numOutputRows")

val initTerm = ctx.addMutableState(ctx.JAVA_BOOLEAN, "initRange", v => s"$v = false;")
val initTerm = ctx.addMutableState(ctx.JAVA_BOOLEAN, "initRange")
val number = ctx.addMutableState(ctx.JAVA_LONG, "number", v => s"$v = 0L;")

val value = ctx.freshName("value")
Expand All @@ -383,10 +383,10 @@ case class RangeExec(range: org.apache.spark.sql.catalyst.plans.logical.Range)
// the metrics.

// Once number == batchEnd, it's time to progress to the next batch.
val batchEnd = ctx.addMutableState(ctx.JAVA_LONG, "batchEnd", v => s"$v = 0;")
val batchEnd = ctx.addMutableState(ctx.JAVA_LONG, "batchEnd")

// How many values should still be generated by this range operator.
val numElementsTodo = ctx.addMutableState(ctx.JAVA_LONG, "numElementsTodo", v => s"$v = 0L;")
val numElementsTodo = ctx.addMutableState(ctx.JAVA_LONG, "numElementsTodo")

// How many values should be generated in the next batch.
val nextBatchTodo = ctx.freshName("nextBatchTodo")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ case class SortMergeJoinExec(
private def genScanner(ctx: CodegenContext): (String, String) = {
// Create class member for next row from both sides.
val leftRow = ctx.addMutableState("InternalRow", "leftRow")
val rightRow = ctx.addMutableState("InternalRow", "rightRow", v => s"$v = null;")
val rightRow = ctx.addMutableState("InternalRow", "rightRow")

// Create variables for join keys from both sides.
val leftKeyVars = createJoinKey(ctx, leftRow, leftKeys, left.output)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,15 +71,15 @@ trait BaseLimitExec extends UnaryExecNode with CodegenSupport {
}

override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
val stopEarly = ctx.addMutableState(ctx.JAVA_BOOLEAN, "stopEarly", v => s"$v = false;")
val stopEarly = ctx.addMutableState(ctx.JAVA_BOOLEAN, "stopEarly")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto


ctx.addNewFunction("stopEarly", s"""
@Override
protected boolean stopEarly() {
return $stopEarly;
}
""", inlineToOuterClass = true)
val countTerm = ctx.addMutableState(ctx.JAVA_INT, "count", v => s"$v = 0;")
val countTerm = ctx.addMutableState(ctx.JAVA_INT, "count")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto

s"""
| if ($countTerm < $limit) {
| $countTerm += 1;
Expand Down