Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge remote-tracking branch 'upstream/master' into SPARK-22856
  • Loading branch information
viirya committed Mar 5, 2018
commit c8c70a9107791cdfccb48dbc729aa8b6a6e9c7ee
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.errors.attachTree
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, FalseLiteral}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral}
import org.apache.spark.sql.types._

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ abstract class Expression extends TreeNode[Expression] {
val isNull = ctx.freshName("isNull")
val value = ctx.freshName("value")
val eval = doGenCode(ctx, ExprCode("",
VariableValue(isNull, ctx.JAVA_BOOLEAN),
VariableValue(value, ctx.javaType(dataType))))
VariableValue(isNull, CodeGenerator.JAVA_BOOLEAN),
VariableValue(value, CodeGenerator.javaType(dataType))))
reduceCodeSize(ctx, eval)
if (eval.code.nonEmpty) {
// Add `this` in the comment.
Expand All @@ -121,9 +121,9 @@ abstract class Expression extends TreeNode[Expression] {
// TODO: support whole stage codegen too
if (eval.code.trim.length > 1024 && ctx.INPUT_ROW != null && ctx.currentVars == null) {
val setIsNull = if (!eval.isNull.isInstanceOf[LiteralValue]) {
val globalIsNull = ctx.addMutableState(ctx.JAVA_BOOLEAN, "globalIsNull")
val globalIsNull = ctx.addMutableState(CodeGenerator.JAVA_BOOLEAN, "globalIsNull")
val localIsNull = eval.isNull
eval.isNull = GlobalValue(globalIsNull, ctx.JAVA_BOOLEAN)
eval.isNull = GlobalValue(globalIsNull, CodeGenerator.JAVA_BOOLEAN)
s"$globalIsNull = $localIsNull;"
} else {
""
Expand Down Expand Up @@ -420,7 +420,7 @@ abstract class UnaryExpression extends Expression {
ev.copy(code = s"""
boolean ${ev.isNull} = false;
${childGen.code}
${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)};
${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
$resultCode""", isNull = FalseLiteral)
}
}
Expand Down Expand Up @@ -520,7 +520,7 @@ abstract class BinaryExpression extends Expression {
boolean ${ev.isNull} = false;
${leftGen.code}
${rightGen.code}
${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)};
${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
$resultCode""", isNull = FalseLiteral)
}
}
Expand Down Expand Up @@ -664,7 +664,7 @@ abstract class TernaryExpression extends Expression {
${leftGen.code}
${midGen.code}
${rightGen.code}
${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)};
${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
$resultCode""", isNull = FalseLiteral)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, FalseLiteral}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral}
import org.apache.spark.sql.types.{DataType, LongType}

/**
Expand Down Expand Up @@ -72,7 +72,7 @@ case class MonotonicallyIncreasingID() extends LeafExpression with Nondeterminis
ctx.addPartitionInitializationStatement(s"$partitionMaskTerm = ((long) partitionIndex) << 33;")

ev.copy(code = s"""
final ${ctx.javaType(dataType)} ${ev.value} = $partitionMaskTerm + $countTerm;
final ${CodeGenerator.javaType(dataType)} ${ev.value} = $partitionMaskTerm + $countTerm;
$countTerm++;""", isNull = FalseLiteral)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
package org.apache.spark.sql.catalyst.expressions

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, FalseLiteral}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral}
import org.apache.spark.sql.types.{DataType, IntegerType}

/**
Expand Down Expand Up @@ -46,7 +46,7 @@ case class SparkPartitionID() extends LeafExpression with Nondeterministic {
val idTerm = "partitionId"
ctx.addImmutableStateIfNotExists(CodeGenerator.JAVA_INT, idTerm)
ctx.addPartitionInitializationStatement(s"$idTerm = partitionIndex;")
ev.copy(code = s"final ${ctx.javaType(dataType)} ${ev.value} = $idTerm;",
ev.copy(code = s"final ${CodeGenerator.javaType(dataType)} ${ev.value} = $idTerm;",
isNull = FalseLiteral)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -601,7 +601,8 @@ case class Least(children: Seq[Expression]) extends Expression {

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val evalChildren = children.map(_.genCode(ctx))
ev.isNull = GlobalValue(ctx.addMutableState(ctx.JAVA_BOOLEAN, ev.isNull), ctx.JAVA_BOOLEAN)
ev.isNull = GlobalValue(ctx.addMutableState(CodeGenerator.JAVA_BOOLEAN, ev.isNull),
CodeGenerator.JAVA_BOOLEAN)
val evals = evalChildren.map(eval =>
s"""
|${eval.code}
Expand Down Expand Up @@ -680,7 +681,8 @@ case class Greatest(children: Seq[Expression]) extends Expression {

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val evalChildren = children.map(_.genCode(ctx))
ev.isNull = GlobalValue(ctx.addMutableState(ctx.JAVA_BOOLEAN, ev.isNull), ctx.JAVA_BOOLEAN)
ev.isNull = GlobalValue(ctx.addMutableState(CodeGenerator.JAVA_BOOLEAN, ev.isNull),
CodeGenerator.JAVA_BOOLEAN)
val evals = evalChildren.map(eval =>
s"""
|${eval.code}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ import org.apache.spark.util.{ParentClassLoader, Utils}
case class ExprCode(var code: String, var isNull: ExprValue, var value: ExprValue)

object ExprCode {
def forNullValue(dataType: DataType): ExprCode = {
val defaultValueLiteral = CodeGenerator.defaultValue(dataType, typedNull = true)
ExprCode(code = "", isNull = TrueLiteral,
value = LiteralValue(defaultValueLiteral, CodeGenerator.javaType(dataType)))
}

def forNonNullValue(value: ExprValue): ExprCode = {
ExprCode(code = "", isNull = FalseLiteral, value = value)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ trait CodegenFallback extends Expression {
ev.copy(code = s"""
$placeHolder
Object $objectTerm = ((Expression) references[$idx]).eval($input);
${ctx.javaType(this.dataType)} ${ev.value} = (${ctx.boxedType(this.dataType)}) $objectTerm;
$javaType ${ev.value} = (${CodeGenerator.boxedType(this.dataType)}) $objectTerm;
""", isNull = FalseLiteral)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ abstract class ExprValue {
// For such cases, we may need to pass the evaluation as parameter.
val canDirectAccess: Boolean

def isPrimitive(ctx: CodegenContext): Boolean = ctx.isPrimitiveType(javaType)
def isPrimitive: Boolean = CodeGenerator.isPrimitiveType(javaType)
}

object ExprValue {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], MutableP
|${ev.code}
|$isNull = ${ev.isNull};
|$value = ${ev.value};
""".stripMargin, GlobalValue(isNull, ctx.JAVA_BOOLEAN), value, i)
""".stripMargin, GlobalValue(isNull, CodeGenerator.JAVA_BOOLEAN), value, i)
} else {
(s"""
|${ev.code}
Expand All @@ -83,8 +83,8 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], MutableP

val updates = validExpr.zip(projectionCodes).map {
case (e, (_, isNull, value, i)) =>
val ev = ExprCode("", isNull, GlobalValue(value, ctx.javaType(e.dataType)))
ctx.updateColumn("mutableRow", e.dataType, i, ev, e.nullable)
val ev = ExprCode("", isNull, GlobalValue(value, CodeGenerator.javaType(e.dataType)))
CodeGenerator.updateColumn("mutableRow", e.dataType, i, ev, e.nullable)
}

val allProjections = ctx.splitExpressionsWithCurrentInputs(projectionCodes.map(_._1))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,9 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection]
val rowClass = classOf[GenericInternalRow].getName

val fieldWriters = schema.map(_.dataType).zipWithIndex.map { case (dt, i) =>
val converter = convertToSafe(ctx, StatementValue(ctx.getValue(tmpInput, dt, i.toString),
ctx.javaType(dt)), dt)
val converter = convertToSafe(ctx,
StatementValue(CodeGenerator.getValue(tmpInput, dt, i.toString),
CodeGenerator.javaType(dt)), dt)
s"""
if (!$tmpInput.isNullAt($i)) {
${converter.code}
Expand Down Expand Up @@ -90,9 +91,9 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection]
val index = ctx.freshName("index")
val arrayClass = classOf[GenericArrayData].getName

val elementConverter = convertToSafe(
ctx, StatementValue(ctx.getValue(tmpInput, elementType, index), ctx.javaType(elementType)),
elementType)
val elementConverter = convertToSafe(ctx,
StatementValue(CodeGenerator.getValue(tmpInput, elementType, index),
CodeGenerator.javaType(elementType)), elementType)
val code = s"""
final ArrayData $tmpInput = $input;
final int $numElements = $tmpInput.numElements();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
// Puts `input` in a local variable to avoid to re-evaluate it if it's a statement.
val tmpInput = ctx.freshName("tmpInput")
val fieldEvals = fieldTypes.zipWithIndex.map { case (dt, i) =>
ExprCode("", StatementValue(s"$tmpInput.isNullAt($i)", ctx.JAVA_BOOLEAN),
StatementValue(ctx.getValue(tmpInput, dt, i.toString), ctx.javaType(dt)))
ExprCode("", StatementValue(s"$tmpInput.isNullAt($i)", CodeGenerator.JAVA_BOOLEAN),
StatementValue(CodeGenerator.getValue(tmpInput, dt, i.toString),
CodeGenerator.javaType(dt)))
}

s"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType
ev.copy(code = s"""
boolean ${ev.isNull} = false;
${childGen.code}
${ctx.javaType(dataType)} ${ev.value} = ${childGen.isNull} ? -1 :
${CodeGenerator.javaType(dataType)} ${ev.value} = ${childGen.isNull} ? -1 :
(${childGen.value}).numElements();""", isNull = FalseLiteral)
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ case class CreateArray(children: Seq[Expression]) extends Expression {
GenArrayData.genCodeToCreateArrayData(ctx, et, evals, false)
ev.copy(
code = preprocess + assigns + postprocess,
value = VariableValue(arrayData, ctx.javaType(dataType)),
value = VariableValue(arrayData, CodeGenerator.javaType(dataType)),
isNull = FalseLiteral)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,8 @@ case class CaseWhen(
// It is initialized to `NOT_MATCHED`, and if it's set to `HAS_NULL` or `HAS_NONNULL`,
// We won't go on anymore on the computation.
val resultState = ctx.freshName("caseWhenResultState")
ev.value = GlobalValue(ctx.addMutableState(ctx.javaType(dataType), ev.value),
ctx.javaType(dataType))
ev.value = GlobalValue(ctx.addMutableState(CodeGenerator.javaType(dataType), ev.value),
CodeGenerator.javaType(dataType))

// these blocks are meant to be inside a
// do {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -678,8 +678,7 @@ abstract class UnixTime
case StringType if right.foldable =>
val df = classOf[DateFormat].getName
if (formatter == null) {
ExprCode("", TrueLiteral, LiteralValue(ctx.defaultValue(dataType),
ctx.javaType(dataType)))
ExprCode.forNullValue(dataType)
} else {
val formatterName = ctx.addReferenceObj("formatter", formatter, df)
val eval1 = left.genCode(ctx)
Expand Down Expand Up @@ -814,7 +813,8 @@ case class FromUnixTime(sec: Expression, format: Expression, timeZoneId: Option[
val df = classOf[DateFormat].getName
if (format.foldable) {
if (formatter == null) {
ExprCode("", TrueLiteral, LiteralValue("(UTF8String) null", ctx.javaType(dataType)))
ExprCode("", TrueLiteral, LiteralValue("(UTF8String) null",
CodeGenerator.javaType(dataType)))
} else {
val formatterName = ctx.addReferenceObj("formatter", formatter, df)
val t = left.genCode(ctx)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package org.apache.spark.sql.catalyst.expressions

import org.apache.spark.rdd.InputFileBlockHolder
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, FalseLiteral}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral}
import org.apache.spark.sql.types.{DataType, LongType, StringType}
import org.apache.spark.unsafe.types.UTF8String

Expand All @@ -42,7 +42,7 @@ case class InputFileName() extends LeafExpression with Nondeterministic {

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val className = InputFileBlockHolder.getClass.getName.stripSuffix("$")
ev.copy(code = s"final ${ctx.javaType(dataType)} ${ev.value} = " +
ev.copy(code = s"final ${CodeGenerator.javaType(dataType)} ${ev.value} = " +
s"$className.getInputFilePath();", isNull = FalseLiteral)
}
}
Expand All @@ -65,7 +65,7 @@ case class InputFileBlockStart() extends LeafExpression with Nondeterministic {

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val className = InputFileBlockHolder.getClass.getName.stripSuffix("$")
ev.copy(code = s"final ${ctx.javaType(dataType)} ${ev.value} = " +
ev.copy(code = s"final ${CodeGenerator.javaType(dataType)} ${ev.value} = " +
s"$className.getStartOffset();", isNull = FalseLiteral)
}
}
Expand All @@ -88,7 +88,7 @@ case class InputFileBlockLength() extends LeafExpression with Nondeterministic {

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val className = InputFileBlockHolder.getClass.getName.stripSuffix("$")
ev.copy(code = s"final ${ctx.javaType(dataType)} ${ev.value} = " +
ev.copy(code = s"final ${CodeGenerator.javaType(dataType)} ${ev.value} = " +
s"$className.getLength();", isNull = FalseLiteral)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -279,11 +279,7 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression {
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val javaType = CodeGenerator.javaType(dataType)
if (value == null) {
val defaultValueLiteral = ctx.defaultValue(javaType) match {
case "null" => s"(($javaType)null)"
case lit => lit
}
ExprCode(code = "", isNull = TrueLiteral, value = LiteralValue(defaultValueLiteral, javaType))
ExprCode.forNullValue(dataType)
} else {
dataType match {
case BooleanType | IntegerType | DateType =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCa
|if (${eval.isNull} || !${eval.value}) {
| throw new RuntimeException($errMsgField);
|}""".stripMargin, isNull = TrueLiteral,
value = LiteralValue("null", ctx.javaType(dataType)))
value = LiteralValue("null", CodeGenerator.javaType(dataType)))
}

override def sql: String = s"assert_true(${child.sql})"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@ case class Coalesce(children: Seq[Expression]) extends Expression {
}

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
ev.isNull = GlobalValue(ctx.addMutableState(ctx.JAVA_BOOLEAN, ev.isNull), ctx.JAVA_BOOLEAN)
ev.isNull = GlobalValue(ctx.addMutableState(CodeGenerator.JAVA_BOOLEAN, ev.isNull),
CodeGenerator.JAVA_BOOLEAN)

// all the evals are meant to be in a do { ... } while (false); loop
val evals = children.map { e =>
Expand Down Expand Up @@ -234,9 +235,8 @@ case class IsNaN(child: Expression) extends UnaryExpression
case DoubleType | FloatType =>
ev.copy(code = s"""
${eval.code}
${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)};
${ev.value} = !${eval.isNull} && Double.isNaN(${eval.value});""",
isNull = FalseLiteral)
${CodeGenerator.javaType(dataType)} ${ev.value} = ${CodeGenerator.defaultValue(dataType)};
${ev.value} = !${eval.isNull} && Double.isNaN(${eval.value});""", isNull = FalseLiteral)
}
}
}
Expand Down Expand Up @@ -322,9 +322,9 @@ case class IsNull(child: Expression) extends UnaryExpression with Predicate {
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val eval = child.genCode(ctx)
val value = if (eval.isNull.isInstanceOf[LiteralValue]) {
LiteralValue(eval.isNull, ctx.JAVA_BOOLEAN)
LiteralValue(eval.isNull, CodeGenerator.JAVA_BOOLEAN)
} else {
VariableValue(eval.isNull, ctx.JAVA_BOOLEAN)
VariableValue(eval.isNull, CodeGenerator.JAVA_BOOLEAN)
}
ExprCode(code = eval.code, isNull = FalseLiteral, value = value)
}
Expand Down Expand Up @@ -357,7 +357,7 @@ case class IsNotNull(child: Expression) extends UnaryExpression with Predicate {
} else if (eval.isNull == FalseLiteral) {
TrueLiteral
} else {
StatementValue(s"(!(${eval.isNull}))", ctx.javaType(dataType))
StatementValue(s"(!(${eval.isNull}))", CodeGenerator.javaType(dataType))
}
ExprCode(code = eval.code, isNull = FalseLiteral, value = value)
}
Expand Down Expand Up @@ -453,7 +453,7 @@ case class AtLeastNNonNulls(n: Int, children: Seq[Expression]) extends Predicate
|do {
| $codes
|} while (false);
|${ctx.JAVA_BOOLEAN} ${ev.value} = $nonnull >= $n;
|${CodeGenerator.JAVA_BOOLEAN} ${ev.value} = $nonnull >= $n;
""".stripMargin, isNull = FalseLiteral)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ trait InvokeLike extends Expression with NonSQLExpression {
def prepareArguments(ctx: CodegenContext): (String, String, ExprValue) = {

val resultIsNull = if (needNullCheck) {
val resultIsNull = ctx.addMutableState(ctx.JAVA_BOOLEAN, "resultIsNull")
GlobalValue(resultIsNull, ctx.JAVA_BOOLEAN)
val resultIsNull = ctx.addMutableState(CodeGenerator.JAVA_BOOLEAN, "resultIsNull")
GlobalValue(resultIsNull, CodeGenerator.JAVA_BOOLEAN)
} else {
FalseLiteral
}
Expand Down Expand Up @@ -446,11 +446,12 @@ case class LambdaVariable(

override def genCode(ctx: CodegenContext): ExprCode = {
val isNullValue = if (nullable) {
VariableValue(isNull, ctx.JAVA_BOOLEAN)
VariableValue(isNull, CodeGenerator.JAVA_BOOLEAN)
} else {
FalseLiteral
}
ExprCode(code = "", value = VariableValue(value, ctx.javaType(dataType)), isNull = isNullValue)
ExprCode(code = "", value = VariableValue(value, CodeGenerator.javaType(dataType)),
isNull = isNullValue)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import scala.collection.immutable.TreeSet

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, FalseLiteral, GenerateSafeProjection, GenerateUnsafeProjection, Predicate => BasePredicate}
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodeGenerator, ExprCode, FalseLiteral, GenerateSafeProjection, GenerateUnsafeProjection, Predicate => BasePredicate}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.util.TypeUtils
import org.apache.spark.sql.types._
Expand Down
Loading
You are viewing a condensed version of this merge commit. You can view the full changes here.