From d715fd577d9028b5e9910448d23ee560e6e42bdb Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 15 Jul 2015 17:20:02 +0800 Subject: [PATCH 1/4] Split projectionCode if it is too large for JVM. --- .../codegen/GenerateMutableProjection.scala | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala index addb8023d9c0..7667b7412965 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala @@ -36,7 +36,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu protected def create(expressions: Seq[Expression]): (() => MutableProjection) = { val ctx = newCodeGenContext() - val projectionCode = expressions.zipWithIndex.map { case (e, i) => + val projectionCodes = expressions.zipWithIndex.map { case (e, i) => val evaluationCode = e.gen(ctx) evaluationCode.code + s""" @@ -45,7 +45,28 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu else ${ctx.setColumn("mutableRow", e.dataType, i, evaluationCode.primitive)}; """ - }.mkString("\n") + } + + val projectionCodeSegments = projectionCodes.grouped(50).toSeq.map(_.mkString("\n")) + + val (projectionCode, projectionFuncs) = if (projectionCodeSegments.length == 1) { + (projectionCodeSegments(0), "") + } else { + val pCode = (0 until projectionCodeSegments.length).map { i => + s"projectSeg$i(_i)" + }.mkString("\n") + + val pFuncs = (0 until projectionCodeSegments.length).map { i => + s""" + public void projectSeg$i(Object _i) { + InternalRow i = (InternalRow) _i; + ${projectionCodeSegments(i)} + } + """ + }.mkString("\n") + (pCode, pFuncs) + } + val code = s""" public Object generate($exprType[] expr) { return new SpecificProjection(expr); @@ -71,6 +92,8 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu return (InternalRow) mutableRow; } + $projectionFuncs + public Object apply(Object _i) { InternalRow i = (InternalRow) _i; $projectionCode From 8775359504677f9f4d4d86adcbdfc28f1f7b4e06 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Wed, 15 Jul 2015 22:20:46 +0800 Subject: [PATCH 2/4] Add semicolon. --- .../expressions/codegen/GenerateMutableProjection.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala index 7667b7412965..7fbb97f87891 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala @@ -53,7 +53,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu (projectionCodeSegments(0), "") } else { val pCode = (0 until projectionCodeSegments.length).map { i => - s"projectSeg$i(_i)" + s"projectSeg$i(_i);" }.mkString("\n") val pFuncs = (0 until projectionCodeSegments.length).map { i => From 12d3794b009a90d21de9a1d52d4f3ea9503f2b58 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 16 Jul 2015 12:44:39 +0800 Subject: [PATCH 3/4] For comment. --- .../expressions/codegen/GenerateMutableProjection.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala index f15f715e3075..b2765b324c83 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala @@ -58,8 +58,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu val pFuncs = (0 until projectionCodeSegments.length).map { i => s""" - public void projectSeg$i(Object _i) { - InternalRow i = (InternalRow) _i; + private final void projectSeg$i(InternalRow i) { ${projectionCodeSegments(i)} } """ From b8e274e50e97303c3dfc8de668f7005c80a069e2 Mon Sep 17 00:00:00 2001 From: Liang-Chi Hsieh Date: Thu, 16 Jul 2015 14:28:22 +0800 Subject: [PATCH 4/4] Fix it. --- .../expressions/codegen/GenerateMutableProjection.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala index b2765b324c83..de140e51c9c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala @@ -53,7 +53,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu (projectionCodeSegments(0), "") } else { val pCode = (0 until projectionCodeSegments.length).map { i => - s"projectSeg$i(_i);" + s"projectSeg$i(i);" }.mkString("\n") val pFuncs = (0 until projectionCodeSegments.length).map { i =>