Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ private[sql] object KeywordNormalizer {
private[sql] abstract class AbstractSparkSQLParser
extends StandardTokenParsers with PackratParsers {

def apply(input: String): LogicalPlan = {
def parse(input: String): LogicalPlan = {
// Initialize the Keywords.
lexical.initialize(reservedWords)
phrase(start)(new lexical.Scanner(input)) match {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -296,7 +296,7 @@ package object dsl {
InsertIntoTable(
analysis.UnresolvedRelation(Seq(tableName)), Map.empty, logicalPlan, overwrite, false)

def analyze: LogicalPlan = EliminateSubQueries(analysis.SimpleAnalyzer(logicalPlan))
def analyze: LogicalPlan = EliminateSubQueries(analysis.SimpleAnalyzer.execute(logicalPlan))
}

object plans { // scalastyle:ignore
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,11 +98,11 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
})

/** Generates the requested evaluator binding the given expression(s) to the inputSchema. */
def apply(expressions: InType, inputSchema: Seq[Attribute]): OutType =
apply(bind(expressions, inputSchema))
def generate(expressions: InType, inputSchema: Seq[Attribute]): OutType =
generate(bind(expressions, inputSchema))

/** Generates the requested evaluator given already bound expression(s). */
def apply(expressions: InType): OutType = cache.get(canonicalize(expressions))
def generate(expressions: InType): OutType = cache.get(canonicalize(expressions))

/**
* Returns a term name that is unique within this instance of a `CodeGenerator`.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], () => Mu
val mutableRowName = newTermName("mutableRow")

protected def canonicalize(in: Seq[Expression]): Seq[Expression] =
in.map(ExpressionCanonicalizer(_))
in.map(ExpressionCanonicalizer.execute)

protected def bind(in: Seq[Expression], inputSchema: Seq[Attribute]): Seq[Expression] =
in.map(BindReferences.bindReference(_, inputSchema))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[Row]] wit
import scala.reflect.runtime.universe._

protected def canonicalize(in: Seq[SortOrder]): Seq[SortOrder] =
in.map(ExpressionCanonicalizer(_).asInstanceOf[SortOrder])
in.map(ExpressionCanonicalizer.execute(_).asInstanceOf[SortOrder])

protected def bind(in: Seq[SortOrder], inputSchema: Seq[Attribute]): Seq[SortOrder] =
in.map(BindReferences.bindReference(_, inputSchema))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ object GeneratePredicate extends CodeGenerator[Expression, (Row) => Boolean] {
import scala.reflect.runtime.{universe => ru}
import scala.reflect.runtime.universe._

protected def canonicalize(in: Expression): Expression = ExpressionCanonicalizer(in)
protected def canonicalize(in: Expression): Expression = ExpressionCanonicalizer.execute(in)

protected def bind(in: Expression, inputSchema: Seq[Attribute]): Expression =
BindReferences.bindReference(in, inputSchema)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
import scala.reflect.runtime.universe._

protected def canonicalize(in: Seq[Expression]): Seq[Expression] =
in.map(ExpressionCanonicalizer(_))
in.map(ExpressionCanonicalizer.execute)

protected def bind(in: Seq[Expression], inputSchema: Seq[Attribute]): Seq[Expression] =
in.map(BindReferences.bindReference(_, inputSchema))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.types.{DataType, BinaryType, BooleanType, NativeType}

object InterpretedPredicate {
def apply(expression: Expression, inputSchema: Seq[Attribute]): (Row => Boolean) =
apply(BindReferences.bindReference(expression, inputSchema))
def create(expression: Expression, inputSchema: Seq[Attribute]): (Row => Boolean) =
create(BindReferences.bindReference(expression, inputSchema))

def apply(expression: Expression): (Row => Boolean) = {
def create(expression: Expression): (Row => Boolean) = {
(r: Row) => expression.eval(r).asInstanceOf[Boolean]
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ abstract class RuleExecutor[TreeType <: TreeNode[_]] extends Logging {
* Executes the batches of rules defined by the subclass. The batches are executed serially
* using the defined execution strategy. Within each batch, rules are also executed serially.
*/
def apply(plan: TreeType): TreeType = {
def execute(plan: TreeType): TreeType = {
var curPlan = plan

batches.foreach { batch =>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,14 @@ class SqlParserSuite extends FunSuite {

test("test long keyword") {
val parser = new SuperLongKeywordTestParser
assert(TestCommand("NotRealCommand") === parser("ThisIsASuperLongKeyWordTest NotRealCommand"))
assert(TestCommand("NotRealCommand") ===
parser.parse("ThisIsASuperLongKeyWordTest NotRealCommand"))
}

test("test case insensitive") {
val parser = new CaseInsensitiveTestParser
assert(TestCommand("NotRealCommand") === parser("EXECUTE NotRealCommand"))
assert(TestCommand("NotRealCommand") === parser("execute NotRealCommand"))
assert(TestCommand("NotRealCommand") === parser("exEcute NotRealCommand"))
assert(TestCommand("NotRealCommand") === parser.parse("EXECUTE NotRealCommand"))
assert(TestCommand("NotRealCommand") === parser.parse("execute NotRealCommand"))
assert(TestCommand("NotRealCommand") === parser.parse("exEcute NotRealCommand"))
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,10 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {


def caseSensitiveAnalyze(plan: LogicalPlan): Unit =
caseSensitiveAnalyzer.checkAnalysis(caseSensitiveAnalyzer(plan))
caseSensitiveAnalyzer.checkAnalysis(caseSensitiveAnalyzer.execute(plan))

def caseInsensitiveAnalyze(plan: LogicalPlan): Unit =
caseInsensitiveAnalyzer.checkAnalysis(caseInsensitiveAnalyzer(plan))
caseInsensitiveAnalyzer.checkAnalysis(caseInsensitiveAnalyzer.execute(plan))

val testRelation = LocalRelation(AttributeReference("a", IntegerType, nullable = true)())
val testRelation2 = LocalRelation(
Expand Down Expand Up @@ -82,7 +82,7 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
a.select(UnresolvedStar(None)).select('a).unionAll(b.select(UnresolvedStar(None)))
}

assert(caseInsensitiveAnalyzer(plan).resolved)
assert(caseInsensitiveAnalyzer.execute(plan).resolved)
}

test("check project's resolved") {
Expand All @@ -98,11 +98,11 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {

test("analyze project") {
assert(
caseSensitiveAnalyzer(Project(Seq(UnresolvedAttribute("a")), testRelation)) ===
caseSensitiveAnalyzer.execute(Project(Seq(UnresolvedAttribute("a")), testRelation)) ===
Project(testRelation.output, testRelation))

assert(
caseSensitiveAnalyzer(
caseSensitiveAnalyzer.execute(
Project(Seq(UnresolvedAttribute("TbL.a")),
UnresolvedRelation(Seq("TaBlE"), Some("TbL")))) ===
Project(testRelation.output, testRelation))
Expand All @@ -115,13 +115,13 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
assert(e.getMessage().toLowerCase.contains("cannot resolve"))

assert(
caseInsensitiveAnalyzer(
caseInsensitiveAnalyzer.execute(
Project(Seq(UnresolvedAttribute("TbL.a")),
UnresolvedRelation(Seq("TaBlE"), Some("TbL")))) ===
Project(testRelation.output, testRelation))

assert(
caseInsensitiveAnalyzer(
caseInsensitiveAnalyzer.execute(
Project(Seq(UnresolvedAttribute("tBl.a")),
UnresolvedRelation(Seq("TaBlE"), Some("TbL")))) ===
Project(testRelation.output, testRelation))
Expand All @@ -134,13 +134,13 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
assert(e.getMessage == "Table Not Found: tAbLe")

assert(
caseSensitiveAnalyzer(UnresolvedRelation(Seq("TaBlE"), None)) === testRelation)
caseSensitiveAnalyzer.execute(UnresolvedRelation(Seq("TaBlE"), None)) === testRelation)

assert(
caseInsensitiveAnalyzer(UnresolvedRelation(Seq("tAbLe"), None)) === testRelation)
caseInsensitiveAnalyzer.execute(UnresolvedRelation(Seq("tAbLe"), None)) === testRelation)

assert(
caseInsensitiveAnalyzer(UnresolvedRelation(Seq("TaBlE"), None)) === testRelation)
caseInsensitiveAnalyzer.execute(UnresolvedRelation(Seq("TaBlE"), None)) === testRelation)
}

def errorTest(
Expand Down Expand Up @@ -219,7 +219,7 @@ class AnalysisSuite extends FunSuite with BeforeAndAfter {
AttributeReference("d", DecimalType.Unlimited)(),
AttributeReference("e", ShortType)())

val plan = caseInsensitiveAnalyzer(
val plan = caseInsensitiveAnalyzer.execute(
testRelation2.select(
'a / Literal(2) as 'div1,
'a / 'b as 'div2,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,12 @@ class DecimalPrecisionSuite extends FunSuite with BeforeAndAfter {

private def checkType(expression: Expression, expectedType: DataType): Unit = {
val plan = Project(Seq(Alias(expression, "c")()), relation)
assert(analyzer(plan).schema.fields(0).dataType === expectedType)
assert(analyzer.execute(plan).schema.fields(0).dataType === expectedType)
}

private def checkComparison(expression: Expression, expectedType: DataType): Unit = {
val plan = Project(Alias(expression, "c")() :: Nil, relation)
val comparison = analyzer(plan).collect {
val comparison = analyzer.execute(plan).collect {
case Project(Alias(e: BinaryComparison, _) :: Nil, _) => e
}.head
assert(comparison.left.dataType === expectedType)
Expand All @@ -64,7 +64,7 @@ class DecimalPrecisionSuite extends FunSuite with BeforeAndAfter {
val plan =
Union(Project(Seq(Alias(left, "l")()), relation),
Project(Seq(Alias(right, "r")()), relation))
val (l, r) = analyzer(plan).collect {
val (l, r) = analyzer.execute(plan).collect {
case Union(left, right) => (left.output.head, right.output.head)
}.head
assert(l.dataType === expectedType)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ class GeneratedEvaluationSuite extends ExpressionEvaluationSuite {
expected: Any,
inputRow: Row = EmptyRow): Unit = {
val plan = try {
GenerateMutableProjection(Alias(expression, s"Optimized($expression)")() :: Nil)()
GenerateMutableProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil)()
} catch {
case e: Throwable =>
val evaluated = GenerateProjection.expressionEvaluator(expression)
Expand All @@ -56,10 +56,10 @@ class GeneratedEvaluationSuite extends ExpressionEvaluationSuite {

val futures = (1 to 20).map { _ =>
future {
GeneratePredicate(EqualTo(Literal(1), Literal(1)))
GenerateProjection(EqualTo(Literal(1), Literal(1)) :: Nil)
GenerateMutableProjection(EqualTo(Literal(1), Literal(1)) :: Nil)
GenerateOrdering(Add(Literal(1), Literal(1)).asc :: Nil)
GeneratePredicate.generate(EqualTo(Literal(1), Literal(1)))
GenerateProjection.generate(EqualTo(Literal(1), Literal(1)) :: Nil)
GenerateMutableProjection.generate(EqualTo(Literal(1), Literal(1)) :: Nil)
GenerateOrdering.generate(Add(Literal(1), Literal(1)).asc :: Nil)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
*/
class GeneratedMutableEvaluationSuite extends ExpressionEvaluationSuite {
override def checkEvaluation(
expression: Expression,
expected: Any,
inputRow: Row = EmptyRow): Unit = {
expression: Expression,
expected: Any,
inputRow: Row = EmptyRow): Unit = {
lazy val evaluated = GenerateProjection.expressionEvaluator(expression)

val plan = try {
GenerateProjection(Alias(expression, s"Optimized($expression)")() :: Nil)
GenerateProjection.generate(Alias(expression, s"Optimized($expression)")() :: Nil)
} catch {
case e: Throwable =>
fail(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ class BooleanSimplificationSuite extends PlanTest with PredicateHelper {

def checkCondition(input: Expression, expected: Expression): Unit = {
val plan = testRelation.where(input).analyze
val actual = Optimize(plan).expressions.head
val actual = Optimize.execute(plan).expressions.head
compareConditions(actual, expected)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ class CombiningLimitsSuite extends PlanTest {
.limit(10)
.limit(5)

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer =
testRelation
.select('a)
Expand All @@ -61,7 +61,7 @@ class CombiningLimitsSuite extends PlanTest {
.limit(7)
.limit(5)

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer =
testRelation
.select('a)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class ConstantFoldingSuite extends PlanTest {
.subquery('y)
.select('a)

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)
val correctAnswer =
testRelation
.select('a.attr)
Expand All @@ -74,7 +74,7 @@ class ConstantFoldingSuite extends PlanTest {
Literal(2) * Literal(3) - Literal(6) / (Literal(4) - Literal(2))
)(Literal(9) / Literal(3) as Symbol("9/3"))

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)

val correctAnswer =
testRelation
Expand All @@ -99,7 +99,7 @@ class ConstantFoldingSuite extends PlanTest {
Literal(2) * 'a + Literal(4) as Symbol("c3"),
'a * (Literal(3) + Literal(4)) as Symbol("c4"))

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)

val correctAnswer =
testRelation
Expand Down Expand Up @@ -127,7 +127,7 @@ class ConstantFoldingSuite extends PlanTest {
(Literal(1) === Literal(1) || 'b > 1) &&
(Literal(1) === Literal(2) || 'b < 10)))

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)

val correctAnswer =
testRelation
Expand All @@ -144,7 +144,7 @@ class ConstantFoldingSuite extends PlanTest {
Cast(Literal("2"), IntegerType) + Literal(3) + 'a as Symbol("c1"),
Coalesce(Seq(Cast(Literal("abc"), IntegerType), Literal(3))) as Symbol("c2"))

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)

val correctAnswer =
testRelation
Expand All @@ -163,7 +163,7 @@ class ConstantFoldingSuite extends PlanTest {
Rand + Literal(1) as Symbol("c1"),
Sum('a) as Symbol("c2"))

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)

val correctAnswer =
testRelation
Expand Down Expand Up @@ -210,7 +210,7 @@ class ConstantFoldingSuite extends PlanTest {
Contains("abc", Literal.create(null, StringType)) as 'c20
)

val optimized = Optimize(originalQuery.analyze)
val optimized = Optimize.execute(originalQuery.analyze)

val correctAnswer =
testRelation
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ class ConvertToLocalRelationSuite extends PlanTest {
UnresolvedAttribute("a").as("a1"),
(UnresolvedAttribute("b") + 1).as("b1"))

val optimized = Optimize(projectOnLocal.analyze)
val optimized = Optimize.execute(projectOnLocal.analyze)

comparePlans(optimized, correctAnswer)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class ExpressionOptimizationSuite extends ExpressionEvaluationSuite {
expected: Any,
inputRow: Row = EmptyRow): Unit = {
val plan = Project(Alias(expression, s"Optimized($expression)")() :: Nil, OneRowRelation)
val optimizedPlan = DefaultOptimizer(plan)
val optimizedPlan = DefaultOptimizer.execute(plan)
super.checkEvaluation(optimizedPlan.expressions.head, expected, inputRow)
}
}
Loading