Skip to content

[SPARK-24722][SQL] pivot() with Column type argument #21699

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 20 commits into from
Closed

[SPARK-24722][SQL] pivot() with Column type argument #21699

Changes from 1 commit
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
889e922
Adding pivot() which takes Column as its argument
MaxGekk Jul 2, 2018
f736ea2
Tests for new function
MaxGekk Jul 2, 2018
5e68226
the since tag is updated
MaxGekk Jul 2, 2018
c82c397
Test for nested columns
MaxGekk Jul 2, 2018
7d0d226
Python test for nested columns
MaxGekk Jul 2, 2018
0fdd11f
Adding ticket number to test's title
MaxGekk Jul 2, 2018
74ddcdd
Making diff shorter
MaxGekk Jul 3, 2018
390d832
Adding a function which accepts Column argument
MaxGekk Jul 3, 2018
d62b7e7
Adding Java-specific functions
MaxGekk Jul 3, 2018
fae4fd2
Tests for column expression
MaxGekk Jul 3, 2018
8ffdc32
Tests for pivot column which refers to multiple other columns
MaxGekk Jul 8, 2018
57c0f64
Test pivoting by constant
MaxGekk Jul 8, 2018
f32a85b
Test for pivoting by an aggregate
MaxGekk Jul 8, 2018
b9996df
Merge remote-tracking branch 'origin/master' into pivot-column
MaxGekk Jul 28, 2018
e76e7ad
Improving comments by referencing to the overloaded methods
MaxGekk Jul 28, 2018
34535a9
Fix expected error message
MaxGekk Jul 29, 2018
e869f85
Merge remote-tracking branch 'origin/master' into pivot-column
MaxGekk Jul 31, 2018
cf55135
Fix expected message
MaxGekk Jul 31, 2018
5da5a2c
Support multiple values
MaxGekk Jul 31, 2018
ca1250b
Revert "Support multiple values"
MaxGekk Aug 3, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Tests for new function
  • Loading branch information
@MaxGekk
MaxGekk committed Jul 2, 2018
commit f736ea2bca27fee37281bcadb333e7b6bbcd6124
33 changes: 24 additions & 9 deletions sql/core/src/test/scala/org/apache/spark/sql/DataFramePivotSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,28 +27,40 @@ class DataFramePivotSuite extends QueryTest with SharedSQLContext {
import testImplicits._

test("pivot courses") {
val expected = Row(2012, 15000.0, 20000.0) :: Row(2013, 48000.0, 30000.0) :: Nil
checkAnswer(
courseSales.groupBy("year").pivot("course", Seq("dotNET", "Java"))
.agg(sum($"earnings")),
Row(2012, 15000.0, 20000.0) :: Row(2013, 48000.0, 30000.0) :: Nil
)
expected)
checkAnswer(
courseSales.groupBy($"year").pivot($"course", Seq("dotNET", "Java"))
.agg(sum($"earnings")),
expected)
}

test("pivot year") {
val expected = Row("dotNET", 15000.0, 48000.0) :: Row("Java", 20000.0, 30000.0) :: Nil
checkAnswer(
courseSales.groupBy("course").pivot("year", Seq(2012, 2013)).agg(sum($"earnings")),
Row("dotNET", 15000.0, 48000.0) :: Row("Java", 20000.0, 30000.0) :: Nil
)
expected)
checkAnswer(
courseSales.groupBy('course).pivot('year, Seq(2012, 2013)).agg(sum('earnings)),
expected)
}

test("pivot courses with multiple aggregations") {
val expected = Row(2012, 15000.0, 7500.0, 20000.0, 20000.0) ::
Row(2013, 48000.0, 48000.0, 30000.0, 30000.0) :: Nil
checkAnswer(
courseSales.groupBy($"year")
.pivot("course", Seq("dotNET", "Java"))
.agg(sum($"earnings"), avg($"earnings")),
Row(2012, 15000.0, 7500.0, 20000.0, 20000.0) ::
Row(2013, 48000.0, 48000.0, 30000.0, 30000.0) :: Nil
)
expected)
checkAnswer(
courseSales.groupBy($"year")
.pivot($"course", Seq("dotNET", "Java"))
.agg(sum($"earnings"), avg($"earnings")),
expected)
}

test("pivot year with string values (cast)") {
Expand Down Expand Up @@ -181,10 +193,13 @@ class DataFramePivotSuite extends QueryTest with SharedSQLContext {
}

test("pivot with datatype not supported by PivotFirst") {
val expected = Row(Seq(1, 1, 1), Seq(2, 2, 2)) :: Nil
checkAnswer(
complexData.groupBy().pivot("b", Seq(true, false)).agg(max("a")),
Row(Seq(1, 1, 1), Seq(2, 2, 2)) :: Nil
)
expected)
checkAnswer(
complexData.groupBy().pivot('b, Seq(true, false)).agg(max('a)),
expected)
}

test("pivot with datatype not supported by PivotFirst 2") {
Expand Down