Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
add selectNested convenience method
  • Loading branch information
gaborbarna committed Feb 13, 2018
commit 62188a605b45737955b6fbabcc5580ae0ec5f011
6 changes: 4 additions & 2 deletions core/src/main/scala/ste/selector.scala
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,9 @@ trait SelectorImplicits {
}

object DFUtils {
implicit class EnhancedDF(df: DataFrame) {
def asNested[A : Encoder : StructTypeSelector]: Dataset[A] = StructTypeSelector[A].select(df, None).as[A]
def selectNested[A](df: DataFrame)(implicit s: StructTypeSelector[A]): DataFrame = s.select(df, None)
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would move this into FlattenedDataFrame .


implicit class FlattenedDataFrame(df: DataFrame) {
def asNested[A : Encoder : StructTypeSelector]: Dataset[A] = selectNested(df).as[A]
}
}
12 changes: 12 additions & 0 deletions core/src/test/scala/ste/StructTypeSelectorSpec.scala
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,18 @@ class StructSelectorSpec extends FlatSpec with Matchers {
import StructSelectorSpec._
val spark = SparkSession.builder().master("local").getOrCreate()

"selectNested" should "return the nested DataFrame" in {
import spark.implicits._
val values = List((1, "a", 2, "b", 3), (4, "c", 5, "d", 6))
val df = values.toDF(StructTypeEncoder[Bar].encode.fields.map(_.name) :_*)
val result = selectNested[Bar](df)
val expected = Array(
Bar(Map("asd" -> Foo(1, "a"), "qwe" -> Foo(2, "b")), 3),
Bar(Map("asd" -> Foo(4, "c"), "qwe" -> Foo(5, "d")), 6)
)
result.as[Bar].collect shouldEqual expected
}

it should "deal with flattened struct" in {
import spark.implicits._
val values = List((1, "a", 2), (3, "b", 4))
Expand Down