From c5204566e8175d0d3575080b188054193f1920ef Mon Sep 17 00:00:00 2001 From: Richard Chen Date: Wed, 31 Aug 2022 10:17:14 -0700 Subject: [PATCH 1/2] fix repeat calls to hasNext --- .../spark/sql/execution/datasources/v2/DataSourceRDD.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala index 09c8756ca0189..d51a169cccfd8 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceRDD.scala @@ -111,12 +111,16 @@ private class PartitionIterator[T]( reader: PartitionReader[T], customMetrics: Map[String, SQLMetric]) extends Iterator[T] { private[this] var valuePrepared = false + private[this] var prevHasNext = true private var numRow = 0L override def hasNext: Boolean = { if (!valuePrepared) { - valuePrepared = reader.next() + if (prevHasNext) { + prevHasNext = reader.next() + } + valuePrepared = prevHasNext } valuePrepared } From fb7482a15eafbb8d919b6f327e19997db442eb44 Mon Sep 17 00:00:00 2001 From: Richard Chen Date: Wed, 31 Aug 2022 12:53:08 -0700 Subject: [PATCH 2/2] trigger build