Skip to content
Prev Previous commit
Next Next commit
test from arrow table
  • Loading branch information
xinrong-meng committed Nov 14, 2024
commit 71bb3cf48b15aeaf678a929ef48b653546b5b27b
10 changes: 4 additions & 6 deletions python/pyspark/sql/tests/test_arrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -544,13 +544,11 @@ def test_createDataFrame_arrow_safe_cast(self):
Row(id=3, value=-647710720),
]

with self.sql_conf({"spark.sql.execution.castArrowTableSafely": False}):
df = self.spark.createDataFrame(table, schema=schema)
self.assertEqual(df.collect(), expected)
df = self.spark.createDataFrame(table, schema=schema)
self.assertEqual(df.collect(), expected)

with self.sql_conf({"spark.sql.execution.castArrowTableSafely": True}):
with self.assertRaises(Exception):
self.spark.createDataFrame(table, schema=schema)
with self.assertRaises(Exception):
self.spark.createDataFrame(table, schema=schema, verifySchema=True)

def _createDataFrame_toggle(self, data, schema=None):
with self.sql_conf({"spark.sql.execution.arrow.pyspark.enabled": False}):
Expand Down