Skip to content
Prev Previous commit
Next Next commit
connect parity
  • Loading branch information
xinrong-meng committed Nov 14, 2024
commit 4be8b042f2ff374db0f995324cabcc6b36ae9a5b
4 changes: 3 additions & 1 deletion python/pyspark/sql/connect/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -611,9 +611,11 @@ def createDataFrame(
if not isinstance(schema, StructType):
schema = from_arrow_schema(data.schema, prefer_timestamp_ntz=prefer_timestamp_ntz)

safe_cast = self._client.get_configs("spark.sql.execution.castArrowTableSafely") == "true"

_table = (
_check_arrow_table_timestamps_localize(data, schema, True, timezone)
.cast(to_arrow_schema(schema, error_on_duplicated_field_names_in_struct=True))
.cast(to_arrow_schema(schema, error_on_duplicated_field_names_in_struct=True), safe=safe_cast)
.rename_columns(schema.names)
)

Expand Down