Skip to content
Next Next commit
reuse config
  • Loading branch information
xinrong-meng committed Nov 14, 2024
commit ad858dcbf0e78bb892ba37d7c2c640a8d863c707
3 changes: 2 additions & 1 deletion python/pyspark/sql/pandas/conversion.py
Original file line number Diff line number Diff line change
Expand Up @@ -785,8 +785,9 @@ def _create_from_arrow_table(
if not isinstance(schema, StructType):
schema = from_arrow_schema(table.schema, prefer_timestamp_ntz=prefer_timestamp_ntz)

safecheck = self._jconf.arrowSafeTypeConversion()
table = _check_arrow_table_timestamps_localize(table, schema, True, timezone).cast(
to_arrow_schema(schema, error_on_duplicated_field_names_in_struct=True)
to_arrow_schema(schema, error_on_duplicated_field_names_in_struct=True), safe=safecheck
)

# Chunk the Arrow Table into RecordBatches
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3353,9 +3353,10 @@ object SQLConf {
buildConf("spark.sql.execution.pandas.convertToArrowArraySafely")
.internal()
.doc("When true, Arrow will perform safe type conversion when converting " +
"Pandas.Series to Arrow array during serialization. Arrow will raise errors " +
"when detecting unsafe type conversion like overflow. When false, disabling Arrow's type " +
"check and do type conversions anyway. This config only works for Arrow 0.11.0+.")
"Pandas.Series to Arrow array during serialization, and when casting Arrow tables to " +
"create DataFrames. Arrow will raise errors when detecting unsafe type conversion like " +
"overflow. When false, disabling Arrow's type check and do type conversions anyway. This " +
"config only works for Arrow 0.11.0+.")
.version("3.0.0")
.booleanConf
.createWithDefault(false)
Expand Down