Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Added more tests
  • Loading branch information
vinjai committed Jun 29, 2024
commit 76f9894a2c5f155fdbeccbb3c7d6e0af11a7518c
3 changes: 0 additions & 3 deletions pyiceberg/io/pyarrow.py
Original file line number Diff line number Diff line change
Expand Up @@ -1820,10 +1820,7 @@ def write_parquet(task: WriteTask) -> DataFile:
file_format=FileFormat.PARQUET,
partition=task.partition_key.partition if task.partition_key else Record(),
file_size_in_bytes=len(fo),
# After this has been fixed:
# https://github.com/apache/iceberg-python/issues/271
sort_order_id=task.sort_order_id,
# sort_order_id=None,
# Just copy these from the table for now
spec_id=table_metadata.default_spec_id,
equality_ids=None,
Expand Down
5 changes: 5 additions & 0 deletions pyiceberg/table/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3784,6 +3784,11 @@ def _sort_table_by_sort_order(arrow_table: pa.Table, schema: Schema, sort_order:

from pyiceberg.utils.arrow_sorting import convert_sort_field_to_pyarrow_sort_options, get_sort_indices_arrow_table

if unsupported_sort_transforms := [field for field in sort_order.fields if not field.transform.supports_pyarrow_transform]:
raise ValueError(
f"Not all sort transforms are supported for writes. Following sort orders cannot be written using pyarrow: {unsupported_sort_transforms}."
)

sort_columns: List[Tuple[SortField, NestedField]] = [
(sort_field, schema.find_field(sort_field.source_id)) for sort_field in sort_order.fields
]
Expand Down
Loading