Skip to content
Prev Previous commit
Next Next commit
adding an update in functions.py
  • Loading branch information
kiszk committed Apr 19, 2018
commit d57c14a290dbcf0b05ea8cf3a52da5d2d46639fc
18 changes: 18 additions & 0 deletions python/pyspark/sql/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -2158,6 +2158,7 @@ def sort_array(col, asc=True):

:param col: name of column or expression

>>> from pyspark.sql.functions import sort_array
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we need this?

>>> df = spark.createDataFrame([([2, 1, 3],),([1],),([],)], ['data'])
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should include None to show where the None comes?

>>> df.select(sort_array(df.data).alias('r')).collect()
[Row(r=[1, 2, 3]), Row(r=[1]), Row(r=[])]
Expand All @@ -2168,6 +2169,23 @@ def sort_array(col, asc=True):
return Column(sc._jvm.functions.sort_array(_to_java_column(col), asc))


@since(2.4)
def array_sort(col):
"""
Collection function: sorts the input array in ascending order. The elements of the input array
must be orderable. Null elements will be placed at the end of the returned array.

:param col: name of column or expression

>>> from pyspark.sql.functions import array_sort
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ditto.

>>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data'])
>>> df.select(array_sort(df.data).alias('r')).collect()
[Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])]
"""
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: an extra space?

sc = SparkContext._active_spark_context
return Column(sc._jvm.functions.array_sort(_to_java_column(col)))


@since(1.5)
@ignore_unicode_prefix
def reverse(col):
Expand Down