Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
dc9d6f0
initial commit
kiszk Apr 13, 2018
3019840
update description
kiszk Apr 13, 2018
8cee6cf
fix test failure
kiszk Apr 13, 2018
2041ec4
address review comments
kiszk Apr 17, 2018
8c2280b
introduce ArraySetUtils to reuse code among array_union/array_interse…
kiszk Apr 17, 2018
b3a3132
fix python test failure
kiszk Apr 18, 2018
a2c7dd1
fix python test failure
kiszk Apr 18, 2018
5313680
simplification
kiszk Apr 18, 2018
98f8d1f
fix pyspark test failure
kiszk Apr 19, 2018
30ee7fc
address review comments
kiszk Apr 20, 2018
cd347e9
add new tests based on review comment
kiszk Apr 20, 2018
d2eaee3
fix mistakes in rebase
kiszk Apr 20, 2018
2ddeb06
fix unexpected changes
kiszk Apr 20, 2018
71b31f0
merge changes in #21103
kiszk Apr 20, 2018
7e71340
use GenericArrayData if UnsafeArrayData cannot be used
kiszk May 4, 2018
04c97c3
use BinaryArrayExpressionWithImplicitCast
kiszk May 4, 2018
401ca7a
update test cases
kiszk May 4, 2018
15b953b
rebase with master
kiszk May 17, 2018
f050922
support complex types
kiszk May 18, 2018
8a27667
add test cases with duplication in an array
kiszk May 19, 2018
e50bc55
rebase with master
kiszk Jun 1, 2018
7e3f2ef
address review comments
kiszk Jun 1, 2018
e5401e7
address review comment
kiszk Jun 1, 2018
3e21e48
keep the order of input array elements
kiszk Jun 10, 2018
3c39506
address review comments
kiszk Jun 20, 2018
6654742
fix scala style error
kiszk Jun 20, 2018
be9f331
address review comment
kiszk Jun 20, 2018
90e84b3
address review comments
kiszk Jun 22, 2018
6f721f0
address review comments
kiszk Jun 22, 2018
0c0d3ba
address review comments
kiszk Jul 8, 2018
4a217bc
cleanup
kiszk Jul 8, 2018
f5ebbe8
eliminate duplicated code
kiszk Jul 8, 2018
763a1f8
address review comments
kiszk Jul 9, 2018
7b51564
address review comment
kiszk Jul 11, 2018
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
address review comment
  • Loading branch information
kiszk committed Jun 27, 2018
commit e5401e7029b9aba192c0e5a69607a57cc41692d8
Original file line number Diff line number Diff line change
Expand Up @@ -3275,14 +3275,10 @@ object ArraySetLike {
i += 1
}

val numBytes = IntegerType.defaultSize.toLong * array.length
val unsafeArraySizeInBytes = UnsafeArrayData.calculateHeaderPortionInBytes(array.length) +
ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes)
// Since UnsafeArrayData.fromPrimitiveArray() uses long[], max elements * 8 bytes can be used
if (unsafeArraySizeInBytes <= Integer.MAX_VALUE * 8) {
UnsafeArrayData.fromPrimitiveArray(array)
} else {
if (useGenericArrayData(LongType.defaultSize, array.length)) {
new GenericArrayData(array)
} else {
UnsafeArrayData.fromPrimitiveArray(array)
}
}

Expand All @@ -3296,17 +3292,21 @@ object ArraySetLike {
i += 1
}

val numBytes = LongType.defaultSize.toLong * array.length
val unsafeArraySizeInBytes = UnsafeArrayData.calculateHeaderPortionInBytes(array.length) +
ByteArrayMethods.roundNumberOfBytesToNearestWord(numBytes)
// Since UnsafeArrayData.fromPrimitiveArray() uses long[], max elements * 8 bytes can be used
if (unsafeArraySizeInBytes <= Integer.MAX_VALUE * 8) {
UnsafeArrayData.fromPrimitiveArray(array)
} else {
if (useGenericArrayData(LongType.defaultSize, array.length)) {
new GenericArrayData(array)
} else {
UnsafeArrayData.fromPrimitiveArray(array)
}
}

def useGenericArrayData(elementSize: Int, length: Int): Boolean = {
// Use the same calculation in UnsafeArrayData.fromPrimitiveArray()
val headerInBytes = UnsafeArrayData.calculateHeaderPortionInBytes(length)
val valueRegionInBytes = elementSize.toLong * length
val totalSizeInLongs = (headerInBytes + valueRegionInBytes + 7) / 8
totalSizeInLongs > Integer.MAX_VALUE / 8
}

def arrayUnion(
array1: ArrayData,
array2: ArrayData,
Expand Down