Skip to content

[SPARK-48019][SQL][FOLLOWUP] Use primitive arrays over object arrays when nulls exist #46372

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gene-db wants to merge 1 commit into from
Closed

[SPARK-48019][SQL][FOLLOWUP] Use primitive arrays over object arrays when nulls exist #46372

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 24 additions & 12 deletions sql/catalyst/src/main/java/org/apache/spark/sql/vectorized/ColumnarArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,31 +49,43 @@ public int numElements() {
return length;
}

/**
* Sets all the appropriate null bits in the input UnsafeArrayData.
*
* @param arrayData The UnsafeArrayData to set the null bits for
* @return The UnsafeArrayData with the null bits set
*/
private UnsafeArrayData setNullBits(UnsafeArrayData arrayData) {
if (data.hasNull()) {
for (int i = 0; i < length; i++) {
if (data.isNullAt(i)) {
arrayData.setNullAt(i);
}
}
}
return arrayData;
}

@Override
public ArrayData copy() {
DataType dt = data.dataType();

if (data.hasNull()) {
// UnsafeArrayData cannot be used if there are any nulls.
return new GenericArrayData(toObjectArray(dt)).copy();
}

if (dt instanceof BooleanType) {
return UnsafeArrayData.fromPrimitiveArray(toBooleanArray());
return setNullBits(UnsafeArrayData.fromPrimitiveArray(toBooleanArray()));
} else if (dt instanceof ByteType) {
return UnsafeArrayData.fromPrimitiveArray(toByteArray());
return setNullBits(UnsafeArrayData.fromPrimitiveArray(toByteArray()));
} else if (dt instanceof ShortType) {
return UnsafeArrayData.fromPrimitiveArray(toShortArray());
return setNullBits(UnsafeArrayData.fromPrimitiveArray(toShortArray()));
} else if (dt instanceof IntegerType || dt instanceof DateType
|| dt instanceof YearMonthIntervalType) {
return UnsafeArrayData.fromPrimitiveArray(toIntArray());
return setNullBits(UnsafeArrayData.fromPrimitiveArray(toIntArray()));
} else if (dt instanceof LongType || dt instanceof TimestampType
|| dt instanceof DayTimeIntervalType) {
return UnsafeArrayData.fromPrimitiveArray(toLongArray());
return setNullBits(UnsafeArrayData.fromPrimitiveArray(toLongArray()));
} else if (dt instanceof FloatType) {
return UnsafeArrayData.fromPrimitiveArray(toFloatArray());
return setNullBits(UnsafeArrayData.fromPrimitiveArray(toFloatArray()));
} else if (dt instanceof DoubleType) {
return UnsafeArrayData.fromPrimitiveArray(toDoubleArray());
return setNullBits(UnsafeArrayData.fromPrimitiveArray(toDoubleArray()));
} else {
return new GenericArrayData(toObjectArray(dt)).copy(); // ensure the elements are copied.
}
Expand Down