Skip to content

[SPARK-23976][Core] Detect length overflow in UTF8String.concat()/ByteArray.concat() #21064

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
kiszk wants to merge 1 commit into from
Closed

[SPARK-23976][Core] Detect length overflow in UTF8String.concat()/ByteArray.concat() #21064

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
initial commit
  • Loading branch information
@kiszk
kiszk committed Apr 13, 2018
commit 6ba94f3b6c4cc3e69e4916a1669e95ef71267a6a
12 changes: 7 additions & 5 deletions common/unsafe/src/main/java/org/apache/spark/unsafe/types/ByteArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,12 @@

package org.apache.spark.unsafe.types;

import org.apache.spark.unsafe.Platform;

import java.util.Arrays;

import com.google.common.primitives.Ints;

import org.apache.spark.unsafe.Platform;

public final class ByteArray {

public static final byte[] EMPTY_BYTE = new byte[0];
Expand Down Expand Up @@ -77,17 +79,17 @@ public static byte[] subStringSQL(byte[] bytes, int pos, int len) {

public static byte[] concat(byte[]... inputs) {
// Compute the total length of the result
int totalLength = 0;
long totalLength = 0;
for (int i = 0; i < inputs.length; i++) {
if (inputs[i] != null) {
totalLength += inputs[i].length;
totalLength += (long)inputs[i].length;
} else {
return null;
}
}

// Allocate a new byte array, and copy the inputs one by one into it
final byte[] result = new byte[totalLength];
final byte[] result = new byte[Ints.checkedCast(totalLength)];
int offset = 0;
for (int i = 0; i < inputs.length; i++) {
int len = inputs[i].length;
Expand Down
8 changes: 4 additions & 4 deletions common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import com.esotericsoftware.kryo.KryoSerializable;
import com.esotericsoftware.kryo.io.Input;
import com.esotericsoftware.kryo.io.Output;

import com.google.common.primitives.Ints;

import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.array.ByteArrayMethods;
import org.apache.spark.unsafe.hash.Murmur3_x86_32;
Expand Down Expand Up @@ -877,17 +877,17 @@ public UTF8String lpad(int len, UTF8String pad) {
*/
public static UTF8String concat(UTF8String... inputs) {
// Compute the total length of the result.
int totalLength = 0;
long totalLength = 0;
for (int i = 0; i < inputs.length; i++) {
if (inputs[i] != null) {
totalLength += inputs[i].numBytes;
totalLength += (long)inputs[i].numBytes;
} else {
return null;
}
}

// Allocate a new byte array, and copy the inputs one by one into it.
final byte[] result = new byte[totalLength];
final byte[] result = new byte[Ints.checkedCast(totalLength)];
int offset = 0;
for (int i = 0; i < inputs.length; i++) {
int len = inputs[i].numBytes;
Expand Down