Skip to content
Closed
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
fb9a42d
add two implementations (sparse and dense) for UnsafeArrayData
kiszk Jun 14, 2016
d931428
fix failures of testsuite
kiszk Jun 15, 2016
9777a2d
fix errors of unit tests
kiszk Jun 15, 2016
000eda4
fix failures of unit tests
kiszk Jun 15, 2016
804f081
make DenseID public
kiszk Jun 23, 2016
e6fb261
Use one implementation approach
kiszk Jun 25, 2016
a313084
fix test failures
kiszk Jun 25, 2016
68d92f7
fix test failures
kiszk Jun 25, 2016
7f2da14
update test suite
kiszk Jun 25, 2016
2f26f6f
fix scala style error
kiszk Jun 25, 2016
ccef63c
revert changes
kiszk Jun 25, 2016
c4f1b5e
addressed comments
kiszk Jun 28, 2016
34a5c6a
add benchmark
kiszk Jun 28, 2016
7a77b20
fix scala style error
kiszk Jun 28, 2016
7b0d4da
addressed comments
kiszk Jul 1, 2016
b4eac29
addressed comments
kiszk Jul 2, 2016
eecf6bd
fix parameters of Platform.OFFSET
kiszk Jul 3, 2016
d88a25a
update benchmark results
kiszk Jul 3, 2016
db15432
add test cases
kiszk Jul 3, 2016
3fa7052
addressed comments
kiszk Jul 4, 2016
4c094c2
addressed comments
kiszk Jul 6, 2016
9887171
update test cases
kiszk Jul 6, 2016
9fe7ad0
address comments
kiszk Jul 7, 2016
e4b4b52
address comments for test cases and benchmark
kiszk Jul 7, 2016
585ca7b
addressed comments
kiszk Jul 8, 2016
9933a06
addressed review comments
kiszk Aug 6, 2016
919e832
fixed test failures
kiszk Aug 7, 2016
0886e3a
update test suites
kiszk Aug 9, 2016
c385bf4
align each of variable length elements to 8 bytes
kiszk Aug 18, 2016
c8813db
fixed test failures
kiszk Aug 20, 2016
aa7cfdb
fixed test failures
kiszk Sep 9, 2016
0b7867b
address review comments
kiszk Sep 20, 2016
ab9a16a
address review comments
kiszk Sep 20, 2016
515701b
address review comments
kiszk Sep 20, 2016
8169abd
change benchmark size
kiszk Sep 26, 2016
e356a79
addressed comments
kiszk Sep 26, 2016
2ef6e3b
update performance results
kiszk Sep 26, 2016
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
addressed comments
  • Loading branch information
kiszk committed Sep 26, 2016
commit 7b0d4dad0c507ef491e2cdd18c3e76530ddeb94b
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
* Instances of `UnsafeArrayData` act as pointers to row data stored in this format.
*/

// todo: there is a lof of duplicated code between UnsafeRow and UnsafeArrayData.
public final class UnsafeArrayData extends ArrayData {

public static int calculateHeaderPortionInBytes(int numFields) {
Expand All @@ -69,7 +68,7 @@ public static int calculateHeaderPortionInBytes(int numFields) {
/** The width of the null tracking bit set plus `numElements`, in bytes */
private int headerInBytes;

private long getFieldOffset(int ordinal, int elementSize) {
private long getElementOffset(int ordinal, int elementSize) {
return baseOffset + headerInBytes + ordinal * elementSize;
}

Expand Down Expand Up @@ -168,51 +167,48 @@ public Object get(int ordinal, DataType dataType) {
@Override
public boolean getBoolean(int ordinal) {
assertIndexIsValid(ordinal);
return Platform.getBoolean(baseObject, getFieldOffset(ordinal, 1));
return Platform.getBoolean(baseObject, getElementOffset(ordinal, 1));
}

@Override
public byte getByte(int ordinal) {
assertIndexIsValid(ordinal);
return Platform.getByte(baseObject, getFieldOffset(ordinal, 1));
return Platform.getByte(baseObject, getElementOffset(ordinal, 1));
}

@Override
public short getShort(int ordinal) {
assertIndexIsValid(ordinal);
return Platform.getShort(baseObject, getFieldOffset(ordinal, 2));
return Platform.getShort(baseObject, getElementOffset(ordinal, 2));
}

@Override
public int getInt(int ordinal) {
assertIndexIsValid(ordinal);
return Platform.getInt(baseObject, getFieldOffset(ordinal, 4));
return Platform.getInt(baseObject, getElementOffset(ordinal, 4));
}

@Override
public long getLong(int ordinal) {
assertIndexIsValid(ordinal);
return Platform.getLong(baseObject, getFieldOffset(ordinal, 8));
return Platform.getLong(baseObject, getElementOffset(ordinal, 8));
}

@Override
public float getFloat(int ordinal) {
assertIndexIsValid(ordinal);
return Platform.getFloat(baseObject, getFieldOffset(ordinal, 4));
return Platform.getFloat(baseObject, getElementOffset(ordinal, 4));
}

@Override
public double getDouble(int ordinal) {
assertIndexIsValid(ordinal);
return Platform.getDouble(baseObject, getFieldOffset(ordinal, 8));
return Platform.getDouble(baseObject, getElementOffset(ordinal, 8));
}

@Override
public Decimal getDecimal(int ordinal, int precision, int scale) {
assertIndexIsValid(ordinal);
if (isNullAt(ordinal)) {
return null;
}
if (isNullAt(ordinal)) return null;
if (precision <= Decimal.MAX_LONG_DIGITS()) {
return Decimal.apply(getLong(ordinal), precision, scale);
} else {
Expand Down Expand Up @@ -395,15 +391,15 @@ private static UnsafeArrayData fromPrimitiveArray(Object arr, int length, final
final int headerSize = calculateHeaderPortionInBytes(length);
if (length > (Integer.MAX_VALUE - headerSize) / elementSize) {
throw new UnsupportedOperationException("Cannot convert this array to unsafe format as " +
"it's too big.");
"it's too big.");
}

final int valueRegionSize = elementSize * length;
final byte[] data = new byte[valueRegionSize + headerSize];

Platform.putInt(data, Platform.BYTE_ARRAY_OFFSET, length);
Platform.copyMemory(arr, Platform.INT_ARRAY_OFFSET, data,
Platform.BYTE_ARRAY_OFFSET + headerSize, valueRegionSize);
Platform.BYTE_ARRAY_OFFSET + headerSize, valueRegionSize);

UnsafeArrayData result = new UnsafeArrayData();
result.pointTo(data, Platform.BYTE_ARRAY_OFFSET, valueRegionSize + headerSize);
Expand Down