Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import scala.reflect.ClassTag;

import org.apache.spark.annotation.Private;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;

/**
* Unfortunately, we need a serializer instance in order to construct a DiskBlockObjectWriter.
Expand All @@ -49,7 +49,7 @@ public void flush() {
try {
s.flush();
} catch (IOException e) {
PlatformDependent.throwException(e);
Platform.throwException(e);
}
}

Expand All @@ -64,7 +64,7 @@ public void close() {
try {
s.close();
} catch (IOException e) {
PlatformDependent.throwException(e);
Platform.throwException(e);
}
}
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.DiskBlockObjectWriter;
import org.apache.spark.storage.TempShuffleBlockId;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.array.ByteArrayMethods;
import org.apache.spark.unsafe.memory.MemoryBlock;
import org.apache.spark.unsafe.memory.TaskMemoryManager;
Expand Down Expand Up @@ -211,16 +211,12 @@ private void writeSortedFile(boolean isLastFile) throws IOException {
final long recordPointer = sortedRecords.packedRecordPointer.getRecordPointer();
final Object recordPage = taskMemoryManager.getPage(recordPointer);
final long recordOffsetInPage = taskMemoryManager.getOffsetInPage(recordPointer);
int dataRemaining = PlatformDependent.UNSAFE.getInt(recordPage, recordOffsetInPage);
int dataRemaining = Platform.getInt(recordPage, recordOffsetInPage);
long recordReadPosition = recordOffsetInPage + 4; // skip over record length
while (dataRemaining > 0) {
final int toTransfer = Math.min(DISK_WRITE_BUFFER_SIZE, dataRemaining);
PlatformDependent.copyMemory(
recordPage,
recordReadPosition,
writeBuffer,
PlatformDependent.BYTE_ARRAY_OFFSET,
toTransfer);
Platform.copyMemory(
recordPage, recordReadPosition, writeBuffer, Platform.BYTE_ARRAY_OFFSET, toTransfer);
writer.write(writeBuffer, 0, toTransfer);
recordReadPosition += toTransfer;
dataRemaining -= toTransfer;
Expand Down Expand Up @@ -447,14 +443,10 @@ public void insertRecord(

final long recordAddress =
taskMemoryManager.encodePageNumberAndOffset(dataPage, dataPagePosition);
PlatformDependent.UNSAFE.putInt(dataPageBaseObject, dataPagePosition, lengthInBytes);
Platform.putInt(dataPageBaseObject, dataPagePosition, lengthInBytes);
dataPagePosition += 4;
PlatformDependent.copyMemory(
recordBaseObject,
recordBaseOffset,
dataPageBaseObject,
dataPagePosition,
lengthInBytes);
Platform.copyMemory(
recordBaseObject, recordBaseOffset, dataPageBaseObject, dataPagePosition, lengthInBytes);
assert(inMemSorter != null);
inMemSorter.insertRecord(recordAddress, partitionId);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
import org.apache.spark.shuffle.ShuffleWriter;
import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.TimeTrackingOutputStream;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.memory.TaskMemoryManager;

@Private
Expand Down Expand Up @@ -244,7 +244,7 @@ void insertRecordIntoSorter(Product2<K, V> record) throws IOException {
assert (serializedRecordSize > 0);

sorter.insertRecord(
serBuffer.getBuf(), PlatformDependent.BYTE_ARRAY_OFFSET, serializedRecordSize, partitionId);
serBuffer.getBuf(), Platform.BYTE_ARRAY_OFFSET, serializedRecordSize, partitionId);
}

@VisibleForTesting
Expand Down
20 changes: 10 additions & 10 deletions core/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
Original file line number Diff line number Diff line change
Expand Up @@ -270,10 +270,10 @@ public boolean hasNext() {

@Override
public Location next() {
int totalLength = PlatformDependent.UNSAFE.getInt(pageBaseObject, offsetInPage);
int totalLength = Platform.getInt(pageBaseObject, offsetInPage);
if (totalLength == END_OF_PAGE_MARKER) {
advanceToNextPage();
totalLength = PlatformDependent.UNSAFE.getInt(pageBaseObject, offsetInPage);
totalLength = Platform.getInt(pageBaseObject, offsetInPage);
}
loc.with(currentPage, offsetInPage);
offsetInPage += 4 + totalLength;
Expand Down Expand Up @@ -402,9 +402,9 @@ private void updateAddressesAndSizes(long fullKeyAddress) {

private void updateAddressesAndSizes(final Object page, final long offsetInPage) {
long position = offsetInPage;
final int totalLength = PlatformDependent.UNSAFE.getInt(page, position);
final int totalLength = Platform.getInt(page, position);
position += 4;
keyLength = PlatformDependent.UNSAFE.getInt(page, position);
keyLength = Platform.getInt(page, position);
position += 4;
valueLength = totalLength - keyLength - 4;

Expand Down Expand Up @@ -572,7 +572,7 @@ public boolean putNewKey(
// There wasn't enough space in the current page, so write an end-of-page marker:
final Object pageBaseObject = currentDataPage.getBaseObject();
final long lengthOffsetInPage = currentDataPage.getBaseOffset() + pageCursor;
PlatformDependent.UNSAFE.putInt(pageBaseObject, lengthOffsetInPage, END_OF_PAGE_MARKER);
Platform.putInt(pageBaseObject, lengthOffsetInPage, END_OF_PAGE_MARKER);
}
final long memoryGranted = shuffleMemoryManager.tryToAcquire(pageSizeBytes);
if (memoryGranted != pageSizeBytes) {
Expand Down Expand Up @@ -608,21 +608,21 @@ public boolean putNewKey(
final long valueDataOffsetInPage = insertCursor;
insertCursor += valueLengthBytes; // word used to store the value size

PlatformDependent.UNSAFE.putInt(dataPageBaseObject, recordOffset,
Platform.putInt(dataPageBaseObject, recordOffset,
keyLengthBytes + valueLengthBytes + 4);
PlatformDependent.UNSAFE.putInt(dataPageBaseObject, keyLengthOffset, keyLengthBytes);
Platform.putInt(dataPageBaseObject, keyLengthOffset, keyLengthBytes);
// Copy the key
PlatformDependent.copyMemory(
Platform.copyMemory(
keyBaseObject, keyBaseOffset, dataPageBaseObject, keyDataOffsetInPage, keyLengthBytes);
// Copy the value
PlatformDependent.copyMemory(valueBaseObject, valueBaseOffset, dataPageBaseObject,
Platform.copyMemory(valueBaseObject, valueBaseOffset, dataPageBaseObject,
valueDataOffsetInPage, valueLengthBytes);

// --- Update bookeeping data structures -----------------------------------------------------

if (useOverflowPage) {
// Store the end-of-page marker at the end of the data page
PlatformDependent.UNSAFE.putInt(dataPageBaseObject, insertCursor, END_OF_PAGE_MARKER);
Platform.putInt(dataPageBaseObject, insertCursor, END_OF_PAGE_MARKER);
} else {
pageCursor += requiredSize;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,9 @@
import com.google.common.primitives.UnsignedLongs;

import org.apache.spark.annotation.Private;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.types.UTF8String;
import org.apache.spark.util.Utils;
import static org.apache.spark.unsafe.PlatformDependent.BYTE_ARRAY_OFFSET;

@Private
public class PrefixComparators {
Expand Down Expand Up @@ -73,7 +72,7 @@ public static long computePrefix(byte[] bytes) {
final int minLen = Math.min(bytes.length, 8);
long p = 0;
for (int i = 0; i < minLen; ++i) {
p |= (128L + PlatformDependent.UNSAFE.getByte(bytes, BYTE_ARRAY_OFFSET + i))
p |= (128L + Platform.getByte(bytes, Platform.BYTE_ARRAY_OFFSET + i))
<< (56 - 8 * i);
}
return p;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
import org.apache.spark.shuffle.ShuffleMemoryManager;
import org.apache.spark.storage.BlockManager;
import org.apache.spark.unsafe.array.ByteArrayMethods;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.memory.MemoryBlock;
import org.apache.spark.unsafe.memory.TaskMemoryManager;
import org.apache.spark.util.Utils;
Expand Down Expand Up @@ -427,14 +427,10 @@ public void insertRecord(

final long recordAddress =
taskMemoryManager.encodePageNumberAndOffset(dataPage, dataPagePosition);
PlatformDependent.UNSAFE.putInt(dataPageBaseObject, dataPagePosition, lengthInBytes);
Platform.putInt(dataPageBaseObject, dataPagePosition, lengthInBytes);
dataPagePosition += 4;
PlatformDependent.copyMemory(
recordBaseObject,
recordBaseOffset,
dataPageBaseObject,
dataPagePosition,
lengthInBytes);
Platform.copyMemory(
recordBaseObject, recordBaseOffset, dataPageBaseObject, dataPagePosition, lengthInBytes);
assert(inMemSorter != null);
inMemSorter.insertRecord(recordAddress, prefix);
}
Expand Down Expand Up @@ -493,18 +489,16 @@ public void insertKVRecord(

final long recordAddress =
taskMemoryManager.encodePageNumberAndOffset(dataPage, dataPagePosition);
PlatformDependent.UNSAFE.putInt(dataPageBaseObject, dataPagePosition, keyLen + valueLen + 4);
Platform.putInt(dataPageBaseObject, dataPagePosition, keyLen + valueLen + 4);
dataPagePosition += 4;

PlatformDependent.UNSAFE.putInt(dataPageBaseObject, dataPagePosition, keyLen);
Platform.putInt(dataPageBaseObject, dataPagePosition, keyLen);
dataPagePosition += 4;

PlatformDependent.copyMemory(
keyBaseObj, keyOffset, dataPageBaseObject, dataPagePosition, keyLen);
Platform.copyMemory(keyBaseObj, keyOffset, dataPageBaseObject, dataPagePosition, keyLen);
dataPagePosition += keyLen;

PlatformDependent.copyMemory(
valueBaseObj, valueOffset, dataPageBaseObject, dataPagePosition, valueLen);
Platform.copyMemory(valueBaseObj, valueOffset, dataPageBaseObject, dataPagePosition, valueLen);

assert(inMemSorter != null);
inMemSorter.insertRecord(recordAddress, prefix);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import java.util.Comparator;

import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.util.collection.Sorter;
import org.apache.spark.unsafe.memory.TaskMemoryManager;

Expand Down Expand Up @@ -164,7 +164,7 @@ public void loadNext() {
final long recordPointer = sortBuffer[position];
baseObject = memoryManager.getPage(recordPointer);
baseOffset = memoryManager.getOffsetInPage(recordPointer) + 4; // Skip over record length
recordLength = PlatformDependent.UNSAFE.getInt(baseObject, baseOffset - 4);
recordLength = Platform.getInt(baseObject, baseOffset - 4);
keyPrefix = sortBuffer[position + 1];
position += 2;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

import org.apache.spark.storage.BlockId;
import org.apache.spark.storage.BlockManager;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;

/**
* Reads spill files written by {@link UnsafeSorterSpillWriter} (see that class for a description
Expand All @@ -42,7 +42,7 @@ final class UnsafeSorterSpillReader extends UnsafeSorterIterator {

private byte[] arr = new byte[1024 * 1024];
private Object baseObject = arr;
private final long baseOffset = PlatformDependent.BYTE_ARRAY_OFFSET;
private final long baseOffset = Platform.BYTE_ARRAY_OFFSET;

public UnsafeSorterSpillReader(
BlockManager blockManager,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
import org.apache.spark.storage.BlockManager;
import org.apache.spark.storage.DiskBlockObjectWriter;
import org.apache.spark.storage.TempLocalBlockId;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;

/**
* Spills a list of sorted records to disk. Spill files have the following format:
Expand Down Expand Up @@ -117,11 +117,11 @@ public void write(
long recordReadPosition = baseOffset;
while (dataRemaining > 0) {
final int toTransfer = Math.min(freeSpaceInWriteBuffer, dataRemaining);
PlatformDependent.copyMemory(
Platform.copyMemory(
baseObject,
recordReadPosition,
writeBuffer,
PlatformDependent.BYTE_ARRAY_OFFSET + (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer),
Platform.BYTE_ARRAY_OFFSET + (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer),
toTransfer);
writer.write(writeBuffer, 0, (DISK_WRITE_BUFFER_SIZE - freeSpaceInWriteBuffer) + toTransfer);
recordReadPosition += toTransfer;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.junit.Test;

import org.apache.spark.HashPartitioner;
import org.apache.spark.unsafe.PlatformDependent;
import org.apache.spark.unsafe.Platform;
import org.apache.spark.unsafe.memory.ExecutorMemoryManager;
import org.apache.spark.unsafe.memory.MemoryAllocator;
import org.apache.spark.unsafe.memory.MemoryBlock;
Expand All @@ -34,11 +34,7 @@ public class UnsafeShuffleInMemorySorterSuite {

private static String getStringFromDataPage(Object baseObject, long baseOffset, int strLength) {
final byte[] strBytes = new byte[strLength];
PlatformDependent.copyMemory(
baseObject,
baseOffset,
strBytes,
PlatformDependent.BYTE_ARRAY_OFFSET, strLength);
Platform.copyMemory(baseObject, baseOffset, strBytes, Platform.BYTE_ARRAY_OFFSET, strLength);
return new String(strBytes);
}

Expand Down Expand Up @@ -74,14 +70,10 @@ public void testBasicSorting() throws Exception {
for (String str : dataToSort) {
final long recordAddress = memoryManager.encodePageNumberAndOffset(dataPage, position);
final byte[] strBytes = str.getBytes("utf-8");
PlatformDependent.UNSAFE.putInt(baseObject, position, strBytes.length);
Platform.putInt(baseObject, position, strBytes.length);
position += 4;
PlatformDependent.copyMemory(
strBytes,
PlatformDependent.BYTE_ARRAY_OFFSET,
baseObject,
position,
strBytes.length);
Platform.copyMemory(
strBytes, Platform.BYTE_ARRAY_OFFSET, baseObject, position, strBytes.length);
position += strBytes.length;
sorter.insertRecord(recordAddress, hashPartitioner.getPartition(str));
}
Expand All @@ -98,7 +90,7 @@ public void testBasicSorting() throws Exception {
Assert.assertTrue("Partition id " + partitionId + " should be >= prev id " + prevPartitionId,
partitionId >= prevPartitionId);
final long recordAddress = iter.packedRecordPointer.getRecordPointer();
final int recordLength = PlatformDependent.UNSAFE.getInt(
final int recordLength = Platform.getInt(
memoryManager.getPage(recordAddress), memoryManager.getOffsetInPage(recordAddress));
final String str = getStringFromDataPage(
memoryManager.getPage(recordAddress),
Expand Down
Loading