Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update as commented
  • Loading branch information
caneGuy committed Aug 24, 2017
commit e48f44f99355c2811c4cc8f094c062dee33ff428
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,13 @@ package object config {
.booleanConf
.createWithDefault(false)

private[spark] val BUFFER_WRITE_CHUNK_SIZE =
ConfigBuilder("spark.buffer.write.chunkSize")
.internal()
.doc("The chunk size during writing out the bytes of ChunkedByteBuffer.")
.bytesConf(ByteUnit.BYTE)
.createWithDefault(64 * 1024 * 1024)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add a checkValue to make sure the value is smaller than Int.Max


private[spark] val CHECKPOINT_COMPRESS =
ConfigBuilder("spark.checkpoint.compress")
.doc("Whether to compress RDD checkpoints. Generally a good idea. Compression will use " +
Expand Down Expand Up @@ -321,13 +328,6 @@ package object config {
.intConf
.createWithDefault(3)

private[spark] val BUFFER_WRITE_CHUNK_SIZE =
ConfigBuilder("spark.buffer.write.chunkSize")
.internal()
.doc("The block size limit when use ChunkedByteBuffer to writeFully bytes.")
.bytesConf(ByteUnit.BYTE)
.createWithDefault(64 * 1024 * 1024)

private[spark] val REDUCER_MAX_REQ_SIZE_SHUFFLE_TO_MEM =
ConfigBuilder("spark.reducer.maxReqSizeShuffleToMem")
.internal()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ private[spark] class ChunkedByteBuffer(var chunks: Array[ByteBuffer]) {
require(chunks.forall(_.position() == 0), "chunks' positions must be 0")

// Chunk size in bytes
private val NIO_BUFFER_LIMIT = SparkEnv.get.conf.get(config.BUFFER_WRITE_CHUNK_SIZE)
private val bufferWriteChunkSize =
Option(SparkEnv.get).map(_.conf.get(BUFFER_WRITE_CHUNK_SIZE))
.getOrElse(BUFFER_WRITE_CHUNK_SIZE.defaultValue.get)

private[this] var disposed: Boolean = false

Expand All @@ -60,9 +62,8 @@ private[spark] class ChunkedByteBuffer(var chunks: Array[ByteBuffer]) {
*/
def writeFully(channel: WritableByteChannel): Unit = {
for (bytes <- getChunks()) {
val capacity = bytes.limit()
while (bytes.remaining() > 0) {
val ioSize = Math.min(bytes.remaining(), NIO_BUFFER_LIMIT)
val ioSize = Math.min(bytes.remaining(), bufferWriteChunkSize)
bytes.limit(bytes.position + ioSize)
channel.write(bytes)
}
Expand Down