Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Throws a proper exception rather than ArrayIndexOutOfBoundsException …
…when temp directories could not be got/created
  • Loading branch information
HyukjinKwon committed Apr 26, 2017
commit 4500c2f2d989bfc1e76e07cbd38a9acbd384d3d5
5 changes: 4 additions & 1 deletion core/src/main/scala/org/apache/spark/util/Utils.scala
Original file line number Diff line number Diff line change
Expand Up @@ -740,7 +740,10 @@ private[spark] object Utils extends Logging {
* always return a single directory.
*/
def getLocalDir(conf: SparkConf): String = {
getOrCreateLocalRootDirs(conf)(0)
getOrCreateLocalRootDirs(conf).headOption.getOrElse {
val dirPaths = getConfiguredLocalDirs(conf)
throw new IOException(s"Failed to get a temp directory under [${dirPaths.mkString(",")}].")
}
}

private[spark] def isRunningInYarnContainer(conf: SparkConf): Boolean = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

package org.apache.spark.storage

import java.io.File
import java.io.{File, IOException}

import org.scalatest.BeforeAndAfter

Expand Down Expand Up @@ -51,4 +51,14 @@ class LocalDirsSuite extends SparkFunSuite with BeforeAndAfter {
assert(new File(Utils.getLocalDir(conf)).exists())
}

test("Utils.getLocalDir() throws an exception if any temporary directory cannot be retrieved") {
assert(!new File("/NONEXISTENT_DIR_ONE").exists())
assert(!new File("/NONEXISTENT_DIR_TWO").exists())
val conf = new SparkConf().set("spark.local.dir", "/NONEXISTENT_PATH_ONE,/NONEXISTENT_PATH_TWO")
val message = intercept[IOException] {
Utils.getLocalDir(conf)
}.getMessage
assert(message ===
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would only suggest not testing the exact message here, but it's not a big deal. Maybe testing for the existence of the paths.

"Failed to get a temp directory under [/NONEXISTENT_PATH_ONE,/NONEXISTENT_PATH_TWO].")
}
}