Skip to content
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update PartitioningAwareFileIndex.scala
  • Loading branch information
jayadevanmurali authored Jan 18, 2017
commit 4334b2ba2a882a94d59459a3d73b5fdb6fda6b80
Original file line number Diff line number Diff line change
Expand Up @@ -385,7 +385,7 @@ object PartitioningAwareFileIndex extends Logging {
logTrace(s"Listing $path")
val fs = path.getFileSystem(hadoopConf)
val name = path.getName.toLowerCase

// [SPARK-17599] Prevent InMemoryFileIndex from failing if path doesn't exist
// Note that statuses only include FileStatus for the files and dirs directly under path,
// and does not include anything else recursively.
Expand All @@ -394,7 +394,7 @@ object PartitioningAwareFileIndex extends Logging {
logWarning(s"The directory $path was not found. Was it deleted very recently?")
Array.empty[FileStatus]
}

val filteredStatuses = statuses.filterNot(status => shouldFilterOut(status.getPath.getName))

val allLeafStatuses = {
Expand Down