Skip to content
This repository was archived by the owner on Jan 9, 2020. It is now read-only.
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
7570eab
[SPARK-22788][STREAMING] Use correct hadoop config for fs append supp…
Dec 20, 2017
7798c9e
[SPARK-22824] Restore old offset for binary compatibility
jose-torres Dec 20, 2017
d762d11
[SPARK-22832][ML] BisectingKMeans unpersist unused datasets
zhengruifeng Dec 20, 2017
c89b431
[SPARK-22849] ivy.retrieve pattern should also consider `classifier`
gatorsmile Dec 20, 2017
792915c
[SPARK-22830] Scala Coding style has been improved in Spark Examples
chetkhatri Dec 20, 2017
b176014
[SPARK-22847][CORE] Remove redundant code in AppStatusListener while …
Ngone51 Dec 20, 2017
0114c89
[SPARK-22845][SCHEDULER] Modify spark.kubernetes.allocation.batch.del…
foxish Dec 21, 2017
fb0562f
[SPARK-22810][ML][PYSPARK] Expose Python API for LinearRegression wit…
yanboliang Dec 21, 2017
9c289a5
[SPARK-22387][SQL] Propagate session configs to data source read/writ…
jiangxb1987 Dec 21, 2017
d3ae3e1
[SPARK-19634][SQL][ML][FOLLOW-UP] Improve interface of dataframe vect…
WeichenXu123 Dec 21, 2017
cb9fc8d
[SPARK-22848][SQL] Eliminate mutable state from Stack
kiszk Dec 21, 2017
59d5263
[SPARK-22324][SQL][PYTHON] Upgrade Arrow to 0.8.0
BryanCutler Dec 21, 2017
0abaf31
[SPARK-22852][BUILD] Exclude -Xlint:unchecked from sbt javadoc flags
easel Dec 21, 2017
4c2efde
[SPARK-22855][BUILD] Add -no-java-comments to sbt docs/scalacOptions
easel Dec 21, 2017
8a0ed5a
[SPARK-22668][SQL] Ensure no global variables in arguments of method …
cloud-fan Dec 21, 2017
d3a1d95
[SPARK-22786][SQL] only use AppStatusPlugin in history server
cloud-fan Dec 21, 2017
4e107fd
[SPARK-22822][TEST] Basic tests for WindowFrameCoercion and DecimalPr…
wangyum Dec 21, 2017
fe65361
[SPARK-22042][FOLLOW-UP][SQL] ReorderJoinPredicates can break when ch…
tejasapatil Dec 21, 2017
7beb375
[SPARK-22861][SQL] SQLAppStatusListener handles multi-job executions.
squito Dec 21, 2017
7ab165b
[SPARK-22648][K8S] Spark on Kubernetes - Documentation
foxish Dec 22, 2017
c0abb1d
[SPARK-22854][UI] Read Spark version from event logs.
Dec 22, 2017
c6f01ca
[SPARK-22750][SQL] Reuse mutable states when possible
mgaido91 Dec 22, 2017
a36b78b
[SPARK-22450][CORE][MLLIB][FOLLOWUP] safely register class for mllib …
zhengruifeng Dec 22, 2017
22e1849
[SPARK-22866][K8S] Fix path issue in Kubernetes dockerfile
foxish Dec 22, 2017
8df1da3
[SPARK-22862] Docs on lazy elimination of columns missing from an enc…
marmbrus Dec 22, 2017
13190a4
[SPARK-22874][PYSPARK][SQL] Modify checking pandas version to use Loo…
ueshin Dec 22, 2017
d23dc5b
[SPARK-22346][ML] VectorSizeHint Transformer for using VectorAssemble…
MrBago Dec 22, 2017
d3cbbdd
[SPARK-22757][Kubernetes] Enable use of remote dependencies in Kubern…
liyinan926 Dec 12, 2017
5d2cbc8
Addressed first round of comments
liyinan926 Dec 15, 2017
4ee76af
Addressed the second round of comments
liyinan926 Dec 16, 2017
9c8051a
Create one task per jar/file to download in the init-container
liyinan926 Dec 16, 2017
1f65417
More review comments
liyinan926 Dec 18, 2017
109ad80
Shorten variable names
liyinan926 Dec 19, 2017
c21fdcf
Removed traits that have only a single implementation
liyinan926 Dec 19, 2017
a3cd71d
Remove unused class arguments
liyinan926 Dec 19, 2017
23c5cd9
Improved documentation
liyinan926 Dec 19, 2017
2ec15c4
Addressed latest round of comments
liyinan926 Dec 20, 2017
5d1f889
Addressed more comments
liyinan926 Dec 21, 2017
9d9c841
Updated names of two configuration properties
liyinan926 Dec 22, 2017
c51bc56
Addressed more comments
liyinan926 Dec 25, 2017
28343fb
Addressed one more comment
liyinan926 Dec 26, 2017
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Addressed more comments
  • Loading branch information
liyinan926 committed Dec 23, 2017
commit 5d1f889d9b930ac06f6902ae4aa80575a6dc872d
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ private[spark] object KubernetesUtils {
}

/**
* Append the given init-container to a pod's list of init-containers..
* Append the given init-container to a pod's list of init-containers.
*
* @param originalPodSpec original specification of the pod
* @param initContainer the init-container to add to the pod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,12 @@ private[spark] class MountSecretsBootstrap(secretNamesToMountPaths: Map[String,
}

var containerBuilder = new ContainerBuilder(container)
secretNamesToMountPaths.foreach {
case (name, path) =>
containerBuilder = containerBuilder
.addNewVolumeMount()
.withName(secretVolumeName(name))
.withMountPath(path)
.endVolumeMount()
secretNamesToMountPaths.foreach { case (name, path) =>
containerBuilder = containerBuilder
.addNewVolumeMount()
.withName(secretVolumeName(name))
.withMountPath(path)
.endVolumeMount()
}

(podBuilder.build(), containerBuilder.build())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,7 @@ private[spark] class DependencyResolutionStep(

override def configureDriver(driverSpec: KubernetesDriverSpec): KubernetesDriverSpec = {
val resolvedSparkJars = KubernetesUtils.resolveFileUris(sparkJars, jarsDownloadPath)
val resolvedSparkFiles = KubernetesUtils.resolveFileUris(
sparkFiles, filesDownloadPath)
val resolvedSparkFiles = KubernetesUtils.resolveFileUris(sparkFiles, filesDownloadPath)

val sparkConf = driverSpec.driverSparkConf.clone()
if (resolvedSparkJars.nonEmpty) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,8 @@ private[spark] class SparkPodInitContainer(
private implicit val downloadExecutor = ExecutionContext.fromExecutorService(
ThreadUtils.newDaemonCachedThreadPool("download-executor", maxThreadPoolSize))

private val jarsDownloadDir = new File(
sparkConf.get(JARS_DOWNLOAD_LOCATION))
private val filesDownloadDir = new File(
sparkConf.get(FILES_DOWNLOAD_LOCATION))
private val jarsDownloadDir = new File(sparkConf.get(JARS_DOWNLOAD_LOCATION))
private val filesDownloadDir = new File(sparkConf.get(FILES_DOWNLOAD_LOCATION))

private val remoteJars = sparkConf.get(INIT_CONTAINER_REMOTE_JARS)
private val remoteFiles = sparkConf.get(INIT_CONTAINER_REMOTE_FILES)
Expand Down Expand Up @@ -82,7 +80,6 @@ private[spark] class SparkPodInitContainer(
Future[Unit] {
fileFetcher.fetchFile(file, downloadDir)
}

}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ class DriverInitContainerBootstrapStepSuite extends SparkFunSuite {
SecondTestInitContainerConfigurationStep.additionalInitContainerPropertyValue)
val hasMatchingProperties = initContainerPropertiesMap == expectedInitContainerProperties
hasMatchingName && hasCorrectNumberOfEntries && hasMatchingProperties

case _ => false
})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class DriverMountSecretsStepSuite extends SparkFunSuite {
private val SECRET_BAR = "bar"
private val SECRET_MOUNT_PATH = "/etc/secrets/driver"

test("Mounts all given secrets") {
test("mounts all given secrets") {
val baseDriverSpec = KubernetesDriverSpec.initialSpec(new SparkConf(false))
val secretNamesToMountPaths = Map(
SECRET_FOO -> SECRET_MOUNT_PATH,
Expand All @@ -38,10 +38,12 @@ class DriverMountSecretsStepSuite extends SparkFunSuite {
val driverPodWithSecretsMounted = configuredDriverSpec.driverPod
val driverContainerWithSecretsMounted = configuredDriverSpec.driverContainer

Seq(s"$SECRET_FOO-volume", s"$SECRET_BAR-volume").foreach(volumeName =>
assert(SecretVolumeUtils.podHasVolume(driverPodWithSecretsMounted, volumeName)))
Seq(s"$SECRET_FOO-volume", s"$SECRET_BAR-volume").foreach(volumeName =>
Seq(s"$SECRET_FOO-volume", s"$SECRET_BAR-volume").foreach { volumeName =>
assert(SecretVolumeUtils.podHasVolume(driverPodWithSecretsMounted, volumeName))
}
Seq(s"$SECRET_FOO-volume", s"$SECRET_BAR-volume").foreach { volumeName =>
assert(SecretVolumeUtils.containerHasVolume(
driverContainerWithSecretsMounted, volumeName, SECRET_MOUNT_PATH)))
driverContainerWithSecretsMounted, volumeName, SECRET_MOUNT_PATH))
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ class BasicInitContainerConfigurationStepSuite extends SparkFunSuite with Before
)}})
}

test("Test of additionalDriverSparkConf with mix of remote files and jars") {
test("additionalDriverSparkConf with mix of remote files and jars") {
val baseInitStep = new BasicInitContainerConfigurationStep(
SPARK_JARS,
SPARK_FILES,
Expand All @@ -88,8 +88,8 @@ class BasicInitContainerConfigurationStepSuite extends SparkFunSuite with Before
Seq.empty[HasMetadata])
val returnContainerSpec = baseInitStep.configureInitContainer(initContainerSpec)
assert(expectedDriverSparkConf === returnContainerSpec.properties)
assert(returnContainerSpec.initContainer.getName == INIT_CONTAINER_NAME)
assert(returnContainerSpec.driverContainer.getName == DRIVER_CONTAINER_NAME)
assert(returnContainerSpec.initContainer.getName === INIT_CONTAINER_NAME)
assert(returnContainerSpec.driverContainer.getName === DRIVER_CONTAINER_NAME)
assert(returnContainerSpec.driverPod.getMetadata.getLabels.asScala === POD_LABEL)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class InitContainerConfigOrchestratorSuite extends SparkFunSuite {
private val SECRET_BAR = "bar"
private val SECRET_MOUNT_PATH = "/etc/secrets/init-container"

test ("including basic configuration step") {
test("including basic configuration step") {
val sparkConf = new SparkConf(true)
.set(INIT_CONTAINER_IMAGE, DOCKER_IMAGE)
.set(s"$KUBERNETES_DRIVER_LABEL_PREFIX$CUSTOM_LABEL_KEY", CUSTOM_LABEL_VALUE)
Expand All @@ -52,8 +52,7 @@ class InitContainerConfigOrchestratorSuite extends SparkFunSuite {
INIT_CONTAINER_CONFIG_MAP_NAME,
INIT_CONTAINER_CONFIG_MAP_KEY,
sparkConf)
val initSteps : Seq[InitContainerConfigurationStep] =
orchestrator.getAllConfigurationSteps
val initSteps = orchestrator.getAllConfigurationSteps
assert(initSteps.lengthCompare(1) == 0)
assert(initSteps.head.isInstanceOf[BasicInitContainerConfigurationStep])
}
Expand All @@ -73,8 +72,7 @@ class InitContainerConfigOrchestratorSuite extends SparkFunSuite {
INIT_CONTAINER_CONFIG_MAP_NAME,
INIT_CONTAINER_CONFIG_MAP_KEY,
sparkConf)
val initSteps : Seq[InitContainerConfigurationStep] =
orchestrator.getAllConfigurationSteps
val initSteps = orchestrator.getAllConfigurationSteps
assert(initSteps.length === 2)
assert(initSteps.head.isInstanceOf[BasicInitContainerConfigurationStep])
assert(initSteps(1).isInstanceOf[InitContainerMountSecretsStep])
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class InitContainerMountSecretsStepSuite extends SparkFunSuite {
private val SECRET_BAR = "bar"
private val SECRET_MOUNT_PATH = "/etc/secrets/init-container"

test("Mounts all given secrets") {
test("mounts all given secrets") {
val baseInitContainerSpec = InitContainerSpec(
Map.empty,
Map.empty,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,7 @@ import org.apache.spark.{SparkConf, SparkFunSuite}
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.util.Utils

class SparkPodInitContainerSuite
extends SparkFunSuite with BeforeAndAfter {

import SparkPodInitContainerSuite.createTempFile
class SparkPodInitContainerSuite extends SparkFunSuite with BeforeAndAfter {

private val DOWNLOAD_JARS_SECRET_LOCATION = createTempFile("txt")
private val DOWNLOAD_FILES_SECRET_LOCATION = createTempFile("txt")
Expand Down Expand Up @@ -79,10 +76,8 @@ class SparkPodInitContainerSuite
.set(JARS_DOWNLOAD_LOCATION, downloadJarsDir.getAbsolutePath)
.set(FILES_DOWNLOAD_LOCATION, downloadFilesDir.getAbsolutePath)
}
}

private object SparkPodInitContainerSuite {
def createTempFile(extension: String): String = {
private def createTempFile(extension: String): String = {
val dir = Utils.createTempDir()
val file = new File(dir, s"${UUID.randomUUID().toString}.$extension")
Files.write(UUID.randomUUID().toString, file, Charsets.UTF_8)
Expand Down