Skip to content

Commit cf2e2e8

Browse files
HyukjinKwonSandishKumarHN
authored andcommitted
[SPARK-40654][PROTOBUF][FOLLOW-UP] Clean up SBT build in Proto component
### What changes were proposed in this pull request? This PR cleans up the syntax, and properly copy protobuf assembly jar (currently it copies connect assembly jar by mistake). ### Why are the changes needed? For consistent code style, and correct SBT build. ### Does this PR introduce _any_ user-facing change? No, this isn't released yet. ### How was this patch tested? CI in this PR should test it out. Closes apache#38240 from HyukjinKwon/SPARK-40654. Authored-by: Hyukjin Kwon <gurwls223@apache.org> Signed-off-by: Hyukjin Kwon <gurwls223@apache.org>
1 parent 72a2303 commit cf2e2e8

File tree

1 file changed

+9
-16
lines changed

1 file changed

+9
-16
lines changed

project/SparkBuild.scala

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ object BuildCommons {
5959
) = Seq(
6060
"core", "graphx", "mllib", "mllib-local", "repl", "network-common", "network-shuffle", "launcher", "unsafe",
6161
"tags", "sketch", "kvstore"
62-
).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connect) ++ Seq(protobuf)
62+
).map(ProjectRef(buildLocation, _)) ++ sqlProjects ++ streamingProjects ++ Seq(connect)
6363

6464
val optionallyEnabledProjects@Seq(kubernetes, mesos, yarn,
6565
sparkGangliaLgpl, streamingKinesisAsl,
@@ -433,7 +433,7 @@ object SparkBuild extends PomBuild {
433433

434434
enable(SparkConnect.settings)(connect)
435435

436-
/* Connector/proto settings */
436+
/* Protobuf settings */
437437
enable(SparkProtobuf.settings)(protobuf)
438438

439439
// SPARK-14738 - Remove docker tests from main Spark build
@@ -703,32 +703,24 @@ object SparkConnect {
703703
}
704704

705705
object SparkProtobuf {
706-
707706
import BuildCommons.protoVersion
708707

709-
private val shadePrefix = "org.sparkproject.spark-protobuf"
710-
val shadeJar = taskKey[Unit]("Shade the Jars")
711-
712708
lazy val settings = Seq(
713709
// Setting version for the protobuf compiler. This has to be propagated to every sub-project
714710
// even if the project is not using it.
715711
PB.protocVersion := BuildCommons.protoVersion,
716712

717713
// For some reason the resolution from the imported Maven build does not work for some
718714
// of these dependendencies that we need to shade later on.
719-
libraryDependencies ++= Seq(
720-
"com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf"
721-
),
715+
libraryDependencies += "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf",
722716

723-
dependencyOverrides ++= Seq(
724-
"com.google.protobuf" % "protobuf-java" % protoVersion
725-
),
717+
dependencyOverrides += "com.google.protobuf" % "protobuf-java" % protoVersion,
726718

727719
(Compile / PB.targets) := Seq(
728-
PB.gens.java -> (Compile / sourceManaged).value,
720+
PB.gens.java -> (Compile / sourceManaged).value,
729721
),
730722

731-
(assembly / test) := false,
723+
(assembly / test) := { },
732724

733725
(assembly / logLevel) := Level.Info,
734726

@@ -744,6 +736,7 @@ object SparkProtobuf {
744736
},
745737
)
746738
}
739+
747740
object Unsafe {
748741
lazy val settings = Seq(
749742
// This option is needed to suppress warnings from sun.misc.Unsafe usage
@@ -1277,7 +1270,7 @@ object CopyDependencies {
12771270
// produce the shaded Jar which happens automatically in the case of Maven.
12781271
// Later, when the dependencies are copied, we manually copy the shaded Jar only.
12791272
val fid = (LocalProject("connect") / assembly).value
1280-
val fidProtobuf = (LocalProject("protobuf")/assembly).value
1273+
val fidProtobuf = (LocalProject("protobuf") / assembly).value
12811274

12821275
(Compile / dependencyClasspath).value.map(_.data)
12831276
.filter { jar => jar.isFile() }
@@ -1292,7 +1285,7 @@ object CopyDependencies {
12921285
Files.copy(fid.toPath, destJar.toPath)
12931286
} else if (jar.getName.contains("spark-protobuf") &&
12941287
!SbtPomKeys.profiles.value.contains("noshade-protobuf")) {
1295-
Files.copy(fid.toPath, destJar.toPath)
1288+
Files.copy(fidProtobuf.toPath, destJar.toPath)
12961289
} else {
12971290
Files.copy(jar.toPath(), destJar.toPath())
12981291
}

0 commit comments

Comments
 (0)