Skip to content

[SPARK-24421][BUILD][CORE] Accessing sun.misc.Cleaner in JDK11 #22993

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
srowen wants to merge 6 commits into from
Closed

[SPARK-24421][BUILD][CORE] Accessing sun.misc.Cleaner in JDK11 #22993

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
ebf280d
Avoid sun.misc.Cleaner in JDK 9+ while retaining Java 8 compatibility…
srowen Nov 9, 2018
f137de7
Update deps
srowen Nov 9, 2018
2db719c
Fix compile error in hive-thriftserver
srowen Nov 10, 2018
7f58ae6
Split java.version on non-digit char
srowen Nov 11, 2018
9d775bb
Use Utils.classForName and lang3 for Java version
srowen Nov 13, 2018
2ee58b2
Update lang3 deps
srowen Nov 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Use Utils.classForName and lang3 for Java version
  • Loading branch information
@srowen
srowen committed Nov 13, 2018
commit 9d775bb9650307a507a848146e05c2b8826630f6
27 changes: 12 additions & 15 deletions core/src/main/scala/org/apache/spark/storage/StorageUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@ import java.nio.{ByteBuffer, MappedByteBuffer}
import scala.collection.Map
import scala.collection.mutable

import org.apache.commons.lang3.{JavaVersion, SystemUtils}
import sun.misc.Unsafe
import sun.nio.ch.DirectBuffer

import org.apache.spark.internal.Logging
import org.apache.spark.util.Utils

/**
* Storage information for each BlockManager.
Expand Down Expand Up @@ -201,27 +203,22 @@ private[spark] object StorageUtils extends Logging {
// reflection. However sun.misc.Unsafe added a invokeCleaner() method in JDK 9+ and this is
// still accessible with reflection.
private val bufferCleaner: DirectBuffer => Unit =
// Split java.version on non-digit chars:
if (System.getProperty("java.version").split("\\D+").head.toInt < 9) {
// scalastyle:off classforname
val cleanerMethod = Class.forName("sun.misc.Cleaner").getMethod("clean")
// scalastyle:on classforname
(buffer: DirectBuffer) => {
if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) {
val cleanerMethod =
Utils.classForName("sun.misc.Unsafe").getMethod("invokeCleaner", classOf[ByteBuffer])
val unsafeField = classOf[Unsafe].getDeclaredField("theUnsafe")
unsafeField.setAccessible(true)
val unsafe = unsafeField.get(null).asInstanceOf[Unsafe]
buffer: DirectBuffer => cleanerMethod.invoke(unsafe, buffer)
} else {
val cleanerMethod = Utils.classForName("sun.misc.Cleaner").getMethod("clean")
buffer: DirectBuffer => {
// Careful to avoid the return type of .cleaner(), which changes with JDK
val cleaner: AnyRef = buffer.cleaner()
if (cleaner != null) {
cleanerMethod.invoke(cleaner)
}
}
} else {
// scalastyle:off classforname
val cleanerMethod =
Class.forName("sun.misc.Unsafe").getMethod("invokeCleaner", classOf[ByteBuffer])
// scalastyle:on classforname
val unsafeField = classOf[Unsafe].getDeclaredField("theUnsafe")
unsafeField.setAccessible(true)
val unsafe = unsafeField.get(null).asInstanceOf[Unsafe]
(buffer: DirectBuffer) => cleanerMethod.invoke(unsafe, buffer)
}

/**
Expand Down
7 changes: 2 additions & 5 deletions core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ import scala.util.Random

import com.google.common.io.Files
import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.SystemUtils
import org.apache.commons.lang3.{JavaVersion, SystemUtils}
import org.apache.commons.math3.stat.inference.ChiSquareTest
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path
Expand Down Expand Up @@ -932,10 +932,7 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties with Logging {
signal(pid, "SIGKILL")
}

val versionParts = System.getProperty("java.version").split("[+.\\-]+", 3)
var majorVersion = versionParts(0).toInt
if (majorVersion == 1) majorVersion = versionParts(1).toInt
if (majorVersion >= 8) {
if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_1_8)) {
// We'll make sure that forcibly terminating a process works by
// creating a very misbehaving process. It ignores SIGTERM and has been SIGSTOPed. On
// older versions of java, this will *not* terminate.
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@
<!-- org.apache.commons/commons-lang/-->
<commons-lang2.version>2.6</commons-lang2.version>
<!-- org.apache.commons/commons-lang3/-->
<commons-lang3.version>3.5</commons-lang3.version>
<commons-lang3.version>3.8.1</commons-lang3.version>
<datanucleus-core.version>3.2.10</datanucleus-core.version>
<janino.version>3.0.10</janino.version>
<jersey.version>2.22.2</jersey.version>
Expand Down