diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala index 926f3a3c114fc..8a349e0227ff6 100644 --- a/core/src/main/scala/org/apache/spark/util/Utils.scala +++ b/core/src/main/scala/org/apache/spark/util/Utils.scala @@ -2774,6 +2774,34 @@ private[spark] object Utils extends Logging { HashCodes.fromBytes(secretBytes).toString() } + /** + * Returns true if and only if the underlying class is a member class. + * + * Note: jdk8u throws a "Malformed class name" error if a given class is a deeply-nested + * inner class (See SPARK-34607 for details). This issue has already been fixed in jdk9+, so + * we can remove this helper method safely if we drop the support of jdk8u. + */ + def isMemberClass(cls: Class[_]): Boolean = { + try { + cls.isMemberClass + } catch { + case _: InternalError => + // We emulate jdk8u `Class.isMemberClass` below: + // public boolean isMemberClass() { + // return getSimpleBinaryName() != null && !isLocalOrAnonymousClass(); + // } + // `getSimpleBinaryName()` returns null if a given class is a top-level class, + // so we replace it with `cls.getEnclosingClass != null`. The second condition checks + // if a given class is not a local or an anonymous class, so we replace it with + // `cls.getEnclosingMethod == null` because `cls.getEnclosingMethod()` return a value + // only in either case (JVM Spec 4.8.6). + // + // Note: The newer jdk evaluates `!isLocalOrAnonymousClass()` first, + // we reorder the conditions to follow it. + cls.getEnclosingMethod == null && cls.getEnclosingClass != null + } + } + /** * Safer than Class obj's getSimpleName which may throw Malformed class name error in scala. * This method mimicks scalatest's getSimpleNameOfAnObjectsClass. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala index a1f0312bd853c..1a16edc5e68d6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/OuterScopes.scala @@ -48,7 +48,7 @@ object OuterScopes { * useful for inner class defined in REPL. */ def getOuterScope(innerCls: Class[_]): () => AnyRef = { - assert(innerCls.isMemberClass) + assert(Utils.isMemberClass(innerCls)) val outerClassName = innerCls.getDeclaringClass.getName val outer = outerScopes.get(outerClassName) if (outer == null) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index 72a60f0cb5c5f..d29697a6e65d6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -446,7 +446,7 @@ case class NewInstance( // Note that static inner classes (e.g., inner classes within Scala objects) don't need // outer pointer registration. val needOuterPointer = - outerPointer.isEmpty && cls.isMemberClass && !Modifier.isStatic(cls.getModifiers) + outerPointer.isEmpty && Utils.isMemberClass(cls) && !Modifier.isStatic(cls.getModifiers) childrenResolved && !needOuterPointer } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala index efce0097c12b0..227c31b358108 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/ExpressionEncoderSuite.scala @@ -233,6 +233,76 @@ class ExpressionEncoderSuite extends CodegenInterpretedPlanTest with AnalysisTes "nested Scala class") } + object OuterLevelWithVeryVeryVeryLongClassName1 { + object OuterLevelWithVeryVeryVeryLongClassName2 { + object OuterLevelWithVeryVeryVeryLongClassName3 { + object OuterLevelWithVeryVeryVeryLongClassName4 { + object OuterLevelWithVeryVeryVeryLongClassName5 { + object OuterLevelWithVeryVeryVeryLongClassName6 { + object OuterLevelWithVeryVeryVeryLongClassName7 { + object OuterLevelWithVeryVeryVeryLongClassName8 { + object OuterLevelWithVeryVeryVeryLongClassName9 { + object OuterLevelWithVeryVeryVeryLongClassName10 { + object OuterLevelWithVeryVeryVeryLongClassName11 { + object OuterLevelWithVeryVeryVeryLongClassName12 { + object OuterLevelWithVeryVeryVeryLongClassName13 { + object OuterLevelWithVeryVeryVeryLongClassName14 { + object OuterLevelWithVeryVeryVeryLongClassName15 { + object OuterLevelWithVeryVeryVeryLongClassName16 { + object OuterLevelWithVeryVeryVeryLongClassName17 { + object OuterLevelWithVeryVeryVeryLongClassName18 { + object OuterLevelWithVeryVeryVeryLongClassName19 { + object OuterLevelWithVeryVeryVeryLongClassName20 { + case class MalformedNameExample(x: Int) + }}}}}}}}}}}}}}}}}}}} + + { + OuterScopes.addOuterScope( + OuterLevelWithVeryVeryVeryLongClassName1 + .OuterLevelWithVeryVeryVeryLongClassName2 + .OuterLevelWithVeryVeryVeryLongClassName3 + .OuterLevelWithVeryVeryVeryLongClassName4 + .OuterLevelWithVeryVeryVeryLongClassName5 + .OuterLevelWithVeryVeryVeryLongClassName6 + .OuterLevelWithVeryVeryVeryLongClassName7 + .OuterLevelWithVeryVeryVeryLongClassName8 + .OuterLevelWithVeryVeryVeryLongClassName9 + .OuterLevelWithVeryVeryVeryLongClassName10 + .OuterLevelWithVeryVeryVeryLongClassName11 + .OuterLevelWithVeryVeryVeryLongClassName12 + .OuterLevelWithVeryVeryVeryLongClassName13 + .OuterLevelWithVeryVeryVeryLongClassName14 + .OuterLevelWithVeryVeryVeryLongClassName15 + .OuterLevelWithVeryVeryVeryLongClassName16 + .OuterLevelWithVeryVeryVeryLongClassName17 + .OuterLevelWithVeryVeryVeryLongClassName18 + .OuterLevelWithVeryVeryVeryLongClassName19 + .OuterLevelWithVeryVeryVeryLongClassName20) + checkCompilationError( + OuterLevelWithVeryVeryVeryLongClassName1 + .OuterLevelWithVeryVeryVeryLongClassName2 + .OuterLevelWithVeryVeryVeryLongClassName3 + .OuterLevelWithVeryVeryVeryLongClassName4 + .OuterLevelWithVeryVeryVeryLongClassName5 + .OuterLevelWithVeryVeryVeryLongClassName6 + .OuterLevelWithVeryVeryVeryLongClassName7 + .OuterLevelWithVeryVeryVeryLongClassName8 + .OuterLevelWithVeryVeryVeryLongClassName9 + .OuterLevelWithVeryVeryVeryLongClassName10 + .OuterLevelWithVeryVeryVeryLongClassName11 + .OuterLevelWithVeryVeryVeryLongClassName12 + .OuterLevelWithVeryVeryVeryLongClassName13 + .OuterLevelWithVeryVeryVeryLongClassName14 + .OuterLevelWithVeryVeryVeryLongClassName15 + .OuterLevelWithVeryVeryVeryLongClassName16 + .OuterLevelWithVeryVeryVeryLongClassName17 + .OuterLevelWithVeryVeryVeryLongClassName18 + .OuterLevelWithVeryVeryVeryLongClassName19 + .OuterLevelWithVeryVeryVeryLongClassName20 + .MalformedNameExample(42), + "deeply nested Scala class") + } + productTest(PrimitiveData(1, 1, 1, 1, 1, 1, true)) productTest(