From 6499b0d7153b1e29bd026b8a46d19ff1d96571f9 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 14 Oct 2025 15:00:23 +0800 Subject: [PATCH 01/10] fix bytecode attributes for patched files --- project/ScalaLibraryPlugin.scala | 27 ++++++++++++++++++++++++--- project/build.sbt | 1 + 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index 3310ff94bdeb..a692a6856606 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -7,6 +7,7 @@ import java.nio.file.Files import xsbti.VirtualFileRef import sbt.internal.inc.Stamper import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport.scalaJSVersion +import org.objectweb.asm.* object ScalaLibraryPlugin extends AutoPlugin { @@ -56,8 +57,8 @@ object ScalaLibraryPlugin extends AutoPlugin { dest = target / (id.toString) ref <- dest.relativeTo((LocalRootProject / baseDirectory).value) } { - // Copy the files to the classDirectory - IO.copyFile(file, dest) + // Read -> Strip Scala 2 Pickles -> Write + IO.write(dest, unpickler(IO.readBytes(file))) // Update the timestamp in the analysis stamps = stamps.markProduct( VirtualFileRef.of(s"$${BASE}/$ref"), @@ -77,7 +78,8 @@ object ScalaLibraryPlugin extends AutoPlugin { // Copy all the specialized classes in the stdlib // no need to update any stamps as these classes exist nowhere in the analysis for (orig <- diff; dest <- orig.relativeTo(reference)) { - IO.copyFile(orig, ((Compile / classDirectory).value / dest.toString())) + // Read -> Strip Scala 2 Pickles -> Write + IO.write((Compile / classDirectory).value / dest.toString, unpickler(IO.readBytes(orig))) } } @@ -103,6 +105,25 @@ object ScalaLibraryPlugin extends AutoPlugin { } (Set(jar)), target) } + /* Remove Scala 2 Pickles from Classfiles */ + private def unpickler(bytes: Array[Byte]): Array[Byte] = { + val reader = new ClassReader(bytes) + val writer = new ClassWriter(0) + val visitor = new ClassVisitor(Opcodes.ASM9, writer) { + override def visitAttribute(attr: Attribute): Unit = attr.`type` match { + case "ScalaSig" | "ScalaInlineInfo" => () + case _ => super.visitAttribute(attr) + } + + override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = + if (desc == "Lscala/reflect/ScalaSignature;" || desc == "Lscala/reflect/ScalaLongSignature;") null + else super.visitAnnotation(desc, visible) + + } + reader.accept(visitor, 0) + writer.toByteArray + } + private lazy val filesToCopy = Set( "scala/Tuple1", "scala/Tuple2", diff --git a/project/build.sbt b/project/build.sbt index 9e96a2327deb..e4de5f54d459 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -1,5 +1,6 @@ // Used by VersionUtil to get gitHash and commitDate libraryDependencies += "org.eclipse.jgit" % "org.eclipse.jgit" % "4.11.0.201803080745-r" +libraryDependencies += "org.ow2.asm" % "asm" % "9.9" libraryDependencies += Dependencies.`jackson-databind` From 57dac79fbeb0c5f2da703edfbde190b63eb507f5 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Sun, 28 Dec 2025 22:00:40 +0100 Subject: [PATCH 02/10] Add validation to ensure Scala 2 pickles are not stored --- project/ScalaLibraryPlugin.scala | 74 +++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index a692a6856606..482a2632588c 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -2,6 +2,7 @@ package dotty.tools.sbtplugin import sbt.* import sbt.Keys.* +import sbt.io.Using import scala.jdk.CollectionConverters.* import java.nio.file.Files import xsbti.VirtualFileRef @@ -15,6 +16,16 @@ object ScalaLibraryPlugin extends AutoPlugin { private val scala2Version = "2.13.16" + /** Scala 2 pickle annotation descriptors that should be stripped from class files */ + private val Scala2PickleAnnotations = Set( + "Lscala/reflect/ScalaSignature;", + "Lscala/reflect/ScalaLongSignature;" + ) + + /** Check if an annotation descriptor is a Scala 2 pickle annotation */ + private def isScala2PickleAnnotation(descriptor: String): Boolean = + Scala2PickleAnnotations.contains(descriptor) + object autoImport { val keepSJSIR = settingKey[Boolean]("Should we patch .sjsir too?") } @@ -22,6 +33,10 @@ object ScalaLibraryPlugin extends AutoPlugin { import autoImport._ override def projectSettings = Seq ( + // Settings to validate that JARs don't contain Scala 2 pickle annotations + Compile / packageBin := (Compile / packageBin) + .map(validateNoScala2Pickles) + .value, (Compile / manipulateBytecode) := { val stream = streams.value val target = (Compile / classDirectory).value @@ -57,8 +72,7 @@ object ScalaLibraryPlugin extends AutoPlugin { dest = target / (id.toString) ref <- dest.relativeTo((LocalRootProject / baseDirectory).value) } { - // Read -> Strip Scala 2 Pickles -> Write - IO.write(dest, unpickler(IO.readBytes(file))) + patchFile(input = file, output = dest) // Update the timestamp in the analysis stamps = stamps.markProduct( VirtualFileRef.of(s"$${BASE}/$ref"), @@ -78,8 +92,10 @@ object ScalaLibraryPlugin extends AutoPlugin { // Copy all the specialized classes in the stdlib // no need to update any stamps as these classes exist nowhere in the analysis for (orig <- diff; dest <- orig.relativeTo(reference)) { - // Read -> Strip Scala 2 Pickles -> Write - IO.write((Compile / classDirectory).value / dest.toString, unpickler(IO.readBytes(orig))) + patchFile( + input = orig, + output = (Compile / classDirectory).value / dest.toString() + ) } } @@ -116,14 +132,60 @@ object ScalaLibraryPlugin extends AutoPlugin { } override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = - if (desc == "Lscala/reflect/ScalaSignature;" || desc == "Lscala/reflect/ScalaLongSignature;") null + if (isScala2PickleAnnotation(desc)) null else super.visitAnnotation(desc, visible) - } reader.accept(visitor, 0) writer.toByteArray } + // Apply the patches to given input file and write the result to the output + def patchFile(input: File, output: File): File = { + if (input.getName.endsWith(".class")) { + IO.write(output, unpickler(IO.readBytes(input))) + } else { + // For .sjsir files, we just copy the file + IO.copyFile(input, output) + } + output + } + + /** Check if class file bytecode contains Scala 2 pickle annotations */ + private def hasScala2Pickles(bytes: Array[Byte]): Boolean = { + var found = false + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = { + if (isScala2PickleAnnotation(desc)) found = true + null + } + } + new ClassReader(bytes).accept( + visitor, + ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES + ) + found + } + + def validateNoScala2Pickles(jar: File): File = { + val classFilesWithPickles = Using.jarFile(verify = true)(jar){ jarFile => + jarFile + .entries().asScala + .filter(_.getName.endsWith(".class")) + .flatMap { entry => + Using.bufferedInputStream(jarFile.getInputStream(entry)){ inputStream => + if (hasScala2Pickles(inputStream.readAllBytes())) Some(entry.getName) + else None + } + } + .toList + } + assert( + classFilesWithPickles.isEmpty, + s"JAR ${jar.getName} contains ${classFilesWithPickles.size} class files with Scala 2 pickle annotations: ${classFilesWithPickles.mkString("\n - ", "\n - ", "")}" + ) + jar + } + private lazy val filesToCopy = Set( "scala/Tuple1", "scala/Tuple2", From a7e214ecea3687c499eb4e45fd7bfe479c619de5 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Sun, 28 Dec 2025 22:07:18 +0100 Subject: [PATCH 03/10] Rename unpickler to removeScala2Pickles --- project/ScalaLibraryPlugin.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index 482a2632588c..b5e7c3ad0ba1 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -122,7 +122,7 @@ object ScalaLibraryPlugin extends AutoPlugin { } /* Remove Scala 2 Pickles from Classfiles */ - private def unpickler(bytes: Array[Byte]): Array[Byte] = { + private def removeScala2Pickles(bytes: Array[Byte]): Array[Byte] = { val reader = new ClassReader(bytes) val writer = new ClassWriter(0) val visitor = new ClassVisitor(Opcodes.ASM9, writer) { @@ -142,7 +142,7 @@ object ScalaLibraryPlugin extends AutoPlugin { // Apply the patches to given input file and write the result to the output def patchFile(input: File, output: File): File = { if (input.getName.endsWith(".class")) { - IO.write(output, unpickler(IO.readBytes(input))) + IO.write(output, removeScala2Pickles(IO.readBytes(input))) } else { // For .sjsir files, we just copy the file IO.copyFile(input, output) From ac420c3ec9cb8257a73192ee6722495a7e899bb2 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Mon, 29 Dec 2025 02:27:09 +0100 Subject: [PATCH 04/10] Add validation for existance of TASTy attribute. Add synthetic TASTY attribute for copied .class files --- project/ScalaLibraryPlugin.scala | 203 ++++++++++++++++-- project/build.sbt | 10 +- project/stubs.scala | 9 + .../src/dotty/tools/tasty/TastyVersion.scala | 2 +- 4 files changed, 207 insertions(+), 17 deletions(-) create mode 100644 project/stubs.scala diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index b5e7c3ad0ba1..553553d14a94 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -4,12 +4,16 @@ import sbt.* import sbt.Keys.* import sbt.io.Using import scala.jdk.CollectionConverters.* +import scala.collection.mutable import java.nio.file.Files +import java.nio.ByteBuffer import xsbti.VirtualFileRef import sbt.internal.inc.Stamper import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport.scalaJSVersion import org.objectweb.asm.* +import dotty.tools.tasty.TastyHeaderUnpickler + object ScalaLibraryPlugin extends AutoPlugin { override def trigger = noTrigger @@ -33,9 +37,13 @@ object ScalaLibraryPlugin extends AutoPlugin { import autoImport._ override def projectSettings = Seq ( - // Settings to validate that JARs don't contain Scala 2 pickle annotations + // Settings to validate that JARs don't contain Scala 2 pickle annotations and have valid TASTY attributes Compile / packageBin := (Compile / packageBin) - .map(validateNoScala2Pickles) + .map{ jar => + validateNoScala2Pickles(jar) + validateTastyAttributes(jar) + jar + } .value, (Compile / manipulateBytecode) := { val stream = streams.value @@ -62,6 +70,9 @@ object ScalaLibraryPlugin extends AutoPlugin { } var stamps = analysis.stamps + val classDir = (Compile / classDirectory).value + val sourceDir = sourceDirectory.value + // Patch the files that are in the list for { (files, reference) <- patches @@ -72,7 +83,7 @@ object ScalaLibraryPlugin extends AutoPlugin { dest = target / (id.toString) ref <- dest.relativeTo((LocalRootProject / baseDirectory).value) } { - patchFile(input = file, output = dest) + patchFile(input = file, output = dest, classDirectory = classDir, sourceDirectory = sourceDir) // Update the timestamp in the analysis stamps = stamps.markProduct( VirtualFileRef.of(s"$${BASE}/$ref"), @@ -80,11 +91,11 @@ object ScalaLibraryPlugin extends AutoPlugin { } - val overwrittenBinaries = Files.walk((Compile / classDirectory).value.toPath()) + val overwrittenBinaries = Files.walk(classDir.toPath()) .iterator() .asScala .map(_.toFile) - .map(_.relativeTo((Compile / classDirectory).value).get) + .map(_.relativeTo(classDir).get) .toSet for ((files, reference) <- patches) { @@ -94,7 +105,9 @@ object ScalaLibraryPlugin extends AutoPlugin { for (orig <- diff; dest <- orig.relativeTo(reference)) { patchFile( input = orig, - output = (Compile / classDirectory).value / dest.toString() + output = classDir / dest.toString(), + classDirectory = classDir, + sourceDirectory = sourceDir ) } } @@ -121,10 +134,15 @@ object ScalaLibraryPlugin extends AutoPlugin { } (Set(jar)), target) } - /* Remove Scala 2 Pickles from Classfiles */ - private def removeScala2Pickles(bytes: Array[Byte]): Array[Byte] = { + /** Remove Scala 2 Pickles from class file and optionally add TASTY attribute. + * + * @param bytes the class file bytecode + * @param tastyUUID optional 16-byte UUID from the corresponding .tasty file (only for primary class) + */ + private def patchClassFile(bytes: Array[Byte], tastyUUID: Option[Array[Byte]]): Array[Byte] = { val reader = new ClassReader(bytes) val writer = new ClassWriter(0) + // Remove Scala 2 pickles and Scala signatures val visitor = new ClassVisitor(Opcodes.ASM9, writer) { override def visitAttribute(attr: Attribute): Unit = attr.`type` match { case "ScalaSig" | "ScalaInlineInfo" => () @@ -136,17 +154,55 @@ object ScalaLibraryPlugin extends AutoPlugin { else super.visitAnnotation(desc, visible) } reader.accept(visitor, 0) + // Only add TASTY attribute for the primary class (not for inner/nested classes) + tastyUUID + .map(new TastyAttribute(_)) + .foreach(writer.visitAttribute) writer.toByteArray } - // Apply the patches to given input file and write the result to the output - def patchFile(input: File, output: File): File = { - if (input.getName.endsWith(".class")) { - IO.write(output, removeScala2Pickles(IO.readBytes(input))) - } else { + /** Apply the patches to given input file and write the result to the output. + * For .class files, strips Scala 2 pickles and adds TASTY attribute only for primary classes. + * + * The TASTY attribute is only added to the "primary" class for each .tasty file: + * - Inner/nested classes (e.g., Outer$Inner.class) don't get TASTY attribute + * - Companion objects (Foo$.class when Foo.class exists) don't get TASTY attribute + * - Only the class whose name matches the .tasty file name gets the attribute + * - Java source files don't produce .tasty files, so they are skipped + * + * @param input the input file (.class or .sjsir) + * @param output the output file location + * @param classDirectory the class directory to look for .tasty files + * @param sourceDirectory the source directory to check for .java files + */ + def patchFile(input: File, output: File, classDirectory: File, sourceDirectory: File): File = { + if (input.getName.endsWith(".sjsir")) { // For .sjsir files, we just copy the file IO.copyFile(input, output) + return output } + + val relativePath = output.relativeTo(classDirectory) + .getOrElse(sys.error(s"Patched file is not relative to class directory: $output")) + .getPath + val classPath = relativePath.stripSuffix(".class") + val basePath = classPath.split('$').head + val javaSourceFile = sourceDirectory / (basePath + ".java") + + // Skip TASTY handling for Java-sourced classes (they don't have .tasty files) + val tastyUUID = + if (javaSourceFile.exists()) None + else { + val tastyFile = classDirectory / (basePath + ".tasty") + assert(tastyFile.exists(), s"TASTY file $tastyFile does not exist for $relativePath / $javaSourceFile") + + // Only add TASTY attribute if this is the primary class (class path equals base path) + // Inner classes, companion objects ($), anonymous classes ($$anon), etc. don't get TASTY attribute + val isPrimaryClass = classPath == basePath + if (isPrimaryClass) Some(extractTastyUUID(IO.readBytes(tastyFile))) + else None + } + IO.write(output, patchClassFile(IO.readBytes(input), tastyUUID)) output } @@ -166,7 +222,7 @@ object ScalaLibraryPlugin extends AutoPlugin { found } - def validateNoScala2Pickles(jar: File): File = { + def validateNoScala2Pickles(jar: File): Unit = { val classFilesWithPickles = Using.jarFile(verify = true)(jar){ jarFile => jarFile .entries().asScala @@ -183,7 +239,6 @@ object ScalaLibraryPlugin extends AutoPlugin { classFilesWithPickles.isEmpty, s"JAR ${jar.getName} contains ${classFilesWithPickles.size} class files with Scala 2 pickle annotations: ${classFilesWithPickles.mkString("\n - ", "\n - ", "")}" ) - jar } private lazy val filesToCopy = Set( @@ -227,4 +282,122 @@ object ScalaLibraryPlugin extends AutoPlugin { "scala/util/Sorting", ) + /** Extract the UUID bytes (16 bytes) from a TASTy file. + * + * Uses the official TastyHeaderUnpickler to parse the header and extract the UUID, + * ensuring correctness and validating the TASTy format. + */ + private def extractTastyUUID(tastyBytes: Array[Byte]): Array[Byte] = { + val unpickler = new TastyHeaderUnpickler(tastyBytes) + val header = unpickler.readFullHeader() + val uuid = header.uuid + + // Convert UUID (two longs) to 16-byte array in big-endian format + val buffer = ByteBuffer.allocate(16) + buffer.putLong(uuid.getMostSignificantBits) + buffer.putLong(uuid.getLeastSignificantBits) + buffer.array() + } + + /** Extract TASTY UUID from class file bytecode, if present */ + private def extractTastyUUIDFromClass(bytes: Array[Byte]): Option[Array[Byte]] = { + val tastyAttr = new TastyAttributeReader() + var result: Option[Array[Byte]] = None + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitAttribute(attr: Attribute): Unit = { + attr match { + case t: TastyAttributeReader => result = t.uuid + case _ => () + } + } + } + new ClassReader(bytes).accept(visitor, Array[Attribute](tastyAttr), 0) + result + } + + /** Validate TASTY attributes in the JAR: + * - If a .class file has a TASTY attribute, verify its UUID matches a .tasty file in the JAR + * - Every .tasty file must have at least one .class file with a matching TASTY attribute + * + * Note: .class files from Java sources don't have .tasty files and are allowed to not have TASTY attributes. + */ + def validateTastyAttributes(jar: File): Unit = { + Using.jarFile(verify = true)(jar) { jarFile => + // Build a map of .tasty file paths to their UUIDs + val tastyEntries = jarFile.entries().asScala + .filter(_.getName.endsWith(".tasty")) + .map { entry => + val bytes = Using.bufferedInputStream(jarFile.getInputStream(entry))(_.readAllBytes()) + val uuid = extractTastyUUID(bytes) + entry.getName -> uuid + } + .toMap + + val errors = mutable.ListBuffer.empty[String] + val referencedTastyFiles = mutable.Set.empty[String] + + // Check each .class file that has a TASTY attribute + jarFile.entries().asScala + .filter(e => e.getName.endsWith(".class")) + .foreach { entry => + val classBytes = Using.bufferedInputStream(jarFile.getInputStream(entry))(_.readAllBytes()) + val classPath = entry.getName + + // Only validate classes that have a TASTY attribute + extractTastyUUIDFromClass(classBytes).foreach[Unit] { classUUID => + // Find a .tasty file with matching UUID + tastyEntries.find{ case (path, tastyUUID) => + java.util.Arrays.equals(classUUID, tastyUUID) && { + val tastyName = file(path).getName().stripSuffix(".tasty") + val className = file(entry.getName()).getName().stripSuffix(".class") + // apparently 2 files might have the same UUID, e.g. param.scala and field.scala + className.startsWith(tastyName) + }} match { + case Some((path, _)) => + referencedTastyFiles += path + case None => + val uuidHex = classUUID.map(b => f"$b%02x").mkString + errors += s"$classPath: has TASTY attribute (UUID=$uuidHex) but no matching .tasty file found in JAR" + } + } + } + + // Check that every .tasty file has at least one .class file referencing it + val unreferencedTastyFiles = tastyEntries.keySet -- referencedTastyFiles + unreferencedTastyFiles.foreach { tastyPath => + errors += s"$tastyPath: no .class file with matching TASTY attribute found" + } + + assert( + errors.isEmpty, + s"JAR ${jar.getName} has ${errors.size} TASTY validation errors:\n - ${errors.mkString("\n - ")}" + ) + } + } + + + /** Custom ASM Attribute for TASTY that can be written to class files */ + private class TastyAttribute(val uuid: Array[Byte]) extends Attribute("TASTY") { + override def write(classWriter: ClassWriter, code: Array[Byte], codeLength: Int, maxStack: Int, maxLocals: Int): ByteVector = { + val bv = new ByteVector(uuid.length) + bv.putByteArray(uuid, 0, uuid.length) + bv + } + } + /** Custom ASM Attribute for reading TASTY attributes from class files */ + private class TastyAttributeReader extends Attribute("TASTY") { + var uuid: Option[Array[Byte]] = None + + override def read(classReader: ClassReader, offset: Int, length: Int, charBuffer: Array[Char], codeOffset: Int, labels: Array[Label]): Attribute = { + val attr = new TastyAttributeReader() + if (length == 16) { + val bytes = new Array[Byte](16) + for (i <- 0 until 16) { + bytes(i) = classReader.readByte(offset + i).toByte + } + attr.uuid = Some(bytes) + } + attr + } + } } diff --git a/project/build.sbt b/project/build.sbt index e4de5f54d459..72216a873dbc 100644 --- a/project/build.sbt +++ b/project/build.sbt @@ -5,4 +5,12 @@ libraryDependencies += "org.ow2.asm" % "asm" % "9.9" libraryDependencies += Dependencies.`jackson-databind` // Used for manipulating YAML files in sidebar generation script -libraryDependencies += "org.yaml" % "snakeyaml" % "2.4" \ No newline at end of file +libraryDependencies += "org.yaml" % "snakeyaml" % "2.4" + +Compile / unmanagedSourceDirectories ++= { + val root = baseDirectory.value.getParentFile() + Seq( + root / "tasty/src", + root / "tasty/src/dotty/tools/tasty/util", + ) +} diff --git a/project/stubs.scala b/project/stubs.scala new file mode 100644 index 000000000000..e9299a3e9388 --- /dev/null +++ b/project/stubs.scala @@ -0,0 +1,9 @@ +// Stubs for Scala 3 stdlib required to compile build unmanaged sources + +package scala { + package annotation { + package internal { + class sharable extends Annotation + } + } +} diff --git a/tasty/src/dotty/tools/tasty/TastyVersion.scala b/tasty/src/dotty/tools/tasty/TastyVersion.scala index b6474f7c7934..e4dd1ad91af3 100644 --- a/tasty/src/dotty/tools/tasty/TastyVersion.scala +++ b/tasty/src/dotty/tools/tasty/TastyVersion.scala @@ -20,7 +20,7 @@ case class TastyVersion private(major: Int, minor: Int, experimental: Int) { def validRange: String = { val min = TastyVersion(major, 0, 0) val max = if (experimental == 0) this else TastyVersion(major, minor - 1, 0) - val extra = Option.when(experimental > 0)(this) + val extra = Option(this).filter(_ => experimental > 0) s"stable TASTy from ${min.show} to ${max.show}${extra.fold("")(e => s", or exactly ${e.show}")}" } } From d67339594ff4ba0551c69f747c286a4288f1e361 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Mon, 29 Dec 2025 10:13:32 +0100 Subject: [PATCH 05/10] Fix sjs builds - use fixed scala-library-nonboostrapped source directory --- project/ScalaLibraryPlugin.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index 553553d14a94..eee7d89054e8 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -71,7 +71,7 @@ object ScalaLibraryPlugin extends AutoPlugin { var stamps = analysis.stamps val classDir = (Compile / classDirectory).value - val sourceDir = sourceDirectory.value + val sourceDir = (LocalProject("scala-library-nonbootstrapped") / sourceDirectory).value // Patch the files that are in the list for { @@ -194,7 +194,7 @@ object ScalaLibraryPlugin extends AutoPlugin { if (javaSourceFile.exists()) None else { val tastyFile = classDirectory / (basePath + ".tasty") - assert(tastyFile.exists(), s"TASTY file $tastyFile does not exist for $relativePath / $javaSourceFile") + assert(tastyFile.exists(), s"TASTY file $tastyFile does not exist for $relativePath") // Only add TASTY attribute if this is the primary class (class path equals base path) // Inner classes, companion objects ($), anonymous classes ($$anon), etc. don't get TASTY attribute From c2d9e9e49ff7b003e1f7fee533ad19b9992f6ca5 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 30 Dec 2025 15:22:01 +0100 Subject: [PATCH 06/10] Remove dependency on scala-library-nonbootstraped project - detect .class files produced by javac based on the source attributes --- project/ScalaLibraryPlugin.scala | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index eee7d89054e8..c2a592ef4185 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -71,7 +71,6 @@ object ScalaLibraryPlugin extends AutoPlugin { var stamps = analysis.stamps val classDir = (Compile / classDirectory).value - val sourceDir = (LocalProject("scala-library-nonbootstrapped") / sourceDirectory).value // Patch the files that are in the list for { @@ -83,7 +82,7 @@ object ScalaLibraryPlugin extends AutoPlugin { dest = target / (id.toString) ref <- dest.relativeTo((LocalRootProject / baseDirectory).value) } { - patchFile(input = file, output = dest, classDirectory = classDir, sourceDirectory = sourceDir) + patchFile(input = file, output = dest, classDirectory = classDir) // Update the timestamp in the analysis stamps = stamps.markProduct( VirtualFileRef.of(s"$${BASE}/$ref"), @@ -107,7 +106,6 @@ object ScalaLibraryPlugin extends AutoPlugin { input = orig, output = classDir / dest.toString(), classDirectory = classDir, - sourceDirectory = sourceDir ) } } @@ -173,9 +171,8 @@ object ScalaLibraryPlugin extends AutoPlugin { * @param input the input file (.class or .sjsir) * @param output the output file location * @param classDirectory the class directory to look for .tasty files - * @param sourceDirectory the source directory to check for .java files */ - def patchFile(input: File, output: File, classDirectory: File, sourceDirectory: File): File = { + def patchFile(input: File, output: File, classDirectory: File): File = { if (input.getName.endsWith(".sjsir")) { // For .sjsir files, we just copy the file IO.copyFile(input, output) @@ -187,11 +184,12 @@ object ScalaLibraryPlugin extends AutoPlugin { .getPath val classPath = relativePath.stripSuffix(".class") val basePath = classPath.split('$').head - val javaSourceFile = sourceDirectory / (basePath + ".java") // Skip TASTY handling for Java-sourced classes (they don't have .tasty files) + val classfileBytes = IO.readBytes(input) + val isJavaSourced = extractSourceFile(classfileBytes).exists(_.endsWith(".java")) val tastyUUID = - if (javaSourceFile.exists()) None + if (isJavaSourced) None else { val tastyFile = classDirectory / (basePath + ".tasty") assert(tastyFile.exists(), s"TASTY file $tastyFile does not exist for $relativePath") @@ -202,7 +200,7 @@ object ScalaLibraryPlugin extends AutoPlugin { if (isPrimaryClass) Some(extractTastyUUID(IO.readBytes(tastyFile))) else None } - IO.write(output, patchClassFile(IO.readBytes(input), tastyUUID)) + IO.write(output, patchClassFile(classfileBytes, tastyUUID)) output } @@ -282,6 +280,21 @@ object ScalaLibraryPlugin extends AutoPlugin { "scala/util/Sorting", ) + /** Extract the SourceFile attribute from class file bytecode */ + private def extractSourceFile(bytes: Array[Byte]): Option[String] = { + var sourceFile: Option[String] = None + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitSource(source: String, debug: String): Unit = + sourceFile = Option(source) + } + // Note: Don't use SKIP_DEBUG here - SourceFile is debug info and would be skipped + new ClassReader(bytes).accept( + visitor, + ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES + ) + sourceFile + } + /** Extract the UUID bytes (16 bytes) from a TASTy file. * * Uses the official TastyHeaderUnpickler to parse the header and extract the UUID, From e831daadc6a1915a76886b285faf1b3ee224703c Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 30 Dec 2025 16:04:02 +0100 Subject: [PATCH 07/10] Ensure that Scala 2 pickle attributes are also removed --- project/ScalaLibraryPlugin.scala | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index c2a592ef4185..7262ed6ea85f 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -26,6 +26,9 @@ object ScalaLibraryPlugin extends AutoPlugin { "Lscala/reflect/ScalaLongSignature;" ) + /** Scala 2 attribute names that should be stripped from class files */ + private val Scala2PickleAttributes = Set("ScalaSig", "ScalaInlineInfo") + /** Check if an annotation descriptor is a Scala 2 pickle annotation */ private def isScala2PickleAnnotation(descriptor: String): Boolean = Scala2PickleAnnotations.contains(descriptor) @@ -142,9 +145,9 @@ object ScalaLibraryPlugin extends AutoPlugin { val writer = new ClassWriter(0) // Remove Scala 2 pickles and Scala signatures val visitor = new ClassVisitor(Opcodes.ASM9, writer) { - override def visitAttribute(attr: Attribute): Unit = attr.`type` match { - case "ScalaSig" | "ScalaInlineInfo" => () - case _ => super.visitAttribute(attr) + override def visitAttribute(attr: Attribute): Unit = { + val shouldRemove = Scala2PickleAttributes.contains(attr.`type`) + if (!shouldRemove) super.visitAttribute(attr) } override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = @@ -204,20 +207,27 @@ object ScalaLibraryPlugin extends AutoPlugin { output } - /** Check if class file bytecode contains Scala 2 pickle annotations */ + /** Check if class file bytecode contains Scala 2 pickle annotations or attributes */ private def hasScala2Pickles(bytes: Array[Byte]): Boolean = { - var found = false + var hasPickleAnnotation = false + var hasScalaSigAttr = false + var hasScalaInlineInfoAttr = false val visitor = new ClassVisitor(Opcodes.ASM9) { override def visitAnnotation(desc: String, visible: Boolean): AnnotationVisitor = { - if (isScala2PickleAnnotation(desc)) found = true + if (isScala2PickleAnnotation(desc)) hasPickleAnnotation = true null } + override def visitAttribute(attr: Attribute): Unit = + if (Scala2PickleAttributes.contains(attr.`type`)) attr.`type` match { + case "ScalaSig" => hasScalaSigAttr = true + case "ScalaInlineInfo" => hasScalaInlineInfoAttr = true + } } new ClassReader(bytes).accept( visitor, ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES ) - found + hasPickleAnnotation || hasScalaSigAttr || hasScalaInlineInfoAttr } def validateNoScala2Pickles(jar: File): Unit = { @@ -388,7 +398,6 @@ object ScalaLibraryPlugin extends AutoPlugin { } } - /** Custom ASM Attribute for TASTY that can be written to class files */ private class TastyAttribute(val uuid: Array[Byte]) extends Attribute("TASTY") { override def write(classWriter: ClassWriter, code: Array[Byte], codeLength: Int, maxStack: Int, maxLocals: Int): ByteVector = { From 596a5fe41bda8c40f6de15a6ec0fe48176fac600 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 30 Dec 2025 16:12:28 +0100 Subject: [PATCH 08/10] Remove `TastyAttributeReader` replace with inline reader using prototypes --- project/ScalaLibraryPlugin.scala | 34 +++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index 7262ed6ea85f..9027090a8aa9 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -324,17 +324,21 @@ object ScalaLibraryPlugin extends AutoPlugin { /** Extract TASTY UUID from class file bytecode, if present */ private def extractTastyUUIDFromClass(bytes: Array[Byte]): Option[Array[Byte]] = { - val tastyAttr = new TastyAttributeReader() var result: Option[Array[Byte]] = None - val visitor = new ClassVisitor(Opcodes.ASM9) { - override def visitAttribute(attr: Attribute): Unit = { - attr match { - case t: TastyAttributeReader => result = t.uuid - case _ => () + val tastyPrototype = new Attribute("TASTY") { + override def read(cr: ClassReader, off: Int, len: Int, buf: Array[Char], codeOff: Int, labels: Array[Label]): Attribute = { + if (len == 16) { + val uuid = Array.tabulate[Byte](16)(i => cr.readByte(off + i).toByte) + result = Some(uuid) } + this } } - new ClassReader(bytes).accept(visitor, Array[Attribute](tastyAttr), 0) + new ClassReader(bytes).accept( + new ClassVisitor(Opcodes.ASM9) {}, + Array(tastyPrototype), + 0 + ) result } @@ -406,20 +410,4 @@ object ScalaLibraryPlugin extends AutoPlugin { bv } } - /** Custom ASM Attribute for reading TASTY attributes from class files */ - private class TastyAttributeReader extends Attribute("TASTY") { - var uuid: Option[Array[Byte]] = None - - override def read(classReader: ClassReader, offset: Int, length: Int, charBuffer: Array[Char], codeOffset: Int, labels: Array[Label]): Attribute = { - val attr = new TastyAttributeReader() - if (length == 16) { - val bytes = new Array[Byte](16) - for (i <- 0 until 16) { - bytes(i) = classReader.readByte(offset + i).toByte - } - attr.uuid = Some(bytes) - } - attr - } - } } From 9178964ec02ff07e7d7314fdff6b14884037f7f7 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 30 Dec 2025 18:07:48 +0100 Subject: [PATCH 09/10] Add assertion to ensure that no new TASTy attributes are generated or removed that would be missing in unpatched sources compiled using Scala 3 --- project/ScalaLibraryPlugin.scala | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index 9027090a8aa9..df6dd25be458 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -171,6 +171,10 @@ object ScalaLibraryPlugin extends AutoPlugin { * - Only the class whose name matches the .tasty file name gets the attribute * - Java source files don't produce .tasty files, so they are skipped * + * Additionally validates that if the original class file (before patching) had a TASTY + * attribute, the patched version will also have one. This prevents accidentally losing + * TASTY attributes during the patching process. + * * @param input the input file (.class or .sjsir) * @param output the output file location * @param classDirectory the class directory to look for .tasty files @@ -182,6 +186,11 @@ object ScalaLibraryPlugin extends AutoPlugin { return output } + // Extract the original TASTY UUID if the class file exists and has one + val originalTastyUUID: Option[Array[Byte]] = + if (output.exists()) extractTastyUUIDFromClass(IO.readBytes(output)) + else None + val relativePath = output.relativeTo(classDirectory) .getOrElse(sys.error(s"Patched file is not relative to class directory: $output")) .getPath @@ -203,6 +212,18 @@ object ScalaLibraryPlugin extends AutoPlugin { if (isPrimaryClass) Some(extractTastyUUID(IO.readBytes(tastyFile))) else None } + + // Validation to ensure that no new TASTY attributes are added or removed when compared with unpatched sources + (tastyUUID, originalTastyUUID) match { + case (None, None) => () // no TASTY attribute, no problem + case (Some(newUUID), Some(originalUUID)) => + assert(java.util.Arrays.equals(originalUUID, newUUID), + s"TASTY UUID mismatch for $relativePath: original=${originalUUID.map(b => f"$b%02x").mkString}, new=${newUUID.map(b => f"$b%02x").mkString}." + ) + case (Some(_), None) => sys.error(s"TASTY attribute defined, but not present in unpatched source $relativePath") + case (None, Some(_)) => sys.error(s"TASTY attribute missing, but present in unpatched $relativePath") + } + IO.write(output, patchClassFile(classfileBytes, tastyUUID)) output } From 47e2dd31113790a96a6d84a70c5951de1a0c3db7 Mon Sep 17 00:00:00 2001 From: Wojciech Mazur Date: Tue, 6 Jan 2026 23:30:00 +0100 Subject: [PATCH 10/10] Validate existance of 'Scala' attribute in all .scala outputputs. Patch specialized classes to add missing attribute (Scala 2 compiler bug?) --- project/ScalaLibraryPlugin.scala | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/project/ScalaLibraryPlugin.scala b/project/ScalaLibraryPlugin.scala index df6dd25be458..497d82a1c876 100644 --- a/project/ScalaLibraryPlugin.scala +++ b/project/ScalaLibraryPlugin.scala @@ -45,6 +45,7 @@ object ScalaLibraryPlugin extends AutoPlugin { .map{ jar => validateNoScala2Pickles(jar) validateTastyAttributes(jar) + validateScalaAttributes(jar) jar } .value, @@ -136,6 +137,7 @@ object ScalaLibraryPlugin extends AutoPlugin { } /** Remove Scala 2 Pickles from class file and optionally add TASTY attribute. + * Also ensures the Scala attribute is present for all Scala-compiled classes. * * @param bytes the class file bytecode * @param tastyUUID optional 16-byte UUID from the corresponding .tasty file (only for primary class) @@ -159,6 +161,11 @@ object ScalaLibraryPlugin extends AutoPlugin { tastyUUID .map(new TastyAttribute(_)) .foreach(writer.visitAttribute) + // Add Scala attribute if not present and this is a Scala-compiled class + def isJavaSourced = extractSourceFile(bytes).exists(_.endsWith(".java")) + if (!hasScalaAttribute(bytes) && !isJavaSourced) { + writer.visitAttribute(new ScalaAttribute) + } writer.toByteArray } @@ -251,6 +258,46 @@ object ScalaLibraryPlugin extends AutoPlugin { hasPickleAnnotation || hasScalaSigAttr || hasScalaInlineInfoAttr } + /** Check if class file bytecode contains a Scala attribute */ + private def hasScalaAttribute(bytes: Array[Byte]): Boolean = { + var hasScala = false + val visitor = new ClassVisitor(Opcodes.ASM9) { + override def visitAttribute(attr: Attribute): Unit = { + if (attr.`type` == "Scala") hasScala = true + } + } + new ClassReader(bytes).accept( + visitor, + ClassReader.SKIP_CODE | ClassReader.SKIP_DEBUG | ClassReader.SKIP_FRAMES + ) + hasScala + } + + /** Validate that all files produced by Scala compiler have a "Scala" attribute. + * Java-sourced files are excluded from this check since they don't have Scala attributes. + */ + def validateScalaAttributes(jar: File): Unit = { + val classFilesWithoutScala = Using.jarFile(verify = true)(jar) { jarFile => + jarFile + .entries().asScala + .filter(_.getName.endsWith(".class")) + .flatMap { entry => + Using.bufferedInputStream(jarFile.getInputStream(entry)) { inputStream => + val bytes = inputStream.readAllBytes() + // Skip Java-sourced files - they won't have Scala attributes + val isJavaSourced = extractSourceFile(bytes).exists(_.endsWith(".java")) + if (!isJavaSourced && !hasScalaAttribute(bytes)) Some(entry.getName) + else None + } + } + .toList + } + assert( + classFilesWithoutScala.isEmpty, + s"JAR ${jar.getName} contains ${classFilesWithoutScala.size} class files without 'Scala' attribute: ${classFilesWithoutScala.mkString("\n - ", "\n - ", "")}" + ) + } + def validateNoScala2Pickles(jar: File): Unit = { val classFilesWithPickles = Using.jarFile(verify = true)(jar){ jarFile => jarFile @@ -431,4 +478,12 @@ object ScalaLibraryPlugin extends AutoPlugin { bv } } + + /** Custom ASM Attribute for Scala attribute marker (empty attribute) */ + private class ScalaAttribute extends Attribute("Scala") { + override def write(classWriter: ClassWriter, code: Array[Byte], codeLength: Int, maxStack: Int, maxLocals: Int): ByteVector = { + // Scala attribute is empty (length = 0x0) + new ByteVector(0) + } + } }