Merge pull request scala#4176 from mpociecha/flat-classpath2

The alternative, flat representation of classpath elements

Author: gkossakowski

bincompat-backward.whitelist.conf

@@ -203,6 +203,11 @@ filter {
     {
         matchName="scala.reflect.runtime.ThreadLocalStorage#MyThreadLocalStorage.values"
         problemName=MissingMethodProblem
+    },
+    // the below method was the unused private (sic!) method but the compatibility checker was complaining about it
+    {
+        matchName="scala.reflect.io.ZipArchive.scala$reflect$io$ZipArchive$$walkIterator"
+        problemName=MissingMethodProblem
     }
   ]
 }

bincompat-forward.whitelist.conf

@@ -292,6 +292,28 @@ filter {
     {
         matchName="scala.collection.immutable.Stream.scala$collection$immutable$Stream$$loop$2"
         problemName=MissingMethodProblem
+    },
+    // changes needed by ZipArchiveFileLookup (the flat classpath representation)
+    {
+        matchName="scala.reflect.io.FileZipArchive.allDirs"
+        problemName=MissingMethodProblem
+    },
+    {
+        matchName="scala.reflect.io.FileZipArchive.root"
+        problemName=MissingMethodProblem
+    },
+    // introduced the harmless method (instead of the repeated code in several places)
+    {
+        matchName="scala.reflect.runtime.Settings#MultiStringSetting.valueSetByUser"
+        problemName=MissingMethodProblem
+    },
+    {
+        matchName="scala.reflect.runtime.Settings#BooleanSetting.valueSetByUser"
+        problemName=MissingMethodProblem
+    },
+    {
+        matchName="scala.reflect.runtime.Settings#IntSetting.valueSetByUser"
+        problemName=MissingMethodProblem
     }
   ]
 }

src/compiler/scala/reflect/macros/contexts/Infrastructure.scala

@@ -12,5 +12,5 @@ trait Infrastructure {
 
   def compilerSettings: List[String] = universe.settings.recreateArgs
 
-  def classPath: List[java.net.URL] = global.classPath.asURLs
+  def classPath: List[java.net.URL] = global.classPath.asURLs.toList
 }

src/compiler/scala/tools/nsc/ClassPathMemoryConsumptionTester.scala

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2014 Contributor. All rights reserved.
+ */
+package scala.tools.nsc
+
+import scala.io.StdIn.readLine
+
+/**
+ * Simple application to check out amount of memory used by chosen classpath representation.
+ * It allows us to create many scalac-like calls based on specified parameters, where each main retains Global.
+ * And we need additional tool (e.g. profiler) to measure memory consumption itself.
+ */
+object ClassPathMemoryConsumptionTester {
+
+  private class TestSettings extends Settings {
+    val requiredInstances = IntSetting("-requiredInstances",
+      "Determine how many times classpath should be loaded", 10, Some((1, 10000)), (_: String) => None)
+  }
+
+  private class MainRetainsGlobal extends scala.tools.nsc.MainClass {
+    var retainedGlobal: Global = _
+    override def doCompile(compiler: Global) {
+      retainedGlobal = compiler
+      super.doCompile(compiler)
+    }
+  }
+
+  def main(args: Array[String]): Unit = {
+    if (args contains "-help") usage()
+    else doTest(args)
+  }
+
+  private def doTest(args: Array[String]) = {
+    val settings = loadSettings(args.toList)
+
+    val mains = (1 to settings.requiredInstances.value) map (_ => new MainRetainsGlobal)
+
+    // we need original settings without additional params to be able to use them later
+    val baseArgs = argsWithoutRequiredInstances(args)
+
+    println(s"Loading classpath ${settings.requiredInstances.value} times")
+    val startTime = System.currentTimeMillis()
+
+    mains map (_.process(baseArgs))
+
+    val elapsed = System.currentTimeMillis() - startTime
+    println(s"Operation finished - elapsed $elapsed ms")
+    println("Memory consumption can be now measured")
+
+    var textFromStdIn = ""
+    while (textFromStdIn.toLowerCase != "exit")
+      textFromStdIn = readLine("Type 'exit' to close application: ")
+  }
+
+  /**
+   * Prints usage information
+   */
+  private def usage(): Unit =
+    println( """Use classpath and sourcepath options like in the case of e.g. 'scala' command.
+               | There's also one additional option:
+               | -requiredInstances <int value> Determine how many times classpath should be loaded
+             """.stripMargin.trim)
+
+  private def loadSettings(args: List[String]) = {
+    val settings = new TestSettings()
+    settings.processArguments(args, processAll = true)
+    if (settings.classpath.isDefault)
+      settings.classpath.value = sys.props("java.class.path")
+    settings
+  }
+
+  private def argsWithoutRequiredInstances(args: Array[String]) = {
+    val instancesIndex = args.indexOf("-requiredInstances")
+    if (instancesIndex == -1) args
+    else args.dropRight(args.length - instancesIndex) ++ args.drop(instancesIndex + 2)
+  }
+}

src/compiler/scala/tools/nsc/GenericRunnerSettings.scala

@@ -5,10 +5,11 @@
 
 package scala.tools.nsc
 
-import scala.tools.util.PathResolver
+import java.net.URL
+import scala.tools.util.PathResolverFactory
 
 class GenericRunnerSettings(error: String => Unit) extends Settings(error) {
-  def classpathURLs = new PathResolver(this).asURLs
+  def classpathURLs: Seq[URL] = PathResolverFactory.create(this).resultAsURLs
 
   val howtorun =
     ChoiceSetting(

src/compiler/scala/tools/nsc/Global.scala

@@ -14,11 +14,10 @@ import scala.compat.Platform.currentTime
 import scala.collection.{ mutable, immutable }
 import io.{ SourceReader, AbstractFile, Path }
 import reporters.{ Reporter, ConsoleReporter }
-import util.{ ClassPath, MergedClassPath, StatisticsInfo, returning, stackTraceString }
+import util.{ ClassFileLookup, ClassPath, MergedClassPath, StatisticsInfo, returning }
 import scala.reflect.ClassTag
-import scala.reflect.internal.util.{ OffsetPosition, SourceFile, NoSourceFile, BatchSourceFile, ScriptSourceFile }
-import scala.reflect.internal.pickling.{ PickleBuffer, PickleFormat }
-import scala.reflect.io.VirtualFile
+import scala.reflect.internal.util.{ SourceFile, NoSourceFile, BatchSourceFile, ScriptSourceFile }
+import scala.reflect.internal.pickling.PickleBuffer
 import symtab.{ Flags, SymbolTable, SymbolTrackers }
 import symtab.classfile.Pickler
 import plugins.Plugins
@@ -35,6 +34,8 @@ import backend.opt.{ Inliners, InlineExceptionHandlers, ConstantOptimization, Cl
 import backend.icode.analysis._
 import scala.language.postfixOps
 import scala.tools.nsc.ast.{TreeGen => AstTreeGen}
+import scala.tools.nsc.classpath.FlatClassPath
+import scala.tools.nsc.settings.ClassPathRepresentationType
 
 class Global(var currentSettings: Settings, var reporter: Reporter)
     extends SymbolTable
@@ -58,7 +59,12 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
 
   class GlobalMirror extends Roots(NoSymbol) {
     val universe: self.type = self
-    def rootLoader: LazyType = new loaders.PackageLoader(classPath)
+    def rootLoader: LazyType = {
+      settings.YclasspathImpl.value match {
+        case ClassPathRepresentationType.Flat => new loaders.PackageLoaderUsingFlatClassPath(FlatClassPath.RootPackage, flatClassPath)
+        case ClassPathRepresentationType.Recursive => new loaders.PackageLoader(recursiveClassPath)
+      }
+    }
     override def toString = "compiler mirror"
   }
   implicit val MirrorTag: ClassTag[Mirror] = ClassTag[Mirror](classOf[GlobalMirror])
@@ -104,7 +110,14 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
   type PlatformClassPath = ClassPath[AbstractFile]
   type OptClassPath = Option[PlatformClassPath]
 
-  def classPath: PlatformClassPath = platform.classPath
+  def classPath: ClassFileLookup[AbstractFile] = settings.YclasspathImpl.value match {
+    case ClassPathRepresentationType.Flat => flatClassPath
+    case ClassPathRepresentationType.Recursive => recursiveClassPath
+  }
+
+  private def recursiveClassPath: ClassPath[AbstractFile] = platform.classPath
+
+  private def flatClassPath: FlatClassPath = platform.flatClassPath
 
   // sub-components --------------------------------------------------
 
@@ -319,7 +332,7 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
           None
       }
 
-    val charset = ( if (settings.encoding.isSetByUser) Some(settings.encoding.value) else None ) flatMap loadCharset getOrElse {
+    val charset = settings.encoding.valueSetByUser flatMap loadCharset getOrElse {
       settings.encoding.value = defaultEncoding // A mandatory charset
       Charset.forName(defaultEncoding)
     }
@@ -334,16 +347,16 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
       }
     }
 
-    ( if (settings.sourceReader.isSetByUser) Some(settings.sourceReader.value) else None ) flatMap loadReader getOrElse {
+    settings.sourceReader.valueSetByUser flatMap loadReader getOrElse {
       new SourceReader(charset.newDecoder(), reporter)
     }
   }
 
-  if (settings.verbose || settings.Ylogcp) {
+  if (settings.verbose || settings.Ylogcp)
     reporter.echo(
-      s"[search path for source files: ${classPath.sourcepaths.mkString(",")}]\n"+
-      s"[search path for class files: ${classPath.asClasspathString}")
-  }
+      s"[search path for source files: ${classPath.asSourcePathString}]\n" +
+      s"[search path for class files: ${classPath.asClassPathString}]"
+    )
 
   // The current division between scala.reflect.* and scala.tools.nsc.* is pretty
   // clunky.  It is often difficult to have a setting influence something without having
@@ -846,6 +859,9 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
 
   /** Extend classpath of `platform` and rescan updated packages. */
   def extendCompilerClassPath(urls: URL*): Unit = {
+    if (settings.YclasspathImpl.value == ClassPathRepresentationType.Flat)
+      throw new UnsupportedOperationException("Flat classpath doesn't support extending the compiler classpath")
+
     val newClassPath = platform.classPath.mergeUrlsIntoClassPath(urls: _*)
     platform.currentClassPath = Some(newClassPath)
     // Reload all specified jars into this compiler instance
@@ -881,8 +897,11 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
    *                entries on the classpath.
    */
   def invalidateClassPathEntries(paths: String*): Unit = {
+    if (settings.YclasspathImpl.value == ClassPathRepresentationType.Flat)
+      throw new UnsupportedOperationException("Flat classpath doesn't support the classpath invalidation")
+
     implicit object ClassPathOrdering extends Ordering[PlatformClassPath] {
-      def compare(a:PlatformClassPath, b:PlatformClassPath) = a.asClasspathString compare b.asClasspathString
+      def compare(a:PlatformClassPath, b:PlatformClassPath) = a.asClassPathString compare b.asClassPathString
     }
     val invalidated, failed = new mutable.ListBuffer[ClassSymbol]
     classPath match {
@@ -910,10 +929,10 @@ class Global(var currentSettings: Settings, var reporter: Reporter)
           informProgress(s"classpath updated on entries [${subst.keys mkString ","}]")
           def mkClassPath(elems: Iterable[PlatformClassPath]): PlatformClassPath =
             if (elems.size == 1) elems.head
-            else new MergedClassPath(elems, classPath.context)
+            else new MergedClassPath(elems, recursiveClassPath.context)
           val oldEntries = mkClassPath(subst.keys)
           val newEntries = mkClassPath(subst.values)
-          mergeNewEntries(newEntries, RootClass, Some(classPath), Some(oldEntries), invalidated, failed)
+          mergeNewEntries(newEntries, RootClass, Some(recursiveClassPath), Some(oldEntries), invalidated, failed)
         }
     }
     def show(msg: String, syms: scala.collection.Traversable[Symbol]) =

src/compiler/scala/tools/nsc/ObjectRunner.scala

@@ -18,14 +18,14 @@ trait CommonRunner {
    *  @throws NoSuchMethodException
    *  @throws InvocationTargetException
    */
-  def run(urls: List[URL], objectName: String, arguments: Seq[String]) {
+  def run(urls: Seq[URL], objectName: String, arguments: Seq[String]) {
     (ScalaClassLoader fromURLs urls).run(objectName, arguments)
   }
 
   /** Catches exceptions enumerated by run (in the case of InvocationTargetException,
    *  unwrapping it) and returns it any thrown in Left(x).
    */
-  def runAndCatch(urls: List[URL], objectName: String, arguments: Seq[String]): Either[Throwable, Boolean] = {
+  def runAndCatch(urls: Seq[URL], objectName: String, arguments: Seq[String]): Either[Throwable, Boolean] = {
     try   { run(urls, objectName, arguments) ; Right(true) }
     catch { case e: Throwable => Left(unwrap(e)) }
   }

src/compiler/scala/tools/nsc/PhaseAssembly.scala

@@ -199,7 +199,7 @@ trait PhaseAssembly {
     // Add all phases in the set to the graph
     val graph = phasesSetToDepGraph(phasesSet)
 
-    val dot = if (settings.genPhaseGraph.isSetByUser) Some(settings.genPhaseGraph.value) else None
+    val dot = settings.genPhaseGraph.valueSetByUser
 
     // Output the phase dependency graph at this stage
     def dump(stage: Int) = dot foreach (n => graphToDotFile(graph, s"$n-$stage.dot"))

src/compiler/scala/tools/nsc/ScriptRunner.scala

@@ -8,7 +8,10 @@ package tools.nsc
 
 import io.{ AbstractFile, Directory, File, Path }
 import java.io.IOException
+import scala.tools.nsc.classpath.DirectoryFlatClassPath
 import scala.tools.nsc.reporters.{Reporter,ConsoleReporter}
+import scala.tools.nsc.settings.ClassPathRepresentationType
+import scala.tools.nsc.util.ClassPath.DefaultJavaContext
 import util.Exceptional.unwrap
 
 /** An object that runs Scala code in script files.
@@ -112,8 +115,10 @@ class ScriptRunner extends HasCompileSocket {
     }
 
     def hasClassToRun(d: Directory): Boolean = {
-      import util.ClassPath.{ DefaultJavaContext => ctx }
-      val cp = ctx.newClassPath(AbstractFile.getDirectory(d))
+      val cp = settings.YclasspathImpl.value match {
+        case ClassPathRepresentationType.Recursive => DefaultJavaContext.newClassPath(AbstractFile.getDirectory(d))
+        case ClassPathRepresentationType.Flat => DirectoryFlatClassPath(d.jfile)
+      }
       cp.findClass(mainClass).isDefined
     }
 

src/compiler/scala/tools/nsc/backend/JavaPlatform.scala

@@ -7,7 +7,10 @@ package scala.tools.nsc
 package backend
 
 import io.AbstractFile
-import util.{ClassPath,MergedClassPath,DeltaClassPath}
+import scala.tools.nsc.classpath.FlatClassPath
+import scala.tools.nsc.settings.ClassPathRepresentationType
+import scala.tools.nsc.util.{ ClassPath, DeltaClassPath, MergedClassPath }
+import scala.tools.util.FlatClassPathResolver
 import scala.tools.util.PathResolver
 
 trait JavaPlatform extends Platform {
@@ -19,10 +22,20 @@ trait JavaPlatform extends Platform {
   private[nsc] var currentClassPath: Option[MergedClassPath[AbstractFile]] = None
 
   def classPath: ClassPath[AbstractFile] = {
+    assert(settings.YclasspathImpl.value == ClassPathRepresentationType.Recursive,
+      "To use recursive classpath representation you must enable it with -YclasspathImpl:recursive compiler option.")
+
     if (currentClassPath.isEmpty) currentClassPath = Some(new PathResolver(settings).result)
     currentClassPath.get
   }
 
+  private[nsc] lazy val flatClassPath: FlatClassPath = {
+    assert(settings.YclasspathImpl.value == ClassPathRepresentationType.Flat,
+      "To use flat classpath representation you must enable it with -YclasspathImpl:flat compiler option.")
+
+    new FlatClassPathResolver(settings).result
+  }
+
   /** Update classpath with a substituted subentry */
   def updateClassPath(subst: Map[ClassPath[AbstractFile], ClassPath[AbstractFile]]) =
     currentClassPath = Some(new DeltaClassPath(currentClassPath.get, subst))