Merge pull request scala#4463 from retronym/topic/indylambda-emit-indy

Use LambdaMetafactory where possible for lambda creation.

Author: lrytz

build.xml

@@ -266,6 +266,11 @@ TODO:
     -->
     <if><not><isset property="maven-deps-done"></isset></not><then>
       <mkdir dir="${user.home}/.m2/repository"/>
+
+      <artifact:remoteRepository id="sonatype-release" url="https://oss.sonatype.org/content/repositories/releases"/>
+      <artifact:remoteRepository id="sonatype-snapshots" url="https://oss.sonatype.org/content/repositories/snapshots"/>
+      <artifact:remoteRepository id="extra-repo" url="${extra.repo.url}"/>
+
       <!-- This task has an issue where if the user directory does not exist, so we create it above. UGH. -->
       <artifact:dependencies pathId="extra.tasks.classpath" filesetId="extra.tasks.fileset">
         <dependency groupId="biz.aQute" artifactId="bnd" version="1.50.0"/>
@@ -307,6 +312,36 @@ TODO:
       <artifact:remoteRepository id="sonatype-release" url="https://oss.sonatype.org/content/repositories/releases"/>
       <artifact:remoteRepository id="extra-repo" url="${extra.repo.url}"/>
 
+      <!-- scala-java8-compat, used by the experimental -target jvm-1.8 support. -->
+      <if><isset property="scala-java8-compat.package"/><then>
+        <property name="scala-java8-compat.version" value="0.2.0"/>
+        <property name="scala-java8-compat.binary.version" value="2.11"/>
+        <artifact:dependencies pathId="scala-java8-compat.classpath" filesetId="scala-java8-compat.fileset">
+          <dependency groupId="org.scala-lang.modules" artifactId="scala-java8-compat_${scala-java8-compat.binary.version}" version="${scala-java8-compat.version}">
+            <exclusion groupId="org.scala-lang" artifactId="scala-library"/>
+          </dependency>
+        </artifact:dependencies>
+        <property name="scala-java8-compat-classes" value="${build-quick.dir}/scala-java8-compat"/>
+        <delete dir="${scala-java8-compat-classes}"/>
+        <unzip dest="${scala-java8-compat-classes}">
+          <fileset refid="scala-java8-compat.fileset"/>
+          <patternset>
+            <include name="**/*.class"/>
+          </patternset>
+        </unzip>
+        <path id="scala-java8-compat.libs">
+          <pathelement location="${scala-java8-compat-classes}"/>
+        </path>
+        <fileset id="scala-java8-compat.fileset" dir="${scala-java8-compat-classes}">
+          <include name="**/*"/>
+        </fileset>
+      </then>
+        <else>
+          <path id="scala-java8-compat.libs"/>
+          <fileset id="scala-java8-compat.fileset" dir="." excludes="**"/>
+        </else>
+      </if>
+
       <!-- prepare, for each of the names below, the property "@{name}.cross", set to the
            necessary cross suffix (usually something like "_2.11.0-M6". -->
       <prepareCross name="scala-xml" />
@@ -718,6 +753,7 @@ TODO:
       <pathelement location="${build-locker.dir}/classes/library"/>
       <path refid="forkjoin.classpath"/>
       <path refid="aux.libs"/>
+      <path refid="scala-java8-compat.libs"/>
     </path>
 
     <path id="locker.reflect.build.path">
@@ -739,6 +775,7 @@ TODO:
       <pathelement location="${build-quick.dir}/classes/library"/>
       <path refid="forkjoin.classpath"/>
       <path refid="aux.libs"/>
+      <path refid="scala-java8-compat.libs"/>
     </path>
 
     <path id="quick.actors.build.path">
@@ -827,6 +864,7 @@ TODO:
     <path id="pack.library.files">
       <fileset dir="${build-quick.dir}/classes/library"/>
       <fileset dir="${forkjoin-classes}"/>
+      <fileset refid="scala-java8-compat.fileset"/>
     </path>
 
     <path id="pack.actors.files">

src/compiler/scala/tools/nsc/backend/jvm/BCodeBodyBuilder.scala

@@ -10,6 +10,7 @@ package backend
 package jvm
 
 import scala.annotation.switch
+import scala.reflect.internal.Flags
 
 import scala.tools.asm
 import GenBCode._
@@ -632,6 +633,10 @@ abstract class BCodeBodyBuilder extends BCodeSkelBuilder {
             case _ =>
               abort(s"Cannot instantiate $tpt of kind: $generatedType")
           }
+        case Apply(_, args) if app.hasAttachment[delambdafy.LambdaMetaFactoryCapable] =>
+          val attachment = app.attachments.get[delambdafy.LambdaMetaFactoryCapable].get
+          genLoadArguments(args, paramTKs(app))
+          genInvokeDynamicLambda(attachment.target, attachment.arity, attachment.functionalInterface)
 
         case Apply(fun @ _, List(expr)) if currentRun.runDefinitions.isBox(fun.symbol) =>
           val nativeKind = tpeTK(expr)
@@ -1280,6 +1285,40 @@ abstract class BCodeBodyBuilder extends BCodeSkelBuilder {
     def genSynchronized(tree: Apply, expectedType: BType): BType
     def genLoadTry(tree: Try): BType
 
+    def genInvokeDynamicLambda(lambdaTarget: Symbol, arity: Int, functionalInterface: Symbol) {
+      val isStaticMethod = lambdaTarget.hasFlag(Flags.STATIC)
+
+      val targetHandle =
+        new asm.Handle(if (lambdaTarget.hasFlag(Flags.STATIC)) asm.Opcodes.H_INVOKESTATIC else asm.Opcodes.H_INVOKEVIRTUAL,
+          classBTypeFromSymbol(lambdaTarget.owner).internalName,
+          lambdaTarget.name.toString,
+          asmMethodType(lambdaTarget).descriptor)
+      val receiver = if (isStaticMethod) None else Some(lambdaTarget.owner)
+      val (capturedParams, lambdaParams) = lambdaTarget.paramss.head.splitAt(lambdaTarget.paramss.head.length - arity)
+      // Requires https://github.com/scala/scala-java8-compat on the runtime classpath
+      val returnUnit = lambdaTarget.info.resultType.typeSymbol == UnitClass
+      val functionalInterfaceDesc: String = classBTypeFromSymbol(functionalInterface).descriptor
+      val desc = (receiver.toList ::: capturedParams).map(sym => toTypeKind(sym.info)).mkString(("("), "", ")") + functionalInterfaceDesc
+
+      // TODO specialization
+      val constrainedType = new MethodBType(lambdaParams.map(p => toTypeKind(p.tpe)), toTypeKind(lambdaTarget.tpe.resultType)).toASMType
+      val abstractMethod = functionalInterface.info.decls.find(_.isDeferred).getOrElse(functionalInterface.info.member(nme.apply))
+      val methodName = abstractMethod.name.toString
+      val applyN = {
+        val mt = asmMethodType(abstractMethod)
+        mt.toASMType
+      }
+
+      bc.jmethod.visitInvokeDynamicInsn(methodName, desc, lambdaMetaFactoryBootstrapHandle,
+          // boostrap args
+        applyN, targetHandle, constrainedType
+      )
+    }
   }
 
+  val lambdaMetaFactoryBootstrapHandle =
+    new asm.Handle(asm.Opcodes.H_INVOKESTATIC,
+      "java/lang/invoke/LambdaMetafactory", "metafactory",
+      "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;Ljava/lang/invoke/MethodType;)Ljava/lang/invoke/CallSite;")
+
 }

src/compiler/scala/tools/nsc/backend/jvm/BCodeIdiomatic.scala

@@ -412,6 +412,9 @@ abstract class BCodeIdiomatic extends SubComponent {
       jmethod.instructions.add(node)
       if (settings.YoptInlinerEnabled) callsitePositions(node) = pos
     }
+    final def invokedynamic(owner: String, name: String, desc: String) {
+      jmethod.visitMethodInsn(Opcodes.INVOKEDYNAMIC, owner, name, desc)
+    }
 
     // can-multi-thread
     final def goTo(label: asm.Label) { jmethod.visitJumpInsn(Opcodes.GOTO, label) }

src/compiler/scala/tools/nsc/transform/Delambdafy.scala

@@ -9,22 +9,29 @@ import scala.reflect.internal.Symbols
 import scala.collection.mutable.LinkedHashMap
 
 /**
- * This transformer is responsible for turning lambdas into anonymous classes.
+ * This transformer is responsible for preparing lambdas for runtime, by either translating to anonymous classes
+ * or to a tree that will be convereted to invokedynamic by the JVM 1.8+ backend.
+ *
  * The main assumption it makes is that a lambda {args => body} has been turned into
  * {args => liftedBody()} where lifted body is a top level method that implements the body of the lambda.
  * Currently Uncurry is responsible for that transformation.
  *
- * From a lambda, Delambdafy will create
+ * From a lambda, Delambdafy will create:
+ *
+ * Under -target:jvm-1.7 and below:
+ *
  * 1) a new top level class that
       a) has fields and a constructor taking the captured environment (including possibly the "this"
  *       reference)
  *    b) an apply method that calls the target method
  *    c) if needed a bridge method for the apply method
  * 2) an instantiation of the newly created class which replaces the lambda
  *
- *  TODO the main work left to be done is to plug into specialization. Primarily that means choosing a
- * specialized FunctionN trait instead of the generic FunctionN trait as a parent and creating the
- * appropriately named applysp method
+ * Under -target:jvm-1.8 with GenBCode:
+ *
+ * 1) An application of the captured arguments to a fictional symbol representing the lambda factory.
+ *    This will be translated by the backed into an invokedynamic using a bootstrap method in JDK8's `LambdaMetaFactory`.
+ *    The captured arguments include `this` if `liftedBody` is unable to be made STATIC.
  */
 abstract class Delambdafy extends Transform with TypingTransformers with ast.TreeDSL with TypeAdaptingTransformer {
   import global._
@@ -79,6 +86,7 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
     sealed abstract class TransformedFunction
     // A class definition for the lambda, an expression instantiating the lambda class
     case class DelambdafyAnonClass(lambdaClassDef: ClassDef, newExpr: Tree) extends TransformedFunction
+    case class InvokeDynamicLambda(tree: Apply) extends TransformedFunction
 
     // here's the main entry point of the transform
     override def transform(tree: Tree): Tree = tree match {
@@ -93,6 +101,9 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
             lambdaClassDefs(pkg) = lambdaClassDef :: lambdaClassDefs(pkg)
 
             super.transform(newExpr)
+          case InvokeDynamicLambda(apply) =>
+            // ... or an invokedynamic call
+            super.transform(apply)
         }
       case _ => super.transform(tree)
     }
@@ -124,6 +135,7 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
       if (!thisReferringMethods.contains(target))
         target setFlag STATIC
 
+      val isStatic = target.hasFlag(STATIC)
 
       /**
        * Creates the apply method for the anonymous subclass of FunctionN
@@ -199,7 +211,7 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
       val abstractFunctionErasedType = AbstractFunctionClass(formals.length).tpe
 
       // anonymous subclass of FunctionN with an apply method
-      def makeAnonymousClass = {
+      def makeAnonymousClass: ClassDef = {
         val parents = addSerializable(abstractFunctionErasedType)
         val funOwner = originalFunction.symbol.owner
 
@@ -232,7 +244,7 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
         // the Optional proxy that will hold a reference to the 'this'
         // object used by the lambda, if any. NoSymbol if there is no this proxy
         val thisProxy = {
-          if (target.hasFlag(STATIC))
+          if (isStatic)
             NoSymbol
           else {
             val sym = lambdaClass.newVariable(nme.FAKE_LOCAL_THIS, originalFunction.pos, SYNTHETIC)
@@ -271,22 +283,39 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
         val body = members ++ List(constr, applyMethodDef) ++ bridgeMethod
 
         // TODO if member fields are private this complains that they're not accessible
-        (localTyper.typedPos(decapturedFunction.pos)(ClassDef(lambdaClass, body)).asInstanceOf[ClassDef], thisProxy)
+        localTyper.typedPos(decapturedFunction.pos)(ClassDef(lambdaClass, body)).asInstanceOf[ClassDef]
       }
 
-      val (anonymousClassDef, thisProxy) = makeAnonymousClass
-
-      pkg.info.decls enter anonymousClassDef.symbol
-
-      val thisArg = optionSymbol(thisProxy) map (_ => gen.mkAttributedThis(oldClass) setPos originalFunction.pos)
-      val captureArgs = captures map (capture => Ident(capture) setPos originalFunction.pos)
-
-      val newStat =
-          Typed(New(anonymousClassDef.symbol, (thisArg.toList ++ captureArgs): _*), TypeTree(abstractFunctionErasedType))
-
-      val typedNewStat = localTyper.typedPos(originalFunction.pos)(newStat)
+      val allCaptureArgs: List[Tree] = {
+        val thisArg = if (isStatic) Nil else (gen.mkAttributedThis(oldClass) setPos originalFunction.pos) :: Nil
+        val captureArgs = captures.iterator.map(capture => gen.mkAttributedRef(capture) setPos originalFunction.pos).toList
+        thisArg ::: captureArgs
+      }
 
-      DelambdafyAnonClass(anonymousClassDef, typedNewStat)
+      val functionalInterface = java8CompatFunctionalInterface(target, originalFunction.tpe)
+      if (functionalInterface.exists) {
+        // Create a symbol representing a fictional lambda factory method that accepts the captured
+        // arguments and returns a Function.
+        val msym   = currentOwner.newMethod(nme.ANON_FUN_NAME, originalFunction.pos, ARTIFACT)
+        val argTypes: List[Type] = allCaptureArgs.map(_.tpe)
+        val params = msym.newSyntheticValueParams(argTypes)
+        msym.setInfo(MethodType(params, originalFunction.tpe))
+        val arity = originalFunction.vparams.length
+
+        // We then apply this symbol to the captures.
+        val apply = localTyper.typedPos(originalFunction.pos)(Apply(Ident(msym), allCaptureArgs)).asInstanceOf[Apply]
+
+        // The backend needs to know the target of the lambda and the functional interface in order
+        // to emit the invokedynamic instruction. We pass this information as tree attachment.
+        apply.updateAttachment(LambdaMetaFactoryCapable(target, arity, functionalInterface))
+        InvokeDynamicLambda(apply)
+      } else {
+        val anonymousClassDef = makeAnonymousClass
+        pkg.info.decls enter anonymousClassDef.symbol
+        val newStat = Typed(New(anonymousClassDef.symbol, allCaptureArgs: _*), TypeTree(abstractFunctionErasedType))
+        val typedNewStat = localTyper.typedPos(originalFunction.pos)(newStat)
+        DelambdafyAnonClass(anonymousClassDef, typedNewStat)
+      }
     }
 
     /**
@@ -436,4 +465,38 @@ abstract class Delambdafy extends Transform with TypingTransformers with ast.Tre
         super.traverse(tree)
     }
   }
+
+  final case class LambdaMetaFactoryCapable(target: Symbol, arity: Int, functionalInterface: Symbol)
+
+  // The functional interface that can be used to adapt the lambda target method `target` to the
+  // given function type. Returns `NoSymbol` if the compiler settings are unsuitable, or `LambdaMetaFactory`
+  // would be unable to generate the correct implementation (e.g. functions referring to derived value classes)
+  private def java8CompatFunctionalInterface(target: Symbol, functionType: Type): Symbol = {
+    val canUseLambdaMetafactory: Boolean = {
+      val hasValueClass = exitingErasure {
+        val methodType: Type = target.info
+        methodType.exists(_.isInstanceOf[ErasedValueType])
+      }
+      val isTarget18 = settings.target.value.contains("jvm-1.8")
+      settings.isBCodeActive && isTarget18 && !hasValueClass
+    }
+
+    def functionalInterface: Symbol = {
+      val sym = functionType.typeSymbol
+      val pack = currentRun.runDefinitions.Scala_Java8_CompatPackage
+      val name1 = specializeTypes.specializedFunctionName(sym, functionType.typeArgs)
+      val paramTps :+ restpe = functionType.typeArgs
+      val arity = paramTps.length
+      if (name1.toTypeName == sym.name) {
+        val returnUnit = restpe.typeSymbol == UnitClass
+        val functionInterfaceArray =
+          if (returnUnit) currentRun.runDefinitions.Scala_Java8_CompatPackage_JProcedure
+          else currentRun.runDefinitions.Scala_Java8_CompatPackage_JFunction
+        functionInterfaceArray.apply(arity)
+      } else {
+        pack.info.decl(name1.toTypeName.prepend("J"))
+      }
+    }
+    if (canUseLambdaMetafactory) functionalInterface else NoSymbol
+  }
 }

src/compiler/scala/tools/nsc/transform/SpecializeTypes.scala

@@ -303,6 +303,17 @@ abstract class SpecializeTypes extends InfoTransform with TypingTransformers {
     }
   }
 
+  def specializedFunctionName(sym: Symbol, args: List[Type]) = exitingSpecialize {
+    require(isFunctionSymbol(sym), sym)
+    val env: TypeEnv = TypeEnv.fromSpecialization(sym, args)
+    specializedClass.get((sym, env)) match {
+      case Some(x) =>
+        x.name
+      case None =>
+        sym.name
+    }
+  }
+
   /** Return the specialized name of 'sym' in the given environment. It
    *  guarantees the same result regardless of the map order by sorting
    *  type variables alphabetically.
@@ -315,10 +326,14 @@ abstract class SpecializeTypes extends InfoTransform with TypingTransformers {
       if (sym.isClass) env.keySet
       else specializedTypeVars(sym).intersect(env.keySet)
     )
+    specializedName(sym.name, tvars, env)
+  }
+
+  private def specializedName(name: Name, tvars: immutable.Set[Symbol], env: TypeEnv): TermName = {
     val (methparams, others) = tvars.toList sortBy ("" + _.name) partition (_.owner.isMethod)
     // debuglog("specName(" + sym + ") env: " + env + " tvars: " + tvars)
 
-    specializedName(sym.name, methparams map env, others map env)
+    specializedName(name, methparams map env, others map env)
   }
 
   /** Specialize name for the two list of types. The first one denotes

src/compiler/scala/tools/nsc/transform/UnCurry.scala

@@ -237,7 +237,7 @@ abstract class UnCurry extends InfoTransform
 
           def canUseDelamdafyMethod = (
                (inConstructorFlag == 0) // Avoiding synthesizing code prone to SI-6666, SI-8363 by using old-style lambda translation
-            && !isSpecialized           // DelambdafyTransformer currently only emits generic FunctionN-s, use the old style in the meantime
+            && (!isSpecialized || (settings.target.value == "jvm-1.8")) // DelambdafyTransformer currently only emits generic FunctionN-s, use the old style in the meantime
           )
           if (inlineFunctionExpansion || !canUseDelamdafyMethod) {
             val parents = addSerializable(abstractFunctionForFunctionType(fun.tpe))

src/reflect/scala/reflect/internal/Definitions.scala

@@ -1513,6 +1513,10 @@ trait Definitions extends api.StandardDefinitions {
 
       def isPolymorphicSignature(sym: Symbol) = PolySigMethods(sym)
       private lazy val PolySigMethods: Set[Symbol] = Set[Symbol](MethodHandle.info.decl(sn.Invoke), MethodHandle.info.decl(sn.InvokeExact)).filter(_.exists)
+
+      lazy val Scala_Java8_CompatPackage = rootMirror.getPackageIfDefined("scala.compat.java8")
+      lazy val Scala_Java8_CompatPackage_JFunction = (0 to MaxTupleArity).toArray map (i => getMemberIfDefined(Scala_Java8_CompatPackage.moduleClass, TypeName("JFunction" + i)))
+      lazy val Scala_Java8_CompatPackage_JProcedure = (0 to MaxTupleArity).toArray map (i => getMemberIfDefined(Scala_Java8_CompatPackage.moduleClass, TypeName("JProcedure" + i)))
     }
   }
 }

src/reflect/scala/reflect/internal/Symbols.scala

@@ -794,7 +794,7 @@ trait Symbols extends api.Symbols { self: SymbolTable =>
 
     final def isAnonymousFunction = isSynthetic && (name containsName tpnme.ANON_FUN_NAME)
     final def isDelambdafyFunction = isSynthetic && (name containsName tpnme.DELAMBDAFY_LAMBDA_CLASS_NAME)
-    final def isDelambdafyTarget  = isSynthetic && isMethod && (name containsName tpnme.ANON_FUN_NAME)
+    final def isDelambdafyTarget  = isArtifact && isMethod && (name containsName tpnme.ANON_FUN_NAME)
     final def isDefinedInPackage  = effectiveOwner.isPackageClass
     final def needsFlatClasses    = phase.flatClasses && rawowner != NoSymbol && !rawowner.isPackageClass
 

test/files/run/t2318.scala

@@ -11,6 +11,7 @@ object Test {
       case _: java.io.FilePermission                                                        => ()
       case x: java.security.SecurityPermission if x.getName contains ".networkaddress."     => () // generality ftw
       case x: java.util.PropertyPermission if x.getName == "sun.net.inetaddr.ttl"           => ()
+      case _: java.lang.reflect.ReflectPermission                                           => () // needed for LambdaMetaFactory
       case _                                                                                => super.checkPermission(perm)
     }
   }