Some progress towards autogenerating the lucene analyzer stages but not quite there yet.

Author: holdenk

build.sbt

@@ -36,6 +36,7 @@ libraryDependencies ++= Seq(
   "com.lucidworks.spark" % "spark-solr" % "3.0.2",
   // internals
   "org.scala-lang" % "scala-reflect" % "2.11.7",
+  "org.reflections" % "reflections" % "0.9.11",
   // testing libraries
   "org.scalatest" %% "scalatest" % "3.0.1",
   "org.scalacheck" %% "scalacheck" % "1.13.4",

src/main/scala/com/sparklingpandas/sparklingml/feature/LuceneAnalyzer.scala

@@ -1,4 +1,4 @@
-package com.sparklingpandas.sparklingml
+package com.sparklingpandas.sparklingml.feature
 
 import org.apache.spark.annotation.DeveloperApi
 import org.apache.spark.ml.UnaryTransformer

src/main/scala/com/sparklingpandas/sparklingml/feature/LuceneAnalyzerGenerators.scala

@@ -0,0 +1,28 @@
+package com.sparklingpandas.sparklingml.feature
+
+import org.apache.spark.annotation.DeveloperApi
+
+import org.apache.lucene.analysis.Analyzer
+
+@DeveloperApi
+object LuceneAnalyzerGenerators {
+  def generate() = {
+    import org.reflections.Reflections
+    import collection.JavaConverters._
+    import scala.reflect.runtime.universe._
+    val reflections = new Reflections("org.apache.lucene");
+    val analyzers = reflections.getSubTypesOf(classOf[org.apache.lucene.analysis.Analyzer])
+    val rm = scala.reflect.runtime.currentMirror
+    val generated = analyzers.asScala.map{ cls =>
+      val constructors = rm.classSymbol(cls).toType.members.collect{
+        case m: MethodSymbol if m.isConstructor && m.isPublic => m }
+      // Since this isn't built with -parameters by default :(
+      // we'd need a local version built with it to auto generate
+      // the code here with the right parameters.
+      // https://docs.oracle.com/javase/tutorial/reflect/member/methodparameterreflection.html
+      // For now we could dump the class names and go from their
+      // or we could play a game of pin the field on the constructor.
+      // local build sounds like the best plan, lets do that l8r
+    }
+  }
+}