Finished caching implementation of the first layer

src/main/scala/epic/dense/CachingLookupAndAffineTransformDense.scala

@@ -34,15 +34,30 @@ case class CachingLookupAndAffineTransformDense[FV](numOutputs: Int,
     val weightst = weights.t
 
     val cache = new HashMap[(Int,Int),DenseVector[Double]]
+//    val caches = Array.tabulate(6)(i => new HashMap[Int,DenseVector[Double]])
 
     def activations(fv: Array[Int]) = {
       val finalVector = DenseVector.zeros[Double](numOutputs)
       for (i <- 0 until fv.size) {
         val wordPosn = fv(i) -> i
-        if (!cache.contains(wordPosn)) {
-          cache.put(wordPosn, weights(i to i+numOutputs, ::) * DenseVector(word2vecFeaturizer.word2vec(wordPosn._1)))
+        if (fv(i) != -1) {
+//          caches(i).synchronized {
+//            if (!caches(i).contains(fv(i))) {
+//              val startIdx = i * word2vecFeaturizer.wordRepSize
+//              caches(i).put(fv(i), weights(::, startIdx until startIdx + word2vecFeaturizer.wordRepSize) * DenseVector(word2vecFeaturizer.word2vec(wordPosn._1)))
+//            }
+//            finalVector += caches(i)(fv(i))
+//          }
+          cache.synchronized {
+            if (!cache.contains(wordPosn)) {
+              val startIdx = i * word2vecFeaturizer.wordRepSize
+              cache.put(wordPosn, weights(::, startIdx until startIdx + word2vecFeaturizer.wordRepSize) * DenseVector(word2vecFeaturizer.word2vec(wordPosn._1)))
+            }
+            finalVector += cache(wordPosn)
+          }
+//          val startIdx = i * word2vecFeaturizer.wordRepSize
+//          finalVector += weights(::, startIdx until startIdx + word2vecFeaturizer.wordRepSize) * DenseVector(word2vecFeaturizer.word2vec(wordPosn._1))
         }
-        finalVector += cache(wordPosn)
       }
       finalVector + bias
     }

src/main/scala/epic/dense/Word2VecSurfaceFeaturizer.scala

@@ -59,10 +59,11 @@ class Word2VecSurfaceFeaturizerIndexed[W](val wordIndex: Index[W],
 
   def assemble(vectors: Seq[Array[Double]]) = vectors.reduce(_ ++ _)
 
-  def convertToVector(indexedWords: Array[Int]) = assemble(indexedWords.map(word2vec(_)))
+  def convertToVector(indexedWords: Array[Int]) = assemble(indexedWords.map(wordIdx => if (wordIdx == -1) zeroVector else word2vec(wordIdx)))
 
   def anchor(words: IndexedSeq[W]): WordVectorAnchoringIndexed[W] = {
     val convertedWords = words.map(converter(_))
+    val indexedWords = convertedWords.map(wordIndex(_))
     new WordVectorAnchoringIndexed[W] {
 
       def featuresForSpan(start: Int, end: Int) = {
@@ -72,11 +73,15 @@ class Word2VecSurfaceFeaturizerIndexed[W](val wordIndex: Index[W],
 
       def featuresForSplit(start: Int, split: Int, end: Int) = {
 //        val vect = new DenseVector[Double](assemble(Seq(fetchVector(start - 1), fetchVector(start), fetchVector(split - 1), fetchVector(split), fetchVector(end - 1), fetchVector(end))))
-        Array(fetchWord(start - 1), fetchWord(start), -1, fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end))
+        Array(fetchWord(start - 1), fetchWord(start), fetchWord(split - 1), fetchWord(split), fetchWord(end - 1), fetchWord(end))
       }
 
+//      private def fetchWord(idx: Int): Int = {
+//        if (idx < 0 || idx >= words.size || !word2vec.contains(convertedWords(idx))) -1 else wordIndex(convertedWords(idx))
+//      }
+
       private def fetchWord(idx: Int): Int = {
-        if (idx < 0 || idx >= words.size || !word2vec.contains(convertedWords(idx))) -1 else wordIndex(convertedWords(idx))
+        if (idx < 0 || idx >= words.size) -1 else indexedWords(idx)
       }
     } 
   }

src/main/scala/epic/parser/models/PositionalNeuralModel.scala

@@ -85,17 +85,17 @@ You can also epic.trees.annotations.KMAnnotator to get more or less Klein and Ma
     // Convert Array[Float] values to DenseVector[Double] values
     val word2vecDoubleVect = word2vec.map(keyValue => (keyValue._1 -> keyValue._2.map(_.toDouble)))
 //    val word2vecDoubleVect = word2vec.map(keyValue => (keyValue._1 -> new DenseVector[Double](keyValue._2.map(_.toDouble))))
-    val surfaceFeaturizer = new Word2VecSurfaceFeaturizer(word2vecDoubleVect, (str: String) => Word2Vec.convertWord(str))
 
-    val transform = if (useNonlinearity) {
-      // Affine transform of word embeddings, tanh, affine transform to output layer
-      new AffineTransformDense(featurizer.index.size, numHidden, new TanhTransform(new AffineTransformDense(numHidden, surfaceFeaturizer.vectorSize, new IdentityTransform[DenseVector[Double]]())))
-    } else {
-      new AffineTransformDense(featurizer.index.size, surfaceFeaturizer.vectorSize, new IdentityTransform[DenseVector[Double]]())
-    }
+//    val surfaceFeaturizer = new Word2VecSurfaceFeaturizer(word2vecDoubleVect, (str: String) => Word2Vec.convertWord(str))
+//    val transform = if (useNonlinearity) {
+//      // Affine transform of word embeddings, tanh, affine transform to output layer
+//      new AffineTransformDense(featurizer.index.size, numHidden, new TanhTransform(new AffineTransformDense(numHidden, surfaceFeaturizer.vectorSize, new IdentityTransform[DenseVector[Double]]())))
+//    } else {
+//      new AffineTransformDense(featurizer.index.size, surfaceFeaturizer.vectorSize, new IdentityTransform[DenseVector[Double]]())
+//    }
 
-//    val indexedSurfaceFeaturizer = Word2VecSurfaceFeaturizerIndexed(word2vecDoubleVect, (str: String) => Word2Vec.convertWord(str))
-//    val transform = new AffineTransformDense(featurizer.index.size, numHidden, new TanhTransform(new CachingLookupAndAffineTransformDense(numHidden, surfaceFeaturizer.vectorSize, indexedSurfaceFeaturizer)))
+    val surfaceFeaturizer = Word2VecSurfaceFeaturizerIndexed(word2vecDoubleVect, (str: String) => Word2Vec.convertWord(str))
+    val transform = new AffineTransformDense(featurizer.index.size, numHidden, new TanhTransform(new CachingLookupAndAffineTransformDense(numHidden, surfaceFeaturizer.vectorSize, surfaceFeaturizer)))
 
     println(surfaceFeaturizer.vectorSize + " x " + numHidden + " x " + featurizer.index.size + " neural net")
 

src/main/scala/epic/parser/models/PositionalTransformModel.scala

@@ -22,6 +22,7 @@ import scala.collection.mutable.HashMap
 import epic.dense.Transform
 import epic.dense.AffineTransformDense
 import epic.dense.Word2VecSurfaceFeaturizer
+import epic.dense.Word2VecSurfaceFeaturizerIndexed
 
 /**
  * TODO
@@ -35,8 +36,8 @@ class PositionalTransformModel[L, L2, W](annotator: (BinarizedTree[L], IndexedSe
                                refinedTopology: RuleTopology[L2],
                                refinements: GrammarRefinements[L, L2],
                                labelFeaturizer: RefinedFeaturizer[L, W, Feature],
-                               surfaceFeaturizer: Word2VecSurfaceFeaturizer[W],
-                               val transform: AffineTransformDense[DenseVector[Double]]) extends ParserModel[L, W] {
+                               surfaceFeaturizer: Word2VecSurfaceFeaturizerIndexed[W],
+                               val transform: AffineTransformDense[Array[Int]]) extends ParserModel[L, W] {
   override type Inference = PositionalTransformModel.Inference[L, L2, W]
 
 
@@ -55,7 +56,6 @@ class PositionalTransformModel[L, L2, W](annotator: (BinarizedTree[L], IndexedSe
 
   override def inferenceFromWeights(weights: DenseVector[Double]): Inference = {
     val (layer, innerLayer) = transform.extractLayerAndPenultimateLayer(weights)
-
     val grammar = new PositionalTransformModel.PositionalTransformGrammar[L, L2, W](topology, lexicon, refinedTopology, refinements, labelFeaturizer, surfaceFeaturizer, layer, innerLayer)
     new Inference(annotator, constrainer, grammar, refinements)
   }
@@ -86,9 +86,9 @@ object PositionalTransformModel {
                                    val refinedTopology: RuleTopology[L2],
                                    val refinements: GrammarRefinements[L, L2],
                                    labelFeaturizer: RefinedFeaturizer[L, W, Feature],
-                                   surfaceFeaturizer: Word2VecSurfaceFeaturizer[W],
-                                   layer: AffineTransformDense[DenseVector[Double]]#Layer,
-                                   penultimateLayer: epic.dense.Transform.Layer[DenseVector[Double],DenseVector[Double]]) extends Grammar[L, W] with Serializable {
+                                   surfaceFeaturizer: Word2VecSurfaceFeaturizerIndexed[W],
+                                   layer: AffineTransformDense[Array[Int]]#Layer,
+                                   penultimateLayer: epic.dense.Transform.Layer[Array[Int],DenseVector[Double]]) extends Grammar[L, W] with Serializable {
 
     override def withPermissiveLexicon: Grammar[L, W] = {
       new PositionalTransformGrammar(topology, lexicon.morePermissive, refinedTopology, refinements, labelFeaturizer, surfaceFeaturizer, layer, penultimateLayer)