Initial commit of LOGO.

project/plugins.sbt

@@ -8,6 +8,7 @@ resolvers += Classpaths.sbtPluginReleases
 resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
 
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.13.0")
+addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "4.0.0")
 
 addSbtPlugin("org.scalanlp" % "sbt-jflex" % "0.3")
 

src/main/scala/epic/logo/ArgmaxInferencer.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait ArgmaxInferencer[T, Y, W] {
+
+  def argmax(weights : Weights[W], instance : T) : (Y, W, Double)
+
+}

src/main/scala/epic/logo/CompoundIterationCallback.scala

@@ -0,0 +1,28 @@
+package epic.logo
+
+class CompoundIterationCallback[T, W](val callbacks : Iterable[IterationCallback[T, W]]) extends IterationCallback[T, W] {
+
+  override def startIteration(iter : Int, weights : Weights[W]) : Unit = {
+    callbacks.foreach(_.startIteration(iter, weights))
+  }
+
+  override def startMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchInput[T, W]]) : Unit = {
+    callbacks.foreach(_.startMinibatch(iter, weights, miniBatch))
+  }
+  override def endMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchOutput[T, W]]) : Unit = {
+    callbacks.foreach(_.endMinibatch(iter, weights, miniBatch))
+  }
+
+  override def endIteration(iter : Int, weights : Weights[W]) : Unit = {
+    callbacks.foreach(_.endIteration(iter, weights))
+  }
+
+  override def objectiveValCheck(primal : Double, dual : Double, iter : Int, w : Weights[W]) : Unit = {
+    callbacks.foreach(_.objectiveValCheck(primal, dual, iter, w))
+  }
+
+  override def converged(weights : Weights[W], data : Seq[Instance[T, W]], iter : Int, numNewConstraints : Int) : Boolean = {
+    callbacks.forall(_.converged(weights, data, iter, numNewConstraints))
+  }
+
+}

src/main/scala/epic/logo/ConvergenceChecker.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait ConvergenceChecker[W] {
+
+  def converged(weights : Weights[W], data : Seq[Instance[_, W]], iter : Int, numNewConstraints : Int) : Boolean
+
+}

src/main/scala/epic/logo/Decoder.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait Decoder[T, W] {
+
+  def decode(weights : Weights[W], instance : T) : (W, Double)
+
+}

src/main/scala/epic/logo/ExpectationInferencer.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait ExpectationInferencer[T, W] {
+
+  def expectations(weights : Weights[W], instance : T) : (W, Double)
+
+}

src/main/scala/epic/logo/FixedIterationConvergenceChecker.scala

@@ -0,0 +1,11 @@
+package epic.logo
+
+import scala.collection.Seq
+
+class FixedIterationConvergenceChecker[W](val maxNumIters : Int) extends ConvergenceChecker[W] {
+
+  def converged(weights : Weights[W], data : Seq[Instance[_, W]], iter : Int, numNewConstraints : Int) : Boolean = {
+    iter >= maxNumIters
+  }
+
+}

src/main/scala/epic/logo/FixedStepSizeUpdater.scala

@@ -0,0 +1,30 @@
+package epic.logo
+import scala.collection.mutable.Buffer
+import scala.runtime.DoubleRef
+import breeze.linalg.norm
+import breeze.math.MutableInnerProductModule
+
+class FixedStepSizeUpdater[W](stepSize : Int => Double, C : Double)(implicit space: MutableInnerProductModule[W, Double]) extends Updater[W] {
+  import space._
+
+  def update(constraints: IndexedSeq[(W, Double)], alphas: Buffer[Double], slack: DoubleRef,
+             w: Weights[W], n: Int, iter: Int): Boolean = {
+    assert(constraints.length == 2)
+    val (df, _) = constraints(0)
+    if (norm(df) == 0.0) return false
+
+    val eta = stepSize(iter)
+    if (C == Double.PositiveInfinity) {
+      w += df * eta
+    } else {
+      w *= (1.0 - eta)
+      w += df * (eta * C / n)
+    }
+    return true
+  }
+
+  def currentSlack(i : Instance[_, W], w : Weights[W]) : Double = {
+    throw new UnsupportedOperationException(this.getClass().getName() + " should be only be used in online mode.")
+  }
+
+}

src/main/scala/epic/logo/Instance.scala

@@ -0,0 +1,13 @@
+package epic.logo
+
+import scala.collection.mutable.Buffer
+import scala.collection.mutable.ArrayBuffer
+import scala.runtime.DoubleRef
+
+class Instance[T, W](val x : T) {
+
+  var slack = new DoubleRef(0.0)
+
+  val alphas = new ArrayBuffer[Double]() //(1.0)
+  val constraints = new ArrayBuffer[(W, Double)]()
+}

src/main/scala/epic/logo/IterationCallback.scala

@@ -0,0 +1,17 @@
+package epic.logo
+
+trait IterationCallback[T, W] {
+
+  def startIteration(iter : Int, weights : Weights[W]) : Unit = {}
+
+  def startMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchInput[T, W]]) : Unit = {}
+  def endMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchOutput[T, W]]) : Unit = {}
+  def endIteration(iter : Int, weights : Weights[W]) : Unit = {}
+
+  def objectiveValCheck(primal : Double, dual : Double, iter : Int, weights : Weights[W]) : Unit = {}
+
+  def converged(weights : Weights[W], data : Seq[Instance[T, W]], iter : Int, numNewConstraints : Int) : Boolean = false
+
+}
+
+case class NullIterationCallback[T, W]() extends IterationCallback[T, W]

src/main/scala/epic/logo/L1Objective.scala

@@ -0,0 +1,39 @@
+package epic.logo
+
+import breeze.linalg._
+import breeze.math.MutableInnerProductModule
+
+object L1Objective {
+  def slack[W](i : Instance[_, W], w : Weights[W])(implicit space: MutableInnerProductModule[W, Double]) = {
+    import space._
+    if (i.constraints.isEmpty)
+      Double.NegativeInfinity
+    else
+      i.constraints.map { case (df, l) => l - w * df }.max
+  }
+
+}
+
+class L1Objective[W](val C : Double)(implicit space: MutableInnerProductModule[W, Double]) extends ObjectiveFunction[W] {
+  import space._
+
+  def calculatePrimalAndDual(w : Weights[W], data : Seq[Instance[_, W]]): (Double, Double) = {
+    w.checkNorm
+    val calc_w = new Weights(space.zeroLike(w.underlying))
+    data.flatMap(instance => instance.alphas zip instance.constraints).map { case (alpha, (df, l)) => df * alpha }.foreach(fv => {
+      calc_w += fv
+    })
+    assert(data.forall(instance => NumUtils.approxEquals(C, instance.alphas.sum, 1e-5)))
+    assert(w.approxEquals(calc_w))
+    val primal = {
+      val slackSum = data.map(i => L1Objective.slack(i, w)).sum
+      0.5 * (w.`^2`) + C * slackSum
+    }
+    val dual = {
+      val lossSum = data.flatMap(instance => instance.alphas zip instance.constraints).map { case (alpha, (df, l)) => alpha * l }.sum
+      -0.5 * (w.`^2`) + lossSum
+    }
+    (primal, dual)
+  }
+
+}

src/main/scala/epic/logo/L1Updater.scala

@@ -0,0 +1,53 @@
+package epic.logo
+
+import scala.collection.mutable.Buffer
+import scala.runtime.DoubleRef
+import scala.util.Random
+import breeze.math.MutableInnerProductModule
+
+class L1Updater[W](C : Double)(implicit space: MutableInnerProductModule[W, Double]) extends Updater[W] {
+  val shuffleRand = new Random()
+  import space._
+
+  def update(constraints : IndexedSeq[(W, Double)], alphas : Buffer[Double], slack : DoubleRef, w : Weights[W], n : Int, iter : Int) : Boolean = {
+    if (constraints.length == 1) {
+      if (alphas(0) != C) {
+        val eta = C - alphas(0)
+        val (df, l) = constraints(0)
+        if (eta != 0.0) {
+          alphas(0) += eta
+          w += df * eta
+          return true
+        }
+      }
+      return false
+    } else {
+      var anyChange = false
+      val indexed = constraints.zipWithIndex
+      val ((df1, l1), s1) = indexed.maxBy { case ((df, l), s) => l - w * df }
+
+      for (((df2, l2), s2) <- shuffleRand.shuffle(indexed) if s2 != s1 && alphas(s2) > 0.0) {
+        val diff = df2 - df1
+        val num = (l2 - l1) - w * diff
+        if (num != 0.0) {
+          val denom = diff dot diff
+          if (denom != 0.0) {
+            val eta = clip(num / denom, -alphas(s2), +alphas(s1))
+            if (eta != 0.0) {
+              alphas(s2) += eta
+              alphas(s1) -= eta
+              w += diff * eta
+              anyChange = true
+            }
+          }
+        }
+
+      }
+      return anyChange
+    }
+  }
+
+  def currentSlack(i : Instance[_, W], w : Weights[W]) = {
+    L1Objective.slack(i, w)
+  }
+}

src/main/scala/epic/logo/L2Objective.scala

@@ -0,0 +1,24 @@
+package epic.logo
+
+import breeze.math.MutableInnerProductModule
+
+class L2Objective[W](val C : Double)(implicit space: MutableInnerProductModule[W, Double]) extends ObjectiveFunction[W] {
+
+  def calculatePrimalAndDual(w : Weights[W], data : Seq[Instance[_, W]]) : (Double, Double) = {
+
+    val primal = {
+      val slackSum = data.map(instance => instance.slack.elem * instance.slack.elem).sum
+      0.5 * (w.`^2`) + C * slackSum
+    }
+    val dual = {
+      val lossSum = data.flatMap(instance => instance.alphas zip instance.constraints).map {
+        case (alpha, (df, l)) => alpha * l
+      }.sum
+      C * lossSum - primal
+    }
+    (primal, dual)
+  }
+
+
+
+}

src/main/scala/epic/logo/L2Updater.scala

@@ -0,0 +1,37 @@
+package epic.logo
+
+import scala.collection.mutable.Buffer
+import scala.runtime.DoubleRef
+import breeze.math.MutableInnerProductModule
+
+class L2Updater[W](C : Double)(implicit space: MutableInnerProductModule[W, Double]) extends Updater[W] {
+  import space._
+
+  def update(constraints: IndexedSeq[(W, Double)], alphas: Buffer[Double], slack: DoubleRef,
+             w: Weights[W], n: Int, iter: Int): Boolean = {
+    for (((df, l), s) <- constraints.zipWithIndex) {
+      val wTdf = w * df
+      val num = l - wTdf - slack.elem
+      if (num != 0.0) {
+        val denom = (1.0 / C) + (df dot df)
+        if (denom != 0.0) {
+          val eta = clip(num / denom, -alphas(s), Double.PositiveInfinity)
+          if (eta != 0.0) {
+            alphas(s) += eta
+            slack.elem += eta / C
+            w += df * eta
+            return true
+
+          }
+        }
+      }
+    }
+    return false
+
+  }
+
+  def currentSlack(i : Instance[_, W], w : Weights[W]) = {
+    i.slack.elem
+  }
+
+}

src/main/scala/epic/logo/LogLikelihoodDecoder.scala

@@ -0,0 +1,16 @@
+package epic.logo
+
+import breeze.math.MutableInnerProductModule
+
+class LogLikelihoodDecoder[T, W](
+    val inferencer: OracleInferencer[T, _, W], val summer: ExpectationInferencer[T, W])(
+        implicit space: MutableInnerProductModule[W, Double]) extends Decoder[T, W] {
+  import space._
+
+  def decode(weights : Weights[W], instance : T) : (W, Double) = {
+    val (y_*, f_*, l_*) = inferencer.oracle(weights, instance)
+    val (f, l) = summer.expectations(weights, instance)
+    ((f_* - f), l - l_*)
+  }
+
+}

src/main/scala/epic/logo/LogoOpts.scala

@@ -0,0 +1,16 @@
+package epic.logo
+
+
+case class LogoOpts(constraintEpsilon: Double = 1e-3,
+
+	miniBatchSize: Int = 1,
+
+	numInnerOptimizationLoops: Int = 10,
+
+	numOuterOptimizationLoops: Int = 10,
+
+	shuffleMinibatches: Boolean = false,
+
+	shuffleSeed: Int = -1,
+
+	convergenceTolerance: Double = 1e-3)

src/main/scala/epic/logo/LossAugmentedArgmaxInferencer.scala

@@ -0,0 +1,9 @@
+package epic.logo
+
+trait LossAugmentedArgmaxInferencer[T, Y, W] extends ArgmaxInferencer[T, Y, W] {
+
+  def argmax(weights : Weights[W], instance : T) : (Y, W, Double) = lossAugmentedArgmax(weights, instance, 1.0, 0.0)
+
+  def lossAugmentedArgmax(weights : Weights[W], instance : T, weightsWeight : Double, lossWeight : Double) : (Y, W, Double)
+
+}

src/main/scala/epic/logo/LossAugmentedExpectationInferencer.scala

@@ -0,0 +1,9 @@
+package epic.logo
+
+trait LossAugmentedExpectationInferencer[T, W] extends ExpectationInferencer[T, W] {
+
+  def expectations(weights : Weights[W], instance : T) : (W, Double) = lossAugmentedExpectations(weights, instance, 1.0, 0.0)
+
+  def lossAugmentedExpectations(weights : Weights[W], instance : T, weightsWeight : Double, lossWeight : Double) : (W, Double)
+
+}

src/main/scala/epic/logo/LossAugmentedMaxMarginDecoder.scala

@@ -0,0 +1,14 @@
+package epic.logo
+
+import breeze.math.MutableInnerProductModule
+
+class LossAugmentedMaxMarginDecoder[T, W](val oracleInferencer: OracleInferencer[T, _, W], val argmaxer: LossAugmentedArgmaxInferencer[T, _, W])(implicit space: MutableInnerProductModule[W, Double]) extends Decoder[T, W] {
+  import space._
+
+  def decode(weights: Weights[W], instance: T): (W, Double) = {
+    val (y_*, f_*, l_*) = oracleInferencer.oracle(weights, instance)
+    val (y, f, l) = argmaxer.lossAugmentedArgmax(weights, instance, 1.0, 1.0)
+    (f_* - f, l - l_*)
+  }
+
+}

src/main/scala/epic/logo/MaxMarginDecoder.scala

@@ -0,0 +1,14 @@
+package epic.logo
+
+import breeze.math.MutableInnerProductModule
+
+class MaxMarginDecoder[T, W](val oracleInferencer : OracleInferencer[T, _, W], val argmaxer : ArgmaxInferencer[T, _, W])(implicit space: MutableInnerProductModule[W, Double]) extends Decoder[T, W] {
+  import space._
+
+  def decode(weights : Weights[W], instance : T) : (W, Double) = {
+    val (y_*, f_*, l_*) = oracleInferencer.oracle(weights, instance)
+    val (y, f, l) = argmaxer.argmax(weights, instance)
+    (f_* - f, l - l_*)
+  }
+
+}

src/main/scala/epic/logo/MaxMarginRankingDecoder.scala

@@ -0,0 +1,13 @@
+package epic.logo
+
+import breeze.math.MutableInnerProductModule
+
+class MaxMarginRankingDecoder[T, W](val inferencer : LossAugmentedArgmaxInferencer[T, _, W], val gamma : Double = 0.0)(implicit space: MutableInnerProductModule[W, Double]) extends Decoder[T, W] {
+import space._
+  def decode(weights: Weights[W], instance: T): (W, Double) = {
+    val (y_min, df_min, l_min) = inferencer.lossAugmentedArgmax(weights, instance, -1.0, -(1.0 + gamma))
+    val (y_max, df_max, l_max) = inferencer.lossAugmentedArgmax(weights, instance, 1.0, 1.0)
+    (df_min - df_max, l_max - (1.0 + gamma) * l_min)
+  }
+
+}