Initial commit of LOGO.

src/main/scala/epic/logo/ArgmaxInferencer.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait ArgmaxInferencer[T, Y, W] {
+
+  def argmax(weights : Weights[W], instance : T) : (Y, FeatureVector[W], Double)
+
+}

src/main/scala/epic/logo/CompoundIterationCallback.scala

@@ -0,0 +1,28 @@
+package epic.logo
+
+class CompoundIterationCallback[T, W](val callbacks : Iterable[IterationCallback[T, W]]) extends IterationCallback[T, W] {
+
+  override def startIteration(iter : Int, weights : Weights[W]) : Unit = {
+    callbacks.foreach(_.startIteration(iter, weights))
+  }
+
+  override def startMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchInput[T, W]]) : Unit = {
+    callbacks.foreach(_.startMinibatch(iter, weights, miniBatch))
+  }
+  override def endMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchOutput[T, W]]) : Unit = {
+    callbacks.foreach(_.endMinibatch(iter, weights, miniBatch))
+  }
+
+  override def endIteration(iter : Int, weights : Weights[W]) : Unit = {
+    callbacks.foreach(_.endIteration(iter, weights))
+  }
+
+  override def objectiveValCheck(primal : Double, dual : Double, iter : Int, w : Weights[W]) : Unit = {
+    callbacks.foreach(_.objectiveValCheck(primal, dual, iter, w))
+  }
+
+  override def converged(weights : Weights[W], data : Seq[Instance[T, W]], iter : Int, numNewConstraints : Int) : Boolean = {
+    callbacks.forall(_.converged(weights, data, iter, numNewConstraints))
+  }
+
+}

src/main/scala/epic/logo/ConvergenceChecker.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait ConvergenceChecker[W] {
+
+  def converged(weights : Weights[W], data : Seq[Instance[_, W]], iter : Int, numNewConstraints : Int) : Boolean
+
+}

src/main/scala/epic/logo/Decoder.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait Decoder[T, W] {
+
+  def decode(weights : Weights[W], instance : T) : (FeatureVector[W], Double)
+
+}

src/main/scala/epic/logo/ExpectationInferencer.scala

@@ -0,0 +1,7 @@
+package epic.logo
+
+trait ExpectationInferencer[T, W] {
+
+  def expectations(weights : Weights[W], instance : T) : (FeatureVector[W], Double)
+
+}

src/main/scala/epic/logo/FeatureVector.scala

@@ -0,0 +1,50 @@
+package epic.logo
+
+import scala.collection.JavaConversions._
+import breeze.linalg._
+import breeze.math.MutableInnerProductModule
+
+case class FeatureVector[W](var data :W)(implicit space: MutableInnerProductModule[W, Double]) {
+  import space._
+
+  def normSquared() = data dot data
+
+  def ^(x : Int) = {
+    assert(x == 2)
+    normSquared()
+  }
+
+
+  override def toString() = data.toString()
+
+  def dotProduct(w : Weights[W]) = (data dot w.array) * w.scale
+
+  def *(w : Weights[W]) = dotProduct(w)
+
+  def *(fv : FeatureVector[W]) = data dot fv.data
+
+  def *(c : Double) = {
+    new FeatureVector(data * c)
+  }
+
+  def subtract(that : FeatureVector[W]) = {
+    new FeatureVector(data - that.data)
+  }
+
+  def add(that : FeatureVector[W]) = {
+     new FeatureVector(data + that.data)
+  }
+
+  def addWeighted(that : FeatureVector[W], c : Double) = {
+    axpy(c, that.data, data)
+  }
+
+  def -(that : FeatureVector[W]) = subtract(that)
+
+  def +(that : FeatureVector[W]) = add(that)
+
+  def +=(that : FeatureVector[W]) = addWeighted(that, 1.0)
+
+  def -=(that : FeatureVector[W]) = addWeighted(that, -1.0)
+
+}

src/main/scala/epic/logo/FixedIterationConvergenceChecker.scala

@@ -0,0 +1,11 @@
+package epic.logo
+
+import scala.collection.Seq
+
+class FixedIterationConvergenceChecker[W](val maxNumIters : Int) extends ConvergenceChecker[W] {
+
+  def converged(weights : Weights[W], data : Seq[Instance[_, W]], iter : Int, numNewConstraints : Int) : Boolean = {
+    iter >= maxNumIters
+  }
+
+}

src/main/scala/epic/logo/FixedStepSizeUpdater.scala

@@ -0,0 +1,27 @@
+package epic.logo
+import scala.collection.mutable.Buffer
+import scala.runtime.DoubleRef
+
+class FixedStepSizeUpdater[W](stepSize : Int => Double, C : Double) extends Updater[W] {
+
+  def update(constraints: IndexedSeq[(FeatureVector[W], Double)], alphas: Buffer[Double], slack: DoubleRef,
+             w: Weights[W], n: Int, iter: Int): Boolean = {
+    assert(constraints.length == 2)
+    val (df, _) = constraints(0)
+    if ((df ^ 2) == 0.0) return false
+
+    val eta = stepSize(iter)
+    if (C == Double.PositiveInfinity) {
+      w += df * eta
+    } else {
+      w *= (1.0 - eta)
+      w += df * (eta * C / n)
+    }
+    return true
+  }
+
+  def currentSlack(i : Instance[_, W], w : Weights[W]) : Double = {
+    throw new UnsupportedOperationException(this.getClass().getName() + " should be only be used in online mode.")
+  }
+
+}

src/main/scala/epic/logo/Instance.scala

@@ -0,0 +1,13 @@
+package epic.logo
+
+import scala.collection.mutable.Buffer
+import scala.collection.mutable.ArrayBuffer
+import scala.runtime.DoubleRef
+
+class Instance[T, W](val x : T) {
+
+  var slack = new DoubleRef(0.0)
+
+  val alphas = new ArrayBuffer[Double]() //(1.0)
+  val constraints = new ArrayBuffer[(FeatureVector[W], Double)]()
+}

src/main/scala/epic/logo/IterationCallback.scala

@@ -0,0 +1,17 @@
+package epic.logo
+
+trait IterationCallback[T, W] {
+
+  def startIteration(iter : Int, weights : Weights[W]) : Unit = {}
+
+  def startMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchInput[T, W]]) : Unit = {}
+  def endMinibatch(iter : Int, weights : Weights[W], miniBatch : Array[MinibatchOutput[T, W]]) : Unit = {}
+  def endIteration(iter : Int, weights : Weights[W]) : Unit = {}
+
+  def objectiveValCheck(primal : Double, dual : Double, iter : Int, weights : Weights[W]) : Unit = {}
+
+  def converged(weights : Weights[W], data : Seq[Instance[T, W]], iter : Int, numNewConstraints : Int) : Boolean = false
+
+}
+
+case class NullIterationCallback[T, W]() extends IterationCallback[T, W]

src/main/scala/epic/logo/L1Objective.scala

@@ -0,0 +1,37 @@
+package epic.logo
+
+import breeze.linalg._
+import breeze.math.MutableInnerProductModule
+
+object L1Objective {
+  def slack[W](i : Instance[_, W], w : Weights[W]) = {
+    if (i.constraints.isEmpty)
+      Double.NegativeInfinity
+    else
+      i.constraints.map { case (df, l) => l - df * w }.max
+  }
+
+}
+
+class L1Objective[W](val C : Double)(implicit space: MutableInnerProductModule[W, Double]) extends ObjectiveFunction[W] {
+
+  def calculatePrimalAndDual(w : Weights[W], data : Seq[Instance[_, W]]): (Double, Double) = {
+    w.checkNorm
+    val calc_w = new Weights(space.zeroLike(w.array))
+    data.flatMap(instance => instance.alphas zip instance.constraints).map { case (alpha, (df, l)) => df * alpha }.foreach(fv => {
+      calc_w += fv
+    })
+    assert(data.forall(instance => NumUtils.approxEquals(C, instance.alphas.sum, 1e-5)))
+    assert(w.approxEquals(calc_w))
+    val primal = {
+      val slackSum = data.map(i => L1Objective.slack(i, w)).sum
+      0.5 * (w ^ 2) + C * slackSum
+    }
+    val dual = {
+      val lossSum = data.flatMap(instance => instance.alphas zip instance.constraints).map { case (alpha, (df, l)) => alpha * l }.sum
+      -0.5 * (w ^ 2) + lossSum
+    }
+    (primal, dual)
+  }
+
+}

src/main/scala/epic/logo/L1Updater.scala

@@ -0,0 +1,51 @@
+package epic.logo
+
+import scala.collection.mutable.Buffer
+import scala.runtime.DoubleRef
+import scala.util.Random
+
+class L1Updater[W](C : Double) extends Updater[W] {
+   val shuffleRand = new Random()
+
+  def update(constraints : IndexedSeq[(FeatureVector[W], Double)], alphas : Buffer[Double], slack : DoubleRef, w : Weights[W], n : Int, iter : Int) : Boolean = {
+    if (constraints.length == 1) {
+      if (alphas(0) != C) {
+        val eta = C - alphas(0)
+        val (df, l) = constraints(0)
+        if (eta != 0.0) {
+          alphas(0) += eta
+          w += df * eta
+          return true
+        }
+      }
+      return false
+    } else {
+      var anyChange = false
+      val indexed = constraints.zipWithIndex
+      val ((df1, l1), s1) = indexed.maxBy { case ((df, l), s) => l - w * df }
+
+      for (((df2, l2), s2) <- shuffleRand.shuffle(indexed) if s2 != s1 && alphas(s2) > 0.0) {
+        val diff = df2 - df1
+        val num = (l2 - l1) - w * diff
+        if (num != 0.0) {
+          val denom = diff ^ 2
+          if (denom != 0.0) {
+            val eta = clip(num / denom, -alphas(s2), +alphas(s1))
+            if (eta != 0.0) {
+              alphas(s2) += eta
+              alphas(s1) -= eta
+              w += diff * eta
+              anyChange = true
+            }
+          }
+        }
+
+      }
+      return anyChange
+    }
+  }
+
+  def currentSlack(i : Instance[_, W], w : Weights[W]) = {
+    L1Objective.slack(i, w)
+  }
+}

src/main/scala/epic/logo/L2Objective.scala

@@ -0,0 +1,22 @@
+package epic.logo
+
+import breeze.math.MutableInnerProductModule
+
+class L2Objective[W](val C : Double)(implicit space: MutableInnerProductModule[W, Double]) extends ObjectiveFunction[W] {
+
+  def calculatePrimalAndDual(w : Weights[W], data : Seq[Instance[_, W]]) : (Double, Double) = {
+
+    val primal = {
+      val slackSum = data.map(instance => instance.slack.elem * instance.slack.elem).sum
+      0.5 * (w ^ 2) + C * slackSum
+    }
+    val dual = {
+      val lossSum = data.flatMap(instance => instance.alphas zip instance.constraints).map { case (alpha, (df, l)) => alpha * l }.sum
+      C * lossSum - primal
+    }
+    (primal, dual)
+  }
+
+
+
+}

src/main/scala/epic/logo/L2Updater.scala

@@ -0,0 +1,35 @@
+package epic.logo
+
+import scala.collection.mutable.Buffer
+import scala.runtime.DoubleRef
+
+class L2Updater[W](C : Double) extends Updater[W] {
+
+  def update(constraints: IndexedSeq[(FeatureVector[W], Double)], alphas: Buffer[Double], slack: DoubleRef,
+             w: Weights[W], n: Int, iter: Int): Boolean = {
+    for (((df, l), s) <- constraints.zipWithIndex) {
+      val wTdf = df * w
+      val num = l - wTdf - slack.elem
+      if (num != 0.0) {
+        val denom = (1.0 / C) + (df ^ 2)
+        if (denom != 0.0) {
+          val eta = clip(num / denom, -alphas(s), Double.PositiveInfinity)
+          if (eta != 0.0) {
+            alphas(s) += eta
+            slack.elem += eta / C
+            w += df * eta
+            return true
+
+          }
+        }
+      }
+    }
+    return false
+
+  }
+
+  def currentSlack(i : Instance[_, W], w : Weights[W]) = {
+    i.slack.elem
+  }
+
+}

src/main/scala/epic/logo/LogLikelihoodDecoder.scala

@@ -0,0 +1,11 @@
+package epic.logo
+
+class LogLikelihoodDecoder[T, W](val inferencer : OracleInferencer[T, _, W], val summer : ExpectationInferencer[T, W]) extends Decoder[T, W] {
+
+  def decode(weights : Weights[W], instance : T) : (FeatureVector[W], Double) = {
+    val (y_*, f_*, l_*) = inferencer.oracle(weights, instance)
+    val (f, l) = summer.expectations(weights, instance)
+    ((f_* - f), l - l_*)
+  }
+
+}

src/main/scala/epic/logo/LogoOpts.scala

@@ -0,0 +1,16 @@
+package epic.logo
+
+
+case class LogoOpts(constraintEpsilon: Double = 1e-3,
+
+	miniBatchSize: Int = 1,
+
+	numInnerOptimizationLoops: Int = 10,
+
+	numOuterOptimizationLoops: Int = 10,
+
+	shuffleMinibatches: Boolean = false,
+
+	shuffleSeed: Int = -1,
+
+	convergenceTolerance: Double = 1e-3)

src/main/scala/epic/logo/LossAugmentedArgmaxInferencer.scala

@@ -0,0 +1,9 @@
+package epic.logo
+
+trait LossAugmentedArgmaxInferencer[T, Y, W] extends ArgmaxInferencer[T, Y, W] {
+
+  def argmax(weights : Weights[W], instance : T) : (Y, FeatureVector[W], Double) = lossAugmentedArgmax(weights, instance, 1.0, 0.0)
+
+  def lossAugmentedArgmax(weights : Weights[W], instance : T, weightsWeight : Double, lossWeight : Double) : (Y, FeatureVector[W], Double)
+
+}

src/main/scala/epic/logo/LossAugmentedExpectationInferencer.scala

@@ -0,0 +1,9 @@
+package epic.logo
+
+trait LossAugmentedExpectationInferencer[T, W] extends ExpectationInferencer[T, W] {
+
+  def expectations(weights : Weights[W], instance : T) : (FeatureVector[W], Double) = lossAugmentedExpectations(weights, instance, 1.0, 0.0)
+
+  def lossAugmentedExpectations(weights : Weights[W], instance : T, weightsWeight : Double, lossWeight : Double) : (FeatureVector[W], Double)
+
+}

src/main/scala/epic/logo/LossAugmentedMaxMarginDecoder.scala

@@ -0,0 +1,11 @@
+package epic.logo
+
+class LossAugmentedMaxMarginDecoder[T, W](val oracleInferencer : OracleInferencer[T, _, W], val argmaxer : LossAugmentedArgmaxInferencer[T, _, W]) extends Decoder[T, W] {
+
+  def decode(weights : Weights[W], instance : T) : (FeatureVector[W], Double) = {
+    val (y_*, f_*, l_*) = oracleInferencer.oracle(weights, instance)
+    val (y, f, l) = argmaxer.lossAugmentedArgmax(weights, instance, 1.0, 1.0)
+    (f_* - f, l - l_*)
+  }
+
+}