From 6ffcc74be5d7d7849744b9f3152994a7e3737d47 Mon Sep 17 00:00:00 2001 From: Eric Kimbrel Date: Fri, 8 May 2015 14:41:39 -0700 Subject: [PATCH] Added simplified runner for Louvain. --- .../graphx/louvain/SimpleLouvainRunner.scala | 47 +++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 dga-graphx/src/main/scala/com/soteradefense/dga/graphx/louvain/SimpleLouvainRunner.scala diff --git a/dga-graphx/src/main/scala/com/soteradefense/dga/graphx/louvain/SimpleLouvainRunner.scala b/dga-graphx/src/main/scala/com/soteradefense/dga/graphx/louvain/SimpleLouvainRunner.scala new file mode 100644 index 0000000..eddfb4b --- /dev/null +++ b/dga-graphx/src/main/scala/com/soteradefense/dga/graphx/louvain/SimpleLouvainRunner.scala @@ -0,0 +1,47 @@ +package com.soteradefense.dga.graphx.louvain + +import java.util.Date + + +import com.soteradefense.dga.graphx.io.formats.EdgeInputFormat +import org.apache.spark.graphx.Graph +import org.apache.spark.{SparkConf, SparkContext} + + +object SimpleLouvainRunner { + + + /** + * Main method for parsing arguments and running analytics. + * @param args Commandline arguments. + */ + def main(args: Array[String]) { + println("Welcome to the simple yarn runner") + + if (args.length < 2 || args(0) == "-h" || args(0) == "--help") { + println("arguments: inputPath outputPath [minProgress] [progressCounter]") + sys.exit(1) + } + + val inputPath = args(0) + var outputPath = args(1) + val minProgress = if (args.length > 2) args(2).toInt else 2000 + val progressCounter = if (args.length > 3) args(3).toInt else 1 + + val sparkConf = new SparkConf() + sparkConf.getAll.foreach(println(_)) + val sc = new SparkContext(sparkConf) + + + val inputFormat = new EdgeInputFormat(inputPath, ",") + val edgeRDD = inputFormat.getEdgeRDD(sc) + val initialGraph = Graph.fromEdges(edgeRDD, None) + + + val runner = new HDFSLouvainRunner(minProgress, progressCounter, outputPath) + runner.run(sc, initialGraph) + + + } + +} \ No newline at end of file