Added Measurement; additional refactoring for warmup and reporting.

Author: Dragos Manolescu

README.md

@@ -8,6 +8,18 @@ $ ./sbt
 > run
 ```
 
+There are three command-line arguments:
+
+com.microWorkflow.jsonScalaPerftest.Main [-n num] [-w num] [-map]
+
+-n is the number of iterations to run; the default value is 100, and the result is the mean value
+-w is the number of iterations for warm-up, i.e., executed prior to taking measurements; the default
+value is 5
+-map is optional and determines whether the measurements cover the
+object model mapping in addition to JSON parsing. Without this argument the measurements
+cover only the latter
+
+
 ## Contact ##
 
 - Dragos Manolescu

src/main/scala/com/microWorkflow/jsonScalaPerftest/Category.scala

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2012 Dragos Manolescu.
+ *
+ * Published under the Apache 2.0 license; see http://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.microWorkflow.jsonScalaPerftest
+
+import java.io.File
+import collection.immutable.HashMap
+
+case class Category(directoryName: String, name: String) {
+
+  def this(fn: String) = {
+    this(fn, fn.substring(fn.lastIndexOf('/') + 1))
+  }
+
+  val datasets = Category.getFilesMatching(directoryName, fn => fn.isFile).map {
+    each => new Dataset(each.getCanonicalPath)
+  }
+
+  override def toString = {
+    val sb = new StringBuilder
+    sb.append(name)
+    sb.append(datasets.map(ds => ds.toString).mkString(": ", ", ", ""))
+    sb.toString()
+  }
+
+  def measure(adaptor: LibraryAdaptor, doMap: Boolean, iterations: Int): HashMap[String, Measurement] = {
+    datasets.foldLeft(new HashMap[String, Measurement])((m,d) => {
+      adaptor.resetTimers()
+      adaptor.measure(d, doMap, iterations)
+      m + (d.name -> Measurement.takeMeasurement(adaptor.timerName))
+    })
+  }
+}
+
+object Category {
+  def getFilesMatching(path: String, p: File => Boolean) = {
+    val dir = new java.io.File(path).getAbsolutePath
+    new File(dir).listFiles.filter(file => p(file))
+  }
+}

src/main/scala/com/microWorkflow/jsonScalaPerftest/Dataset.scala

@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2012 Dragos Manolescu.
+ *
+ * Published under the Apache 2.0 license; see http://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.microWorkflow.jsonScalaPerftest
+
+import io.Source
+
+case class Dataset(fileName: String, name: String) {
+
+  def this(fn: String) = {
+    this(fn, fn.substring(fn.lastIndexOf('/') + 1, fn.lastIndexOf('.')))
+  }
+
+  val docs: List[String] = Source.fromFile(fileName).getLines().toList
+
+  override def toString = {
+    val sb = new StringBuilder
+    sb.append(name)
+    sb.append(" (")
+    sb.append(docs.length)
+    sb.append(" documents)")
+    sb.toString()
+  }
+}

src/main/scala/com/microWorkflow/jsonScalaPerftest/LibraryAdaptor.scala

@@ -1,18 +1,18 @@
+/*
+ * Copyright (c) 2012 Dragos Manolescu.
+ *
+ * Published under the Apache 2.0 license; see http://www.apache.org/licenses/LICENSE-2.0.html
+ */
 package com.microWorkflow.jsonScalaPerftest
 
 import com.yammer.metrics.Metrics
 import java.util.concurrent.TimeUnit
 
-/**
- * Created with IntelliJ IDEA.
- * User: dam
- * Date: 12/2/12
- * Time: 7:58 PM
- * To change this template use File | Settings | File Templates.
- */
 abstract class LibraryAdaptor(name: String) extends TimeMeasurements {
-  lazy val initTimer = Metrics.newTimer(getClass, "%s-init".format(name), TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS)
-  lazy val mainTimer = Metrics.newTimer(getClass, "%s-main".format(name), TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS)
+
+  val timerName = "%s-main".format(name)
+
+  lazy val mainTimer = Metrics.newTimer(getClass, timerName, TimeUnit.MILLISECONDS, TimeUnit.SECONDS)
 
   def getName = name
 
@@ -22,18 +22,15 @@ abstract class LibraryAdaptor(name: String) extends TimeMeasurements {
 
   def runOnce(json: String, doMap:Boolean): Any
 
-  def measure(dataset: Dataset, doMap:Boolean) = {
-    val context1 = initTimer.time()
-    try {
-      initialize()
-    } finally {
-      context1.stop()
-    }
+  def measure(dataset: Dataset, doMap:Boolean, iterations: Int) {
+    initialize()
 
     val context2 = mainTimer.time()
     try {
-      for (doc <- dataset.docs)
-        runOnce(doc, doMap)
+      for (count <- 1 to iterations) {
+        for (doc <- dataset.docs)
+          runOnce(doc, doMap)
+      }
     } catch {
       case ex:Exception => println("%s bombed while processing %s (%s)".format(name, dataset.name, ex.getMessage))
     } finally {
@@ -42,7 +39,6 @@ abstract class LibraryAdaptor(name: String) extends TimeMeasurements {
   }
 
   def resetTimers() {
-    initTimer.clear()
     mainTimer.clear()
   }
 }

src/main/scala/com/microWorkflow/jsonScalaPerftest/Main.scala

@@ -1,63 +1,13 @@
+/*
+ * Copyright (c) 2012 Dragos Manolescu.
+ *
+ * Published under the Apache 2.0 license; see http://www.apache.org/licenses/LICENSE-2.0.html
+ */
 package com.microWorkflow.jsonScalaPerftest
 
-import java.io.File
-import io.Source
-import com.yammer.metrics.reporting.CsvReporter
-import java.util.concurrent.TimeUnit
-import fr.janalyse.jmx.JMX
-import com.yammer.metrics.Metrics
+import collection.immutable.HashMap
 
-case class Dataset(fileName: String, name: String) {
-
-  def this(fn: String) = {
-    this(fn, fn.substring(fn.lastIndexOf('/') + 1, fn.lastIndexOf('.')))
-  }
-
-  val docs: List[String] = Source.fromFile(fileName).getLines().toList
-
-  override def toString = {
-    val sb = new StringBuilder
-    sb.append(name)
-    sb.append(" (")
-    sb.append(docs.length)
-    sb.append(" documents)")
-    sb.toString()
-  }
-}
-
-case class Category(directoryName: String, name: String) {
-
-  def this(fn: String) = {
-    this(fn, fn.substring(fn.lastIndexOf('/') + 1))
-  }
-
-  val datasets = Category.getFilesMatching(directoryName, fn => fn.isFile).map {
-    each => new Dataset(each.getCanonicalPath)
-  }
-
-  override def toString = {
-    val sb = new StringBuilder
-    sb.append(name)
-    sb.append(datasets.map(ds => ds.toString).mkString(": ", ", ", ""))
-    sb.toString()
-  }
-
-  def measure(adapter: LibraryAdaptor, doMap: Boolean) {
-    for (dataset <- datasets) {
-      adapter.measure(dataset, doMap)
-    }
-  }
-}
-
-object Category {
-  def getFilesMatching(path: String, p: File => Boolean) = {
-    val dir = new java.io.File(path).getAbsolutePath
-    new File(dir).listFiles.filter(file => p(file))
-  }
-}
-
-
-case class Experiment(measurementsPath: String) {
+case class Experiment(warmUpIterations: Int=5) {
 
   val adapters = Array(new liftjson.LiftJsonAdaptor("lift")
     , new jerkson.JerksonAdaptor("jerkson")
@@ -70,62 +20,65 @@ case class Experiment(measurementsPath: String) {
     , new jackson.JacksonAdaptor("jackson")
   )
 
-  val loopCounter = Metrics.newCounter(this.getClass, "main loop")
-
-  val categories = Category.getFilesMatching("data", f => f.isDirectory).map(d => new Category(d.getCanonicalPath))
+  val categories = Category
+    .getFilesMatching("data", f => f.isDirectory)
+    .map(d => new Category(d.getCanonicalPath))
 
-  def run(iterations: Int, doMap: Boolean) = {
-    loopCounter.clear()
-    val oldFiles = Category.getFilesMatching(measurementsPath, f => f.isFile)
-    println("Removing %d old files".format(oldFiles.length))
-    if (oldFiles != null)
-      oldFiles.foreach(f => f.delete())
-    CsvReporter.enable(new File(measurementsPath), 100, TimeUnit.MILLISECONDS)
+  def run(iterations: Int, doMap: Boolean): Array[(String, HashMap[String, Measurement])] = {
     val adaptersToTest = if (doMap) adapters.filter(_.hasMap) else adapters
-    for (count <- 1 to iterations) {
-      categories.flatMap(c => (adaptersToTest.map {
-        each => c.measure(each, doMap)
-      }))
-      loopCounter.inc()
-    }
+     categories.flatMap(c => (adaptersToTest.map {
+        each => (c.name -> c.measure(each, doMap, iterations))
+     }))
   }
 
-  def takeMeasurements(iterations: Int, doMap: Boolean) {
-    val where = new File(measurementsPath)
-    where.mkdirs()
-    print("Priming caches...")
-    run(iterations, doMap)
-    print("Measuring timings...")
+  def takeMeasurements(iterations: Int, doMap: Boolean): Array[(String, HashMap[String, Measurement])] = {
+    println("Warming up (%d iterations)...".format(warmUpIterations))
+    run(warmUpIterations, doMap)
+    println("Measuring (%d iterations)...".format(iterations))
     run(iterations, doMap)
   }
-
-  def printMeanValues() {
-    JMX.once() {
-      jmx => {
-        val beans = jmx.mbeans().filter(b => b.name.contains("microWorkflow".toCharArray))
-        for (bean <- beans) {
-          val mean = bean.getDouble("Mean") match {
-            case Some(d) => d
-            case None => 0.0
-          }
-          println("%s: %fms".format(bean.name.substring(bean.name.indexOf('=') + 1), mean))
-        }
-      }
-    }
-  }
 }
 
 
 object Main {
+  val usage =
+    """
+      |Usage: com.microWorkflow.jsonScalaPerftest.Main [-n num] [-w num] [-map]
+    """.stripMargin
 
   def main(args: Array[String]) {
-    val where = if (args.length>0) args(0) else "/tmp/measurements"
-    val iterations:Int = if (args.length>1) args(1).toInt else 100
-    val doMap = if (args.length>2) args(2).equalsIgnoreCase("-map") else false
-    println("Runing %d iterations %s object mapping; test results in %s"
-      .format(iterations, if (doMap) "with" else "without", where))
-    val e = Experiment(where)
-    e.takeMeasurements(iterations, doMap)
-    println("Done")
+    if (args.isEmpty) println(usage)
+    else {
+
+      def nextOption(map: Map[Symbol, Any], list: List[String]): Map[Symbol, Any] = {
+        list match {
+          case Nil => map
+          case "-n" :: value :: tail => nextOption(map + ('iterations -> value.toInt), tail)
+          case "-w" :: value :: tail => nextOption(map + ('warmUp -> value.toInt), tail)
+          case "-map" :: tail => nextOption(map + ('map -> true), tail)
+          case _ :: tail =>
+            println("Don't know what to do with '%s'%s".format(list.head, usage))
+            sys.exit(1)
+        }
+      }
+      val options = nextOption(new HashMap[Symbol, Any](), args.toList)
+
+      val iterations = options.getOrElse('iterations, 100).asInstanceOf[Int]
+      val doMap = options.getOrElse('map, false).asInstanceOf[Boolean]
+      val warmUpIterations = options.getOrElse('warmUp, 5).asInstanceOf[Int]
+
+      println("Runing %d iterations %s object mapping"
+        .format(iterations, if (doMap) "with" else "without"))
+      val experiment = Experiment(warmUpIterations)
+      val ms = experiment.takeMeasurements(iterations, doMap)
+      for (m <- ms) {
+        print("Category: %s\n".format(m._1))
+        for (d <- m._2.iterator)
+        print("\t dataset '%s', measurement: %s\n".format(d._1, d._2))
+      }
+
+      sys.exit()
+    }
+
   }
 }

src/main/scala/com/microWorkflow/jsonScalaPerftest/Measurement.scala

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2012 Dragos Manolescu.
+ *
+ * Published under the Apache 2.0 license; see http://www.apache.org/licenses/LICENSE-2.0.html
+ */
+package com.microWorkflow.jsonScalaPerftest
+
+import fr.janalyse.jmx.{RichMBean, JMX}
+
+case class Measurement(name: String, value: Double, u: String="") {
+
+  val unit = if (u.isEmpty) None else Some(u)
+
+  override def toString = {
+    val sb = new StringBuilder
+    sb.append(name)
+    sb.append(" = ")
+    sb.append(value)
+    sb.append(" ")
+    sb.append(unit match {
+      case Some(s) => s
+      case None    => "(adimensional)"
+    })
+    sb.toString()
+  }
+}
+
+object Measurement {
+
+  def takeMeasurement(name: String) = {
+    Measurement(name, getValuesFor(name, "Mean"), "ms")
+  }
+
+  def getDoubleFrom(bean: RichMBean, key: String) = {
+    bean.getDouble(key) match {
+      case Some(d) => d
+      case None => 0.0
+    }
+  }
+
+  def getValuesFor(nameKey: String, valueKey: String): Double = {
+    JMX.once() {
+      jmx => {
+        jmx.mbeans()
+          .filter(b => b.name.contains(nameKey.toCharArray))
+          .map(bean => getDoubleFrom(bean, valueKey))
+          .head
+      }
+    }
+  }
+}
+