benchmark.html

---
layout: default
title: Benchmarks
disqus: yes
sitemap:
  lastmod: 2014-03-11
  priority: 0.9
  changefreq: monthly
---

<h1 id="benchmark" class="page-header">{{ page.title }}</h1>

<p class="lead">
  This page summarizes how much time is required to solve different
  optimization problems. Various solvers are used to compare the
  efficiency of the different available approaches.
</p>


<div class="bs-callout bs-callout-info">
  <h4>Methodology</h4>
  <ol>
    <li>Run the RobOptim test suite for each solver, the
      Hock-Schittkowski Collection problems are included, see
      the <a href="https://github.com/roboptim/roboptim-shared-tests">roboptim-shared-tests</a>
      repository to see the actual implementation. The "b" versions
      correspond to minor variations of a same problem but using the
      RobOptim in a different manner.
    </li>

    <li>Convert the RobOptim log files, i.e. <code>journal.log</code>, to a
      YAML file using
      <a href="https://github.com/roboptim/roboptim-analysis/blob/master/bin/log_to_yaml">log_to_yaml.py</a>.</li>
    <li>Tests for which solvers are failing are finally removed from
      the dataset (manually for now).</li>
  </ol>

  <p>The log file timing information are generated
    using <a href="http://www.boost.org/doc/libs/1_55_0/doc/html/date_time/posix_time.html">Boost.DateTime</a>.
  </p>

  <p>The tests provided here are too small to test sparse solvers in
    a robust way. Values for these solvers should be handled with
    great care!</p>

  <p><a href="http://www.ai7.uni-bayreuth.de/test_problem_coll.pdf">The
      document defining all the problems is available here.</a>
  </p>

</div>

<div id="benchmark_schittkowski_dl"></div>
<div id="benchmark_schittkowski" style="width: 100%; height: 300em"></div>

<!-- Load highcharts API -->
<script src="http://code.highcharts.com/highcharts.js"></script>
<script src="http://code.highcharts.com/modules/exporting.js"></script>

<!-- Load underscore API -->
<script src="underscore-min.js"></script>

<script type="text/javascript">
  var benchmark = {};
  {% for test in site.data.benchmark %}
      if (!benchmark["{{test.testsuite}}"]) {benchmark["{{test.testsuite}}"] = {};}
      if (!benchmark["{{test.testsuite}}"]["{{test.problem}}"]) {benchmark["{{test.testsuite}}"]["{{test.problem}}"] = {};}
      benchmark["{{test.testsuite}}"]["{{test.problem}}"]["{{test.solver}}"] = "{{test.time}}";
  {% endfor %}

  var cfsqp_fail = ["15", "16", "20", "25", "28", "33", "40", "47", "48", "49",
                    "50", "51", "52", "53", "55", "56", "61", "63", "66", "72",
                    "76", "78", "79", "80", "81"]
  for (var i = 0, tot = cfsqp_fail.length; i < tot; i++) {
      benchmark["schittkowski"]["problem-".concat(cfsqp_fail[i])]["cfsqp"] = null;
  }

  var ipopt_fail = ["13", "16", "20", "25", "28", "44", "55", "56"]
  for (var i = 0, tot = ipopt_fail.length; i < tot; i++) {
      benchmark["schittkowski"]["problem-".concat(ipopt_fail[i])]["ipopt"] = null;
  }

  var ipopt_sparse_fail = ["13", "16", "20", "25", "28", "44", "55", "56"]
  for (var i = 0, tot = ipopt_sparse_fail.length; i < tot; i++) {
      benchmark["schittkowski"]["problem-".concat(ipopt_sparse_fail[i])]["ipopt-sparse"] = null;
  }

  var nag_nlp_fail = ["2", "8", "13", "14", "16", "20", "23",
                      "25", "28", "33", "47", "55", "56", "72"]
  for (var i = 0, tot = nag_nlp_fail.length; i < tot; i++) {
      benchmark["schittkowski"]["problem-".concat(nag_nlp_fail[i])]["nag-nlp"] = null;
  }

  var nag_nlp_sparse_fail = ["2", "8", "13", "14", "16", "20", "23", "24",
                             "25", "28", "33", "44", "47", "55", "56", "72"]
  for (var i = 0, tot = nag_nlp_sparse_fail.length; i < tot; i++) {
      benchmark["schittkowski"]["problem-".concat(nag_nlp_sparse_fail[i])]["nag-nlp-sparse"] = null;
  }

  var nlopt_fail = ["2", "8", "9", "12", "16", "19", "20", "25", "26", "28",
                    "33", "34", "36", "43", "44", "47", "49", "51", "55",
                    "56", "65", "81"]
  for (var i = 0, tot = nlopt_fail.length; i < tot; i++) {
      benchmark["schittkowski"]["problem-".concat(nlopt_fail[i])]["nlopt"] = null;
  }

  var pagmo_fail = ["6", "7", "10", "13", "15", "24", "26", "27", "31", "39",
                    "40", "47", "56", "60", "61", "63", "71", "78", "79",
                    "80", "81", "71b"]
  for (var i = 0, tot = pagmo_fail.length; i < tot; i++) {
      benchmark["schittkowski"]["problem-".concat(pagmo_fail[i])]["pagmo"] = null;
  }

  var solverPrettyNames = {
      "cfsqp":          "CFSQP",
      "ipopt":          "IPOPT",
      "ipopt-sparse":   "IPOPT (sparse)",
      "nag-nlp":        "NAG",
      "nag-nlp-sparse": "NAG (sparse)",
      "nlopt":          "NLopt (alpha)",
      "pagmo":          "PaGMO (alpha)"
  };

  var bench_colors = [
    'blue',
    'red',
    'orange',
    'green',
    'purple',
    'dodgerblue',
    'magenta'
  ]

  // Add data labels to bars
  // Note: this kinda botches display, so only use it locally
  var use_data_labels = false;

  // Put the legend at the top of the graph
  // Note: this is probably better for display on a website
  var use_top_legend = true;

  var legend = {};

  if (use_top_legend)
    {
      legend["align"] = 'right';
      legend["layout"] = 'vertical';
      legend["verticalAlign"] = 'top';
      legend["x"] = -40;
      legend["y"] = 100;
      legend["floating"] = true;
      legend["borderWidth"] = 1;
      legend["backgroundColor"] = '#FFFFFF';
      legend["shadow"] = true;
      legend["reversed"] = false;
  }

  function problemNameToId(pbName)
  {
    var pbId = pbName.replace('problem-', '');
    pbId = ('000000' + pbId);
    pbId = pbId.substr(pbId.length - 3);
    return pbId;
  }

  function problemIdToName(pbId)
  {
    return 'problem-'.concat(pbId.replace(/^0+/, ''));
  }

  result = {}

  // Solver list
  var problems = benchmark["schittkowski"];
  result["solvers"] = _.chain(problems)
                       .map(function(el) {
                         return _.keys(el);
                       })
                       .flatten()
                       .uniq()
                       .value();

  // Problem list
  var pb_names = _.keys(problems);
  result["problems"] = _.chain(pb_names)
                        .map(problemNameToId)
                        .sortBy(function(el){ return el; })
                        .value();

  // Benchmark data: name (solver), data (for each problem), color, index
  result["series"] = _.chain(result["solvers"])
                      .map(function(el) {
                        var solver_data = {};
                        // Solver name
                        solver_data["name"] = (solverPrettyNames[el] ? solverPrettyNames[el] : el);
                        // Index
                        var index = _.indexOf(_.keys(solverPrettyNames), el);
                        solver_data["index"] = _.size(_.keys(solverPrettyNames)) - index;
                        solver_data["legendIndex"] = index;
                        // Color
                        solver_data["color"] = bench_colors[index];
                        // Solver data for each problem
                        solver_data["data"] = _.map(result["problems"],
                                                    function(pbId) {
                                                      try {
                                                          var value = {
                                                            dataLabels: {
                                                              enabled: use_data_labels,
                                                              format: "<b>" + solver_data["name"] + "</b>",
                                                              useHtml: true,
                                                            },
                                                            y: parseFloat(problems[problemIdToName(pbId)][el])
                                                          };
                                                        return value;
                                                      }
                                                      catch(err) {
                                                        console.error("problem " + pbId + " undefined for solver " + el);
                                                        return null;
                                                      }
                                                      return null;
                                                    });
                        return solver_data;
                      })
                      .value();

  console.log(result)

  var size = 10;
  var height = _.size(result["problems"]) * _.size(result["solvers"]) * 2 * size

  $(function() {
    $('#benchmark_schittkowski').highcharts({
      chart: {
          type: 'bar',
          height: height
      },
      title: {
        text: 'Hock-Schittkowski Collection Test Suite'
      },
      subtitle: {
        text: 'RobOptim Benchmark'
      },
      plotOptions: {
          bar: {
            pointPadding: 100
          },
          series: {
            pointPadding: 0.1,
            groupPadding: 0.1,
          }
      },
      xAxis: {
        categories: result["problems"],
        title: {
          text: "Problem Id"
        },
        reversed: true,
      },
      yAxis: {
        min: 0,
        title: {
          text: "Time (seconds)"
        }
      },
      legend: legend,
      exporting: {
        sourceHeight: height,
        filename: "hock_schittkowski_benchmark"
      },
      series: result["series"]
    });
  });

</script>

<!-- vim: set ts=2 sw=2 ft=liquid si: -->