From dc8d48bf528da1e6fd588146e8002edec81c4c20 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 11 Jun 2016 03:41:40 +0000
Subject: [PATCH 01/49] start implementation of log parsing

---
 processors/processExpand.js |  2 +-
 processors/processReduce.js | 22 ++++++++++++-------
 sql/create_tables.sql       | 44 ++++++++++++++++++++++++++++++++++++-
 svc/parser.js               |  3 ++-
 4 files changed, 60 insertions(+), 11 deletions(-)

diff --git a/processors/processExpand.js b/processors/processExpand.js
index c91c15ca3..e67b97647 100644
--- a/processors/processExpand.js
+++ b/processors/processExpand.js
@@ -449,7 +449,7 @@ function processExpand(entries, meta)
     //null means all types
     var reqs = {
         parsed_data: null,
-        //expanded: null,
+        expanded: null,
         tf_data:
         {
             killed: 1,
diff --git a/processors/processReduce.js b/processors/processReduce.js
index 41493a8fa..f35890898 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -8,15 +8,21 @@ function processReduce(entries)
     //group by player_slot, type, targethero, targetillusion
     for (var i = 0; i < entries.length; i++)
     {
-        //group big categories: actions, combat log damage
         var e = entries[i];
-        var identifier = [e.player_slot, e.type, e.key].join(":");
-        e.value = e.value || 1;
-        //var identifier = e.type;
-        //e.value = 1;
-        reduceMap[identifier] = reduceMap[identifier] ? reduceMap[identifier] + e.value : e.value || 1;
+        reduceMap[e.type] = reduceMap[e.type] ? reduceMap[e.type] + 1 : 1;
     }
-    //var fs = require('fs');
-    //fs.writeFileSync('./output3.json', JSON.stringify(reduceMap, null , 2));
+    console.log(reduceMap);
+    return entries.filter(function(e)
+    {
+        if (e.type === "actions")
+        {
+            return false;
+        }
+        if (e.type === "interval" && e.time % 60 !== 0)
+        {
+            return false;
+        }
+        return true;
+    });
 }
 module.exports = processReduce;
\ No newline at end of file
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 80f3fec9d..72e5d6370 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -172,4 +172,46 @@ CREATE TABLE notable_players (
   is_locked boolean,
   is_pro boolean,
   locked_until integer
-);
\ No newline at end of file
+);
+
+CREATE TABLE match_logs (
+  match_id bigint,
+  time int,
+  type varchar(100),
+  team int,
+  unit varchar(100),
+  key varchar(100),
+  value int,
+  slot int,
+  player_slot int,
+  player1 int,
+  player2 int,
+  attackername string,
+  targetname string,
+  sourcename string,
+  targetsourcename string,
+  attackerhero boolean,
+  targethero boolean,
+  attackerillusion boolean,
+  targetillusion boolean,
+  inflictor varchar(100),
+  gold_reason int,
+  xp_reason int,
+  valuename varchar(100),
+  stun_duration real,
+  slow_duration real,
+  gold int,
+  lh int,
+  xp int,
+  x int,
+  y int,
+  stuns real,
+  hero_id int,
+  life_state int,
+  level int,
+  kills int,
+  deaths int,
+  assists int,
+  denies int
+);
+CREATE INDEX ON match_logs(match_id);
\ No newline at end of file
diff --git a/svc/parser.js b/svc/parser.js
index 7843e0609..8384ab224 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -263,6 +263,8 @@ function runParse(match, job, cb)
                 var message = "time spent on post-processing match ";
                 console.time(message);
                 var meta = processMetadata(entries);
+                var logs = processReduce(entries);
+                console.log('logs: %s', logs.length);
                 var res = processExpand(entries, meta);
                 var parsed_data = processParsedData(res.parsed_data);
                 var teamfights = processTeamfights(res.tf_data, meta);
@@ -273,7 +275,6 @@ function runParse(match, job, cb)
                 parsed_data.radiant_xp_adv = ap.radiant_xp_adv;
                 parsed_data.upload = upload;
                 //processMultiKillStreaks();
-                //processReduce(res.expanded);
                 console.timeEnd(message);
                 return cb(err, parsed_data);
             }

From 8774cd487b583d34758ac2a9a7129f8d949731b2 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 11 Jun 2016 08:33:22 +0000
Subject: [PATCH 02/49] update schema

---
 processors/processReduce.js | 21 +++++++++++--------
 sql/create_tables.sql       |  8 ++++----
 store/queries.js            | 41 +++++++++++++++++++++++++++++++++++++
 svc/parser.js               |  4 ++--
 4 files changed, 59 insertions(+), 15 deletions(-)

diff --git a/processors/processReduce.js b/processors/processReduce.js
index f35890898..1b11d4bda 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -2,16 +2,8 @@
  * A processor to reduce the event stream by grouping similar events.
  * NOT CURRENTLY IN PRODUCTION USE
  **/
-function processReduce(entries)
+function processReduce(entries, match)
 {
-    var reduceMap = {};
-    //group by player_slot, type, targethero, targetillusion
-    for (var i = 0; i < entries.length; i++)
-    {
-        var e = entries[i];
-        reduceMap[e.type] = reduceMap[e.type] ? reduceMap[e.type] + 1 : 1;
-    }
-    console.log(reduceMap);
     return entries.filter(function(e)
     {
         if (e.type === "actions")
@@ -22,7 +14,18 @@ function processReduce(entries)
         {
             return false;
         }
+        if (!e.time)
+        {
+            return false;
+        }
         return true;
+    }).map(function(e)
+    {
+        return Object.assign(
+        {}, e,
+        {
+            match_id: match.match_id
+        });
     });
 }
 module.exports = processReduce;
\ No newline at end of file
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 47e7b4e3d..7a7b418b3 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -187,10 +187,10 @@ CREATE TABLE match_logs (
   player_slot int,
   player1 int,
   player2 int,
-  attackername string,
-  targetname string,
-  sourcename string,
-  targetsourcename string,
+  attackername varchar(100),
+  targetname varchar(100),
+  sourcename varchar(100),
+  targetsourcename varchar(100),
   attackerhero boolean,
   targethero boolean,
   attackerillusion boolean,
diff --git a/store/queries.js b/store/queries.js
index c4952d6ed..bd6e3446c 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -193,6 +193,7 @@ function insertMatch(db, redis, match, options, cb)
     {
         "u": upsertMatch,
         "uc": upsertMatchCassandra,
+        "uml": upsertMatchLogs,
         "upc": updatePlayerCaches,
         "uct": updateCounts,
         "cmc": clearMatchCache,
@@ -309,6 +310,45 @@ function insertMatch(db, redis, match, options, cb)
         });
     }
 
+    function upsertMatchLogs(cb)
+    {
+        if (!match.logs)
+        {
+            return cb();
+        }
+        else
+        {
+            db.transaction(function(trx)
+            {
+                trx.raw(`DELETE FROM match_logs where match_id = ?`, [match.match_id]).asCallback(function(err)
+                {
+                    if (err)
+                    {
+                        return exit(err);
+                    }
+                    async.eachLimit(match.logs, 10000, function(e, cb)
+                    {
+                        trx('match_logs').insert(e).asCallback(cb);
+                    }, exit);
+                });
+
+                function exit(err)
+                {
+                    if (err)
+                    {
+                        console.error(err);
+                        trx.rollback(err);
+                    }
+                    else
+                    {
+                        trx.commit();
+                    }
+                    cb(err);
+                }
+            });
+        }
+    }
+
     function updatePlayerCaches(cb)
     {
         var copy = createMatchCopy(match, players, options);
@@ -400,6 +440,7 @@ function insertMatch(db, redis, match, options, cb)
                 duration: match.duration,
                 replay_blob_key: match.replay_blob_key,
                 pgroup: match.pgroup,
+                leagueid: match.leagueid,
             },
             {
                 lifo: options.lifo,
diff --git a/svc/parser.js b/svc/parser.js
index 8384ab224..3355508fd 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -263,8 +263,7 @@ function runParse(match, job, cb)
                 var message = "time spent on post-processing match ";
                 console.time(message);
                 var meta = processMetadata(entries);
-                var logs = processReduce(entries);
-                console.log('logs: %s', logs.length);
+                var logs = match.leagueid ? processReduce(entries, match) : undefined;
                 var res = processExpand(entries, meta);
                 var parsed_data = processParsedData(res.parsed_data);
                 var teamfights = processTeamfights(res.tf_data, meta);
@@ -274,6 +273,7 @@ function runParse(match, job, cb)
                 parsed_data.radiant_gold_adv = ap.radiant_gold_adv;
                 parsed_data.radiant_xp_adv = ap.radiant_xp_adv;
                 parsed_data.upload = upload;
+                parsed_data.logs = logs;
                 //processMultiKillStreaks();
                 console.timeEnd(message);
                 return cb(err, parsed_data);

From 1078a11e7eef2d39d3c7925a537adc902545b5f5 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Mon, 13 Jun 2016 00:30:23 +0000
Subject: [PATCH 03/49] node6

---
 Dockerfile        | 19 ++++++++++---------
 config.js         |  1 +
 deploy.js         |  6 +++---
 package.json      |  8 ++------
 scripts/deploy.sh |  6 +++---
 scripts/gce.sh    |  6 +++---
 6 files changed, 22 insertions(+), 24 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2f617adbf..ba5cacf65 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,21 +1,21 @@
-FROM mhart/alpine-node:5.10.1
+FROM mhart/alpine-node:6.2.1
 
 # Tools
 RUN apk update && apk add git curl wget bash
 
 # Java
-# RUN apk update && apk add openjdk8=8.77.03-r0
+#RUN apk update && apk add openjdk8=8.92.14-r1
 # Here we install GNU libc (aka glibc) and set C.UTF-8 locale as default.
 
-RUN ALPINE_GLIBC_BASE_URL="https://github.com/andyshinn/alpine-pkg-glibc/releases/download" && \
-    ALPINE_GLIBC_PACKAGE_VERSION="2.23-r1" && \
+RUN ALPINE_GLIBC_BASE_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" && \
+    ALPINE_GLIBC_PACKAGE_VERSION="2.23-r2" && \
     ALPINE_GLIBC_BASE_PACKAGE_FILENAME="glibc-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
     ALPINE_GLIBC_BIN_PACKAGE_FILENAME="glibc-bin-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
     ALPINE_GLIBC_I18N_PACKAGE_FILENAME="glibc-i18n-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
     apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
     wget \
-        "https://raw.githubusercontent.com/andyshinn/alpine-pkg-glibc/master/andyshinn.rsa.pub" \
-        -O "/etc/apk/keys/andyshinn.rsa.pub" && \
+        "https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub" \
+        -O "/etc/apk/keys/sgerrand.rsa.pub" && \
     wget \
         "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
         "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
@@ -25,7 +25,7 @@ RUN ALPINE_GLIBC_BASE_URL="https://github.com/andyshinn/alpine-pkg-glibc/release
         "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
         "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \
     \
-    rm "/etc/apk/keys/andyshinn.rsa.pub" && \
+    rm "/etc/apk/keys/sgerrand.rsa.pub" && \
     /usr/glibc-compat/bin/localedef --force --inputfile POSIX --charmap UTF-8 C.UTF-8 || true && \
     echo "export LANG=C.UTF-8" > /etc/profile.d/locale.sh && \
     \
@@ -91,7 +91,8 @@ RUN apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
     rm "/tmp/"*
 
 # Maven
-# RUN apk update && apk add maven=3.3.9-r0
+# as of 2016-06-12 maven apk is still in edge branch and not accessible by default
+#RUN apk update && apk add maven=3.3.9-r0
 ENV MAVEN_HOME="/usr/share/maven"
 ENV MAVEN_VERSION="3.3.9"
 RUN cd / && \
@@ -115,4 +116,4 @@ RUN npm run webpack
 
 ENV PATH /usr/src/yasp/node_modules/pm2/bin:$PATH
 
-CMD [ "node", "deploy.js" ]
\ No newline at end of file
+CMD [ "npm", "start" ]
\ No newline at end of file
diff --git a/config.js b/config.js
index 658b0ab0a..444367392 100644
--- a/config.js
+++ b/config.js
@@ -39,6 +39,7 @@ var defaults = {
     "PROXY_URLS": "", //comma separated list of proxy urls to use
     "STEAM_API_HOST": "api.steampowered.com", //the list of hosts to fetch Steam API data from
     "ROLE": "", //for specifying the file that should be run when deploy.js is invoked
+    "GROUP": "", //for specifying the group of apps that should be run when deploy.js is invoked
     "MMSTATS_DATA_INTERVAL": 3, //minutes between requests for MMStats data
     "DEFAULT_DELAY": 1000, // delay between API requests (default: 1000)
     "SCANNER_DELAY": 300, //delay for scanner API requests (more time-sensitive)
diff --git a/deploy.js b/deploy.js
index 47f0df51a..37e4de4f6 100644
--- a/deploy.js
+++ b/deploy.js
@@ -2,6 +2,7 @@
  * Deployment entry point for the application.
  **/
 var args = process.argv.slice(2);
+var group = args[0] || process.env.GROUP;
 var cp = require('child_process');
 if (process.env.PROVIDER === "gce")
 {
@@ -12,17 +13,16 @@ if (process.env.ROLE)
     //if role variable is set just run that script
     require('./svc/' + process.env.ROLE + ".js");
 }
-else if (args[0])
+else if (group)
 {
     var pm2 = require('pm2');
     var async = require('async');
     var manifest = require('./profiles/everything.json').apps;
-    //if argument supplied use pm2 to run processes in that group
     pm2.connect(function()
     {
         async.each(manifest, function start(app, cb)
         {
-            if (args[0] === app.group)
+            if (group === app.group)
             {
                 console.log(app.script, app.instances);
                 pm2.start(app.script,
diff --git a/package.json b/package.json
index 6165eadd7..2e7b97634 100644
--- a/package.json
+++ b/package.json
@@ -17,7 +17,8 @@
     "kube": "kubectl get rc -o name --selector tier=backend | cut -d '/' -f2 | xargs -n1 kubectl rolling-update --image=yasp/yasp:latest",
     "rediskeys": "redis-cli keys '*' | cut -d':' -f1 | sort | uniq -c",
     "resetpicks": "redis-cli keys 'picks_*' | xargs redis-cli del",
-    "updateconstants": "node tasks/updateconstants"
+    "updateconstants": "node tasks/updateconstants",
+    "dockerbuild": "sudo docker build -t yasp/yasp ."
   },
   "repository": {
     "type": "git",
@@ -56,7 +57,6 @@
     "jade": "^1.11.0",
     "jquery": "^2.2.0",
     "knex": "^0.10.0",
-    "material-ui": "^0.15.0-alpha.2",
     "moment": "^2.11.2",
     "multer": "^1.1.0",
     "ndjson": "^1.4.3",
@@ -93,9 +93,5 @@
     "supertest": "^1.1.0",
     "url-loader": "^0.5.7",
     "webpack": "^1.12.12"
-  },
-  "engines": {
-    "node": "6.0.0",
-    "npm": "3.8.8"
   }
 }
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index 5ef923192..e8204b539 100644
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -4,7 +4,7 @@ DATETIME=$(date +%s)
 
 if [ "$1" = "parser" ] || [[ $# -eq 0 ]]; then
 gcloud compute instance-templates create parser-$DATETIME --machine-type n1-highcpu-2 --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --metadata startup-script='#!/bin/bash
-sudo docker run -d --name=parser --restart=always -e PROVIDER=gce -e ROLE=parser yasp/yasp:latest sh -c "node deploy.js"
+sudo docker run -d --name=parser --restart=always -e PROVIDER=gce -e ROLE=parser yasp/yasp:latest sh -c "npm start"
 sudo docker start parser
 '
 gcloud alpha compute rolling-updates start --group parser-group-1 --template parser-$DATETIME
@@ -12,7 +12,7 @@ fi
 
 if [ "$1" = "backend" ] || [[ $# -eq 0 ]]; then
 gcloud compute instance-templates create backend-$DATETIME --machine-type n1-highcpu-4 --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --tags "http-server" --metadata startup-script='#!/bin/bash
-sudo docker run -d --name yasp --restart=always --net=host -e PROVIDER=gce yasp/yasp:latest sh -c "node deploy.js core"
+sudo docker run -d --name yasp --restart=always --net=host -e PROVIDER=gce -e GROUP=core yasp/yasp:latest sh -c "npm start"
 sudo docker start yasp
 '
 gcloud alpha compute rolling-updates start --group backend-group-1 --template backend-$DATETIME
@@ -20,7 +20,7 @@ fi
 
 if [ "$1" = "web" ] || [[ $# -eq 0 ]]; then
 gcloud compute instance-templates create web-$DATETIME --machine-type g1-small --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --tags "http-server" --metadata startup-script='#!/bin/bash
-sudo docker run -d --name=web --restart=always --net=host -e FRONTEND_PORT=80 -e PROVIDER=gce -e ROLE=web yasp/yasp:latest sh -c "node deploy.js"
+sudo docker run -d --name=web --restart=always --net=host -e FRONTEND_PORT=80 -e PROVIDER=gce -e ROLE=web yasp/yasp:latest sh -c "npm start"
 sudo docker start web
 '
 gcloud alpha compute rolling-updates start --group web-group-1 --template web-$DATETIME --min-instance-update-time 180
diff --git a/scripts/gce.sh b/scripts/gce.sh
index 9e5e36d43..4c16a1d65 100644
--- a/scripts/gce.sh
+++ b/scripts/gce.sh
@@ -26,7 +26,7 @@ gcloud compute http-health-checks delete -q lb-check
 gcloud compute instance-groups managed delete -q web-group-1
 gcloud compute instance-templates delete -q web-1
 gcloud compute instance-templates create web-1 --machine-type g1-small --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --tags "http-server" --metadata startup-script='#!/bin/bash
-sudo docker run -d --name=web --restart=always --net=host -e FRONTEND_PORT=80 -e PROVIDER=gce -e ROLE=web yasp/yasp:latest sh -c "node deploy.js"
+sudo docker run -d --name=web --restart=always --net=host -e FRONTEND_PORT=80 -e PROVIDER=gce -e ROLE=web yasp/yasp:latest sh -c "npm start"
 sudo docker start web
 '
 gcloud compute instance-groups managed create "web-group-1" --base-instance-name "web-group-1" --template "web-1" --size "0"
@@ -40,7 +40,7 @@ gcloud compute instance-groups managed set-autoscaling "web-group-1" --cool-down
 gcloud compute instance-groups managed delete -q backend-group-1
 gcloud compute instance-templates delete -q backend-1
 gcloud compute instance-templates create backend-1 --machine-type n1-highcpu-4 --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --tags "http-server" --metadata startup-script='#!/bin/bash
-sudo docker run -d --name yasp --restart=always --net=host -e PROVIDER=gce yasp/yasp:latest sh -c "node deploy.js core"
+sudo docker run -d --name yasp --restart=always --net=host -e PROVIDER=gce -e GROUP=core yasp/yasp:latest sh -c "npm start"
 sudo docker start yasp
 '
 gcloud compute instance-groups managed create "backend-group-1" --base-instance-name "backend-group-1" --template "backend-1" --size "1"
@@ -49,7 +49,7 @@ gcloud compute instance-groups managed create "backend-group-1" --base-instance-
 gcloud compute instance-groups managed delete -q parser-group-1
 gcloud compute instance-templates delete -q parser-1
 gcloud compute instance-templates create parser-1 --machine-type n1-highcpu-2 --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --metadata startup-script='#!/bin/bash
-    sudo docker run -d --name=parser --restart=always -e PROVIDER=gce -e ROLE=parser yasp/yasp:latest sh -c "node deploy.js"
+    sudo docker run -d --name=parser --restart=always -e PROVIDER=gce -e ROLE=parser yasp/yasp:latest sh -c "npm start"
     sudo docker start parser
 '
 gcloud compute instance-groups managed create "parser-group-1" --base-instance-name "parser-group-1" --template "parser-1" --size "1"

From 240d755a12ac7aadcb0f5e545b74b9033b376f79 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Mon, 13 Jun 2016 00:44:10 +0000
Subject: [PATCH 04/49] debian dockerfile

---
 Dockerfile | 134 ++++++++++++++++++-----------------------------------
 1 file changed, 46 insertions(+), 88 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index ba5cacf65..21d323537 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,94 +1,52 @@
-FROM mhart/alpine-node:6.2.1
-
-# Tools
-RUN apk update && apk add git curl wget bash
+FROM node:6.2.1
 
 # Java
 #RUN apk update && apk add openjdk8=8.92.14-r1
-# Here we install GNU libc (aka glibc) and set C.UTF-8 locale as default.
-
-RUN ALPINE_GLIBC_BASE_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" && \
-    ALPINE_GLIBC_PACKAGE_VERSION="2.23-r2" && \
-    ALPINE_GLIBC_BASE_PACKAGE_FILENAME="glibc-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
-    ALPINE_GLIBC_BIN_PACKAGE_FILENAME="glibc-bin-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
-    ALPINE_GLIBC_I18N_PACKAGE_FILENAME="glibc-i18n-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
-    apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
-    wget \
-        "https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub" \
-        -O "/etc/apk/keys/sgerrand.rsa.pub" && \
-    wget \
-        "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
-        "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
-        "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \
-    apk add --no-cache \
-        "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
-        "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
-        "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \
-    \
-    rm "/etc/apk/keys/sgerrand.rsa.pub" && \
-    /usr/glibc-compat/bin/localedef --force --inputfile POSIX --charmap UTF-8 C.UTF-8 || true && \
-    echo "export LANG=C.UTF-8" > /etc/profile.d/locale.sh && \
-    \
-    apk del glibc-i18n && \
-    \
-    apk del build-dependencies && \
-    rm \
-        "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
-        "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
-        "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME"
-
-ENV LANG=C.UTF-8
-
-ENV JAVA_VERSION=8 \
-    JAVA_UPDATE=77 \
-    JAVA_BUILD=03 \
-    JAVA_HOME="/usr/lib/jvm/default-jvm"
-
-RUN apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
-    cd "/tmp" && \
-    wget --header "Cookie: oraclelicense=accept-securebackup-cookie;" \
-        "http://download.oracle.com/otn-pub/java/jdk/${JAVA_VERSION}u${JAVA_UPDATE}-b${JAVA_BUILD}/jdk-${JAVA_VERSION}u${JAVA_UPDATE}-linux-x64.tar.gz" && \
-    tar -xzf "jdk-${JAVA_VERSION}u${JAVA_UPDATE}-linux-x64.tar.gz" && \
-    mkdir -p "/usr/lib/jvm" && \
-    mv "/tmp/jdk1.${JAVA_VERSION}.0_${JAVA_UPDATE}" "/usr/lib/jvm/java-${JAVA_VERSION}-oracle" && \
-    ln -s "java-${JAVA_VERSION}-oracle" "$JAVA_HOME" && \
-    ln -s "$JAVA_HOME/bin/"* "/usr/bin/" && \
-    rm -rf "$JAVA_HOME/"*src.zip && \
-    rm -rf "$JAVA_HOME/lib/missioncontrol" \
-           "$JAVA_HOME/lib/visualvm" \
-           "$JAVA_HOME/lib/"*javafx* \
-           "$JAVA_HOME/jre/lib/plugin.jar" \
-           "$JAVA_HOME/jre/lib/ext/jfxrt.jar" \
-           "$JAVA_HOME/jre/bin/javaws" \
-           "$JAVA_HOME/jre/lib/javaws.jar" \
-           "$JAVA_HOME/jre/lib/desktop" \
-           "$JAVA_HOME/jre/plugin" \
-           "$JAVA_HOME/jre/lib/"deploy* \
-           "$JAVA_HOME/jre/lib/"*javafx* \
-           "$JAVA_HOME/jre/lib/"*jfx* \
-           "$JAVA_HOME/jre/lib/amd64/libdecora_sse.so" \
-           "$JAVA_HOME/jre/lib/amd64/"libprism_*.so \
-           "$JAVA_HOME/jre/lib/amd64/libfxplugins.so" \
-           "$JAVA_HOME/jre/lib/amd64/libglass.so" \
-           "$JAVA_HOME/jre/lib/amd64/libgstreamer-lite.so" \
-           "$JAVA_HOME/jre/lib/amd64/"libjavafx*.so \
-           "$JAVA_HOME/jre/lib/amd64/"libjfx*.so && \
-    rm -rf "$JAVA_HOME/jre/bin/jjs" \
-           "$JAVA_HOME/jre/bin/keytool" \
-           "$JAVA_HOME/jre/bin/orbd" \
-           "$JAVA_HOME/jre/bin/pack200" \
-           "$JAVA_HOME/jre/bin/policytool" \
-           "$JAVA_HOME/jre/bin/rmid" \
-           "$JAVA_HOME/jre/bin/rmiregistry" \
-           "$JAVA_HOME/jre/bin/servertool" \
-           "$JAVA_HOME/jre/bin/tnameserv" \
-           "$JAVA_HOME/jre/bin/unpack200" \
-           "$JAVA_HOME/jre/lib/ext/nashorn.jar" \
-           "$JAVA_HOME/jre/lib/jfr.jar" \
-           "$JAVA_HOME/jre/lib/jfr" \
-           "$JAVA_HOME/jre/lib/oblique-fonts" && \
-    apk del build-dependencies && \
-    rm "/tmp/"*
+# A few problems with compiling Java from source:
+#  1. Oracle.  Licensing prevents us from redistributing the official JDK.
+#  2. Compiling OpenJDK also requires the JDK to be installed, and it gets
+#       really hairy.
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+		bzip2 \
+		unzip \
+		xz-utils \
+	&& rm -rf /var/lib/apt/lists/*
+
+RUN echo 'deb http://httpredir.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/jessie-backports.list
+
+# Default to UTF-8 file.encoding
+ENV LANG C.UTF-8
+
+# add a simple script that can auto-detect the appropriate JAVA_HOME value
+# based on whether the JDK or only the JRE is installed
+RUN { \
+		echo '#!/bin/sh'; \
+		echo 'set -e'; \
+		echo; \
+		echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \
+	} > /usr/local/bin/docker-java-home \
+	&& chmod +x /usr/local/bin/docker-java-home
+
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
+
+ENV JAVA_VERSION 8u91
+ENV JAVA_DEBIAN_VERSION 8u91-b14-1~bpo8+1
+
+# see https://bugs.debian.org/775775
+# and https://github.com/docker-library/java/issues/19#issuecomment-70546872
+ENV CA_CERTIFICATES_JAVA_VERSION 20140324
+
+RUN set -x \
+	&& apt-get update \
+	&& apt-get install -y \
+		openjdk-8-jdk="$JAVA_DEBIAN_VERSION" \
+		ca-certificates-java="$CA_CERTIFICATES_JAVA_VERSION" \
+	&& rm -rf /var/lib/apt/lists/* \
+	&& [ "$JAVA_HOME" = "$(docker-java-home)" ]
+
+# see CA_CERTIFICATES_JAVA_VERSION notes above
+RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure
 
 # Maven
 # as of 2016-06-12 maven apk is still in edge branch and not accessible by default

From 12d046725a122b38e88679f224256f2e3d64fe5f Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Mon, 13 Jun 2016 01:09:23 +0000
Subject: [PATCH 05/49] fix rsa key

---
 Dockerfile | 135 +++++++++++++++++++++++++++++++++++------------------
 1 file changed, 89 insertions(+), 46 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 21d323537..9b8128f88 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,52 +1,95 @@
-FROM node:6.2.1
+FROM mhart/alpine-node:6.2.1
+
+# Tools
+RUN apk update && apk add git curl wget bash
 
 # Java
+# as of 2016-06-12, running on alpine openjdk causes a unsatisfiedlinkerror crash in Java
 #RUN apk update && apk add openjdk8=8.92.14-r1
-# A few problems with compiling Java from source:
-#  1. Oracle.  Licensing prevents us from redistributing the official JDK.
-#  2. Compiling OpenJDK also requires the JDK to be installed, and it gets
-#       really hairy.
-
-RUN apt-get update && apt-get install -y --no-install-recommends \
-		bzip2 \
-		unzip \
-		xz-utils \
-	&& rm -rf /var/lib/apt/lists/*
-
-RUN echo 'deb http://httpredir.debian.org/debian jessie-backports main' > /etc/apt/sources.list.d/jessie-backports.list
-
-# Default to UTF-8 file.encoding
-ENV LANG C.UTF-8
-
-# add a simple script that can auto-detect the appropriate JAVA_HOME value
-# based on whether the JDK or only the JRE is installed
-RUN { \
-		echo '#!/bin/sh'; \
-		echo 'set -e'; \
-		echo; \
-		echo 'dirname "$(dirname "$(readlink -f "$(which javac || which java)")")"'; \
-	} > /usr/local/bin/docker-java-home \
-	&& chmod +x /usr/local/bin/docker-java-home
-
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-
-ENV JAVA_VERSION 8u91
-ENV JAVA_DEBIAN_VERSION 8u91-b14-1~bpo8+1
-
-# see https://bugs.debian.org/775775
-# and https://github.com/docker-library/java/issues/19#issuecomment-70546872
-ENV CA_CERTIFICATES_JAVA_VERSION 20140324
-
-RUN set -x \
-	&& apt-get update \
-	&& apt-get install -y \
-		openjdk-8-jdk="$JAVA_DEBIAN_VERSION" \
-		ca-certificates-java="$CA_CERTIFICATES_JAVA_VERSION" \
-	&& rm -rf /var/lib/apt/lists/* \
-	&& [ "$JAVA_HOME" = "$(docker-java-home)" ]
-
-# see CA_CERTIFICATES_JAVA_VERSION notes above
-RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure
+# Here we install GNU libc (aka glibc) and set C.UTF-8 locale as default.
+
+RUN ALPINE_GLIBC_BASE_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" && \
+    ALPINE_GLIBC_PACKAGE_VERSION="2.23-r2" && \
+    ALPINE_GLIBC_BASE_PACKAGE_FILENAME="glibc-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
+    ALPINE_GLIBC_BIN_PACKAGE_FILENAME="glibc-bin-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
+    ALPINE_GLIBC_I18N_PACKAGE_FILENAME="glibc-i18n-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
+    apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
+    wget \
+        "https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$ALPINE_GLIBC_PACKAGE_VERSION/sgerrand.rsa.pub" \
+        -O "/etc/apk/keys/sgerrand.rsa.pub" && \
+    wget \
+        "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
+        "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
+        "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \
+    apk add --no-cache \
+        "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
+        "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
+        "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \
+    \
+    rm "/etc/apk/keys/sgerrand.rsa.pub" && \
+    /usr/glibc-compat/bin/localedef --force --inputfile POSIX --charmap UTF-8 C.UTF-8 || true && \
+    echo "export LANG=C.UTF-8" > /etc/profile.d/locale.sh && \
+    \
+    apk del glibc-i18n && \
+    \
+    apk del build-dependencies && \
+    rm \
+        "$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
+        "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
+        "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME"
+
+ENV LANG=C.UTF-8
+
+ENV JAVA_VERSION=8 \
+    JAVA_UPDATE=77 \
+    JAVA_BUILD=03 \
+    JAVA_HOME="/usr/lib/jvm/default-jvm"
+
+RUN apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
+    cd "/tmp" && \
+    wget --header "Cookie: oraclelicense=accept-securebackup-cookie;" \
+        "http://download.oracle.com/otn-pub/java/jdk/${JAVA_VERSION}u${JAVA_UPDATE}-b${JAVA_BUILD}/jdk-${JAVA_VERSION}u${JAVA_UPDATE}-linux-x64.tar.gz" && \
+    tar -xzf "jdk-${JAVA_VERSION}u${JAVA_UPDATE}-linux-x64.tar.gz" && \
+    mkdir -p "/usr/lib/jvm" && \
+    mv "/tmp/jdk1.${JAVA_VERSION}.0_${JAVA_UPDATE}" "/usr/lib/jvm/java-${JAVA_VERSION}-oracle" && \
+    ln -s "java-${JAVA_VERSION}-oracle" "$JAVA_HOME" && \
+    ln -s "$JAVA_HOME/bin/"* "/usr/bin/" && \
+    rm -rf "$JAVA_HOME/"*src.zip && \
+    rm -rf "$JAVA_HOME/lib/missioncontrol" \
+           "$JAVA_HOME/lib/visualvm" \
+           "$JAVA_HOME/lib/"*javafx* \
+           "$JAVA_HOME/jre/lib/plugin.jar" \
+           "$JAVA_HOME/jre/lib/ext/jfxrt.jar" \
+           "$JAVA_HOME/jre/bin/javaws" \
+           "$JAVA_HOME/jre/lib/javaws.jar" \
+           "$JAVA_HOME/jre/lib/desktop" \
+           "$JAVA_HOME/jre/plugin" \
+           "$JAVA_HOME/jre/lib/"deploy* \
+           "$JAVA_HOME/jre/lib/"*javafx* \
+           "$JAVA_HOME/jre/lib/"*jfx* \
+           "$JAVA_HOME/jre/lib/amd64/libdecora_sse.so" \
+           "$JAVA_HOME/jre/lib/amd64/"libprism_*.so \
+           "$JAVA_HOME/jre/lib/amd64/libfxplugins.so" \
+           "$JAVA_HOME/jre/lib/amd64/libglass.so" \
+           "$JAVA_HOME/jre/lib/amd64/libgstreamer-lite.so" \
+           "$JAVA_HOME/jre/lib/amd64/"libjavafx*.so \
+           "$JAVA_HOME/jre/lib/amd64/"libjfx*.so && \
+    rm -rf "$JAVA_HOME/jre/bin/jjs" \
+           "$JAVA_HOME/jre/bin/keytool" \
+           "$JAVA_HOME/jre/bin/orbd" \
+           "$JAVA_HOME/jre/bin/pack200" \
+           "$JAVA_HOME/jre/bin/policytool" \
+           "$JAVA_HOME/jre/bin/rmid" \
+           "$JAVA_HOME/jre/bin/rmiregistry" \
+           "$JAVA_HOME/jre/bin/servertool" \
+           "$JAVA_HOME/jre/bin/tnameserv" \
+           "$JAVA_HOME/jre/bin/unpack200" \
+           "$JAVA_HOME/jre/lib/ext/nashorn.jar" \
+           "$JAVA_HOME/jre/lib/jfr.jar" \
+           "$JAVA_HOME/jre/lib/jfr" \
+           "$JAVA_HOME/jre/lib/oblique-fonts" && \
+    apk del build-dependencies && \
+    rm "/tmp/"*
 
 # Maven
 # as of 2016-06-12 maven apk is still in edge branch and not accessible by default

From f51a9125b8c2f5595bfd27655b7ad7452492cbd2 Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Sun, 12 Jun 2016 23:58:33 -0700
Subject: [PATCH 06/49] Update patch.json

---
 json/patch.json | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/json/patch.json b/json/patch.json
index 0d84d0909..29f97b544 100644
--- a/json/patch.json
+++ b/json/patch.json
@@ -70,5 +70,9 @@
 {
   "name": "6.87",
   "date": "2016-04-26T01:00:00Z"
+},
+{
+  "name": "6.88",
+  "date": "2016-06-12T08:00:00Z"
 }
 ]

From 5f735194eb744aa9b43452c57da26ae4fe8f81d1 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Tue, 14 Jun 2016 02:52:02 +0000
Subject: [PATCH 07/49] use query strings to pass params for
 rankings/benchmarks

---
 routes/api.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index 5c24912d1..c2afa0c3f 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -145,9 +145,9 @@ module.exports = function(db, redis, cassandra)
             res.json(result);
         });
     });
-    api.get('/rankings/:hero_id', function(req, res, cb)
+    api.get('/rankings', function(req, res, cb)
     {
-        queries.getHeroRankings(db, redis, req.params.hero_id,
+        queries.getHeroRankings(db, redis, req.query.hero_id,
         {}, function(err, result)
         {
             if (err)
@@ -157,11 +157,11 @@ module.exports = function(db, redis, cassandra)
             res.json(result);
         });
     });
-    api.get('/benchmarks/:hero_id', function(req, res, cb)
+    api.get('/benchmarks', function(req, res, cb)
     {
         queries.getBenchmarks(db, redis,
         {
-            hero_id: req.params.hero_id
+            hero_id: req.query.hero_id
         }, function(err, result)
         {
             if (err)

From 5eac1fe4055e3449e6ad2f9fbfa9ff7451e6b5dc Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Tue, 14 Jun 2016 06:35:01 +0000
Subject: [PATCH 08/49] add extra config for parallelism

---
 Dockerfile    | 2 +-
 config.js     | 2 ++
 svc/mmr.js    | 2 +-
 svc/parser.js | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 9b8128f88..ededbc988 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM mhart/alpine-node:6.2.1
+FROM mhart/alpine-node:5.10.1
 
 # Tools
 RUN apk update && apk add git curl wget bash
diff --git a/config.js b/config.js
index 444367392..0ee61c505 100644
--- a/config.js
+++ b/config.js
@@ -44,6 +44,8 @@ var defaults = {
     "DEFAULT_DELAY": 1000, // delay between API requests (default: 1000)
     "SCANNER_DELAY": 300, //delay for scanner API requests (more time-sensitive)
     "SCANNER_PARALLELISM": 1, //Number of simultaneous API requests to make in scanner
+    "MMR_PARALLELISM": 15,
+    "PARSER_PARALLELISM": 1,
     "PLAYER_MATCH_LIMIT": 50000, //max results to return from player matches
     "BENCHMARK_RETENTION_HOURS": 1, //hours in block to retain benchmark data for percentile
     "MATCH_RATING_RETENTION_HOURS": 12, //hours in block to retain match rating data for percentile
diff --git a/svc/mmr.js b/svc/mmr.js
index 8071b283c..68916d074 100644
--- a/svc/mmr.js
+++ b/svc/mmr.js
@@ -10,7 +10,7 @@ var config = require('../config');
 var mQueue = queue.getQueue('mmr');
 var getData = utility.getData;
 var retrieverArr = config.RETRIEVER_HOST.split(",");
-mQueue.process(retrieverArr.length * 10, processMmr);
+mQueue.process(retrieverArr.length * config.MMR_PARALLELISM, processMmr);
 mQueue.on('completed', function(job)
 {
     job.remove();
diff --git a/svc/parser.js b/svc/parser.js
index 7843e0609..0da314d12 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -59,7 +59,7 @@ app.get('/redis/:key', function(req, res, cb)
 });
 app.listen(config.PARSER_PORT);
 //END EXPRESS
-pQueue.process(1, function(job, cb)
+pQueue.process(config.PARSER_PARALLELISM, function(job, cb)
 {
     console.log("parse job: %s", job.jobId);
     var match = job.data.payload;

From 82010a9d11e5c1f83cb1fef7310dfab2548dfdc9 Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Mon, 13 Jun 2016 23:45:53 -0700
Subject: [PATCH 09/49] Update package.json

---
 package.json | 1 -
 1 file changed, 1 deletion(-)

diff --git a/package.json b/package.json
index 2e7b97634..5556fa686 100644
--- a/package.json
+++ b/package.json
@@ -53,7 +53,6 @@
     "font-awesome": "^4.5.0",
     "heatmap.js": "2.0.0",
     "http-proxy": "^1.13.1",
-    "isomorphic-fetch": "^2.2.1",
     "jade": "^1.11.0",
     "jquery": "^2.2.0",
     "knex": "^0.10.0",

From cf5b9fc5328bf5a5872a879ec4043ce86c77bee0 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 16 Jun 2016 02:28:29 +0000
Subject: [PATCH 10/49] fix hero_id 0 issue

---
 store/buildPlayer.js         | 14 +++++++++-----
 views/mixins/hero_table.jade |  5 +++--
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/store/buildPlayer.js b/store/buildPlayer.js
index 84606c068..05d0b5157 100644
--- a/store/buildPlayer.js
+++ b/store/buildPlayer.js
@@ -199,12 +199,16 @@ function buildPlayer(options, cb)
                     {
                         for (var id in aggData.hero_id.counts)
                         {
-                            heroes_list.push(
+                            //exclude invalid hero_ids
+                            if (Number(id))
                             {
-                                hero_id: id,
-                                games: aggData.hero_id.counts[id],
-                                win: aggData.hero_id.win_counts[id]
-                            });
+                                heroes_list.push(
+                                {
+                                    hero_id: id,
+                                    games: aggData.hero_id.counts[id],
+                                    win: aggData.hero_id.win_counts[id]
+                                });
+                            }
                         }
                     }
                     else if (aggData.heroes)
diff --git a/views/mixins/hero_table.jade b/views/mixins/hero_table.jade
index a5d76a6a6..eaa19f786 100644
--- a/views/mixins/hero_table.jade
+++ b/views/mixins/hero_table.jade
@@ -19,8 +19,9 @@ mixin hero_table(rows, short)
         - var hero = constants.heroes[h.hero_id]
         tr
           td.text-left.small
-            img.img-sm(src=hero.img, title=hero.localized_name)
-            =hero.localized_name
+            if hero
+              img.img-sm(src=hero.img, title=hero.localized_name)
+              =hero.localized_name
           td.rankable=h.games
           td.rankable
             +progress_bar(h.win, h.games)

From ddada3c441963d3832625815920e5ce2e79b6b99 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 16 Jun 2016 02:59:08 +0000
Subject: [PATCH 11/49] update readme

---
 README.md | 34 ++++++++++++++++++----------------
 1 file changed, 18 insertions(+), 16 deletions(-)

diff --git a/README.md b/README.md
index 67a44209d..20c511d5e 100644
--- a/README.md
+++ b/README.md
@@ -20,15 +20,12 @@ Tech Stack
 * Storage: PostgreSQL/Redis/Cassandra
 * Parser: Java (powered by [clarity](https://github.com/skadistats/clarity))
 
-Quickstart
+Quickstart (Docker)
 ----
-* Using Docker will let you run the code in a container with all dependencies properly installed.
 * Install Docker: `curl -sSL https://get.docker.com/ | sh`
 * Clone the repo: `git clone https://github.com/yasp-dota/yasp`
 * Go into the directory: `cd yasp`
-* Create .env file with required config values in KEY=VALUE format (see config.js for a full listing of options) `cp .env_example .env`
-  * `STEAM_API_KEY` You need this in order to access the Steam Web API.  
-  * `STEAM_USER, STEAM_PASS` The retriever requires a Steam account in order to fetch replay salts.  We recommend creating a new account for this purpose (you won't be able to log into the account while the retriever is using it).  If you don't care about getting replay salts/downloading replays then you can skip this step.
+* Build the Docker container: `sudo docker build -t yasp/yasp .`
 * Start a new container running the image, and map your local directory into the container: `sudo docker run -v $(pwd):/usr/src/yasp -di --name yasp --net=host yasp/yasp:latest`
 * Start the external dependencies in separate containers.
   * `sudo docker run -d --name postgres --net=host postgres:9.5`
@@ -37,20 +34,17 @@ Quickstart
 * Initialize Postgres: `sudo docker exec -i postgres psql -U postgres < sql/init.sql`
 * Create tables: `sudo docker exec -i postgres psql -U postgres yasp < sql/create_tables.sql`
 * Set up Cassandra (optional): `sudo docker exec -i cassandra cqlsh < sql/cassandra.cql`
+* Create .env file with required config values in KEY=VALUE format (see config.js for a full listing of options) `cp .env_example .env`
+  * `STEAM_API_KEY` You need this in order to access the Steam Web API.  
+  * `STEAM_USER, STEAM_PASS` The retriever requires a Steam account in order to fetch replay salts.  We recommend creating a new account for this purpose (you won't be able to log into the account while the retriever is using it).  If you don't care about getting replay salts/downloading replays then you can skip this step.
 * Get a terminal into the running container: `sudo docker exec -it yasp bash`
-* Build inside the container: `npm run build`
 * Start the services you want to run:
-  * `pm2 start profiles/basic.json` This starts all the basic services to be able to open the site in a browser and request parses by ID (which is a useful end-to-end test).  Use `profiles/everything.json` to start everything.
-  * `pm2 start svc/web.js --watch` This starts a specific service and watches it for changes.
-  * `pm2 logs web` You can use this command to inspect the output of a service.
-  * `pm2 delete all` Stop and remove all the services.
-* Alternatively, if you have Docker Compose [installed](https://docs.docker.com/compose/install/) you can just run `docker-compose up`.
- * 3 containers will be built and launched - one with postgres database, one with redis and one with web service.
- * Database is inited and tables are created automatically.
- * By default, minimal configuration necessairy to open the site in a browser and request parses by ID is started. This can be overridden via `docker-compose.override.yml`.
- * `sudo docker exec -it yasp_web_1 bash` will give you a terminal into the running web container.
+  * `pm2 start profiles/basic.json` This starts all the basic services to be able to read the API and request parses (which is a useful end-to-end test).  Use `profiles/everything.json` to start everything.
+  * Useful PM2 commands:
+    * `pm2 start svc/web.js --watch` This starts a specific service and watches it for changes.
+    * `pm2 logs web` You can use this command to inspect the output of a service.
+    * `pm2 delete all` Stop and remove all the services.
 * Useful commands
-  * `npm run watch`: If you want to make changes to client side JS, you will want to run the watch script in order to automatically rebuild after making changes.
   * `npm test` runs the full test suite.  Use `mocha` for more fine-grained control over the tests you want to run.
   * `node tasks/updateconstants` pulls latest constants data and saves to `json` directory.
 * Get some starter data
@@ -61,6 +55,14 @@ Quickstart
 * Submit a pull request.  Wait for it to be reviewed and merged.
 * Congratulations!  You're a contributor.
 
+Docker Compose
+----
+* Alternatively, if you have Docker Compose [installed](https://docs.docker.com/compose/install/) you can just run `docker-compose up`.
+ * 3 containers will be built and launched - one with postgres database, one with redis and one with web service.
+ * Database is inited and tables are created automatically.
+ * By default, minimal configuration necessairy to open the site in a browser and request parses by ID is started. This can be overridden via `docker-compose.override.yml`.
+ * `sudo docker exec -it yasp_web_1 bash` will give you a terminal into the running web container.
+
 Getting Help
 ----
 * Feel free to open a new issue to ask questions/get help!

From 11fbb6f94f7c639d6c902082ba2934025f45b009 Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Wed, 15 Jun 2016 21:08:25 -0700
Subject: [PATCH 12/49] per_min every benchmark

---
 util/benchmarks.js | 22 +++++-----------------
 1 file changed, 5 insertions(+), 17 deletions(-)

diff --git a/util/benchmarks.js b/util/benchmarks.js
index 1797a93cc..f8b5fa0a3 100644
--- a/util/benchmarks.js
+++ b/util/benchmarks.js
@@ -19,25 +19,13 @@ var benchmarks = {
     {
         return (p.hero_damage / m.duration * 60);
     },
-    "kills": function(m, p)
+    "hero_healing_per_min": function(m, p)
     {
-        return p.kills;
+        return (p.hero_healing / m.duration * 60);
     },
-    "last_hits": function(m, p)
+    "tower_damage_per_min": function(m, p)
     {
-        return p.last_hits;
-    },
-    "hero_damage": function(m, p)
-    {
-        return p.hero_damage;
-    },
-    "tower_damage": function(m, p)
-    {
-        return p.tower_damage;
-    },
-    "hero_healing": function(m, p)
-    {
-        return p.hero_healing;
+        return (p.tower_damage / m.duration * 60);
     },
 };
-module.exports = benchmarks;
\ No newline at end of file
+module.exports = benchmarks;

From ad74c716520dbf2032f8ede4f39cc3d87fb8d150 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 16 Jun 2016 09:09:41 +0000
Subject: [PATCH 13/49] make profiler add new players

---
 store/queries.js | 16 +++++++++++++++-
 svc/profiler.js  | 21 ++++++++++-----------
 svc/web.js       |  2 +-
 3 files changed, 26 insertions(+), 13 deletions(-)

diff --git a/store/queries.js b/store/queries.js
index c4952d6ed..c602eaacb 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -198,6 +198,7 @@ function insertMatch(db, redis, match, options, cb)
         "cmc": clearMatchCache,
         "t": telemetry,
         "dm": decideMmr,
+        "dpro": decideProfile,
         "dp": decideParse,
     }, function(err, results)
     {
@@ -362,7 +363,7 @@ function insertMatch(db, redis, match, options, cb)
     {
         async.each(match.players, function(p, cb)
         {
-            if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id !== constants.anonymous_account_id && (p.account_id in options.userPlayers || (config.ENABLE_RANDOM_MMR_UPDATE && match.match_id % 3 === 0)))
+            if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && (p.account_id in options.userPlayers || (config.ENABLE_RANDOM_MMR_UPDATE && match.match_id % 3 === 0)))
             {
                 addToQueue(mQueue,
                 {
@@ -381,6 +382,19 @@ function insertMatch(db, redis, match, options, cb)
         }, cb);
     }
 
+    function decideProfile(cb)
+    {
+        async.each(match.players, function(p, cb)
+        {
+            if (options.origin === "scanner" && p.account_id && p.account_id !== constants.anonymous_account_id)
+            {
+                redis.lpush('profilerQueue', p.account_id);
+                redis.ltrim('profilerQueue', 0, 99);
+            }
+            cb();
+        }, cb);
+    }
+
     function decideParse(cb)
     {
         if (match.parse_status !== 0)
diff --git a/svc/profiler.js b/svc/profiler.js
index 68582f244..cfd2d4aae 100644
--- a/svc/profiler.js
+++ b/svc/profiler.js
@@ -2,8 +2,10 @@
  * Worker to fetch updated player profiles
  **/
 var constants = require('../constants');
+var config = require('../config');
 var queries = require('../store/queries');
 var db = require('../store/db');
+var redis = require('../store/redis');
 var utility = require('../util/utility');
 var insertPlayer = queries.insertPlayer;
 var getData = utility.getData;
@@ -24,25 +26,22 @@ function start()
 
 function getSummaries(cb)
 {
-    db.raw(`
-        SELECT account_id 
-        FROM players 
-        TABLESAMPLE SYSTEM_ROWS(100)
-    `).asCallback(function(err, results)
+    redis.lrange('profilerQueue', 0, -1, function(err, results)
     {
         if (err)
         {
             return cb(err);
         }
-        if (results.rows.length === 0)
+        console.log('players sampled: %s', results.length);
+        results = results.map(function(account_id)
         {
-            console.log('No account_ids found...');
-            return cb();
-        }
-        console.log('players sampled: %s', results.rows.length);
+            return {
+                account_id: account_id
+            };
+        });
         var container = utility.generateJob("api_summaries",
         {
-            players: results.rows
+            players: results
         });
         getData(container.url, function(err, body)
         {
diff --git a/svc/web.js b/svc/web.js
index d78e2b004..13ebd837e 100644
--- a/svc/web.js
+++ b/svc/web.js
@@ -142,7 +142,7 @@ app.use(function telemetry(req, res, cb)
         };
         */
         redis.lpush("load_times", timeEnd - timeStart);
-        redis.ltrim("load_times", 0, 10000);
+        redis.ltrim("load_times", 0, 9999);
     });
     cb();
 });

From cfbb3b91a334e9b293096d6e6cb1156e5be1a37d Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Fri, 17 Jun 2016 04:04:07 +0000
Subject: [PATCH 14/49] remove ndjson dep

---
 package.json  |  1 -
 svc/parser.js | 28 ++++++++++++++++------------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/package.json b/package.json
index 5556fa686..b88d0ee1f 100644
--- a/package.json
+++ b/package.json
@@ -58,7 +58,6 @@
     "knex": "^0.10.0",
     "moment": "^2.11.2",
     "multer": "^1.1.0",
-    "ndjson": "^1.4.3",
     "numeral": "^1.5.3",
     "passport": "0.3.2",
     "passport-steam": "1.0.1",
diff --git a/svc/parser.js b/svc/parser.js
index 0da314d12..90ed32b2f 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -27,7 +27,7 @@ var progress = require('request-progress');
 var stream = require('stream');
 var pQueue = queue.getQueue('parse');
 var async = require('async');
-var ndjson = require('ndjson');
+const readline = require('readline');
 var spawn = cp.spawn;
 var insertMatch = queries.insertMatch;
 var benchmarkMatch = queries.benchmarkMatch;
@@ -176,7 +176,8 @@ function runParse(match, job, cb)
     // Streams
     var inStream = progress(request(
     {
-        url: url
+        url: url,
+        encoding: null,
     }));
     inStream.on('progress', function(state)
     {
@@ -211,10 +212,11 @@ function runParse(match, job, cb)
     }
     bz.stdin.on('error', exit);
     bz.stdout.on('error', exit);
+    inStream.pipe(bz.stdin);
     var parser = spawn("java", [
         "-jar",
-        "-Xmx64m",
-        "./java_parser/target/stats-0.1.0.jar"
+        "-Xmx128m",
+        "./java_parser/target/stats-0.1.0.jar",
         ],
     {
         stdio: ['pipe', 'pipe', 'pipe'],
@@ -226,22 +228,24 @@ function runParse(match, job, cb)
     {
         console.log(data.toString());
     });
-    var parseStream = ndjson.parse();
-    parseStream.on('data', function handleStream(e)
+    bz.stdout.pipe(parser.stdin);
+    const parseStream = readline.createInterface(
+    {
+        input: parser.stdout
+    });
+    parseStream.on('line', function handleStream(e)
     {
+        e = JSON.parse(e);
         if (e.type === 'epilogue')
         {
             console.log('received epilogue');
             incomplete = false;
+            parseStream.close();
+            exit();
         }
         entries.push(e);
     });
-    parseStream.on('end', exit);
-    parseStream.on('error', exit);
-    // Pipe together the streams
-    inStream.pipe(bz.stdin);
-    bz.stdout.pipe(parser.stdin);
-    parser.stdout.pipe(parseStream);
+    request.debug = true
 
     function exit(err)
     {

From 63207e4aa7b44ae17e254291c5ddc00dbf5d0efb Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 18 Jun 2016 05:51:30 +0000
Subject: [PATCH 15/49] add api

---
 java_parser/src/main/java/yasp/Main.java |  8 ++++----
 routes/api.js                            | 11 +++++++++++
 sql/create_tables.sql                    |  2 --
 store/queries.js                         |  6 +++++-
 4 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java
index f507f938f..edca25d14 100644
--- a/java_parser/src/main/java/yasp/Main.java
+++ b/java_parser/src/main/java/yasp/Main.java
@@ -70,8 +70,8 @@ private class Entry {
         public Integer gold_reason;
         public Integer xp_reason;
         public String valuename;
-        public Float stun_duration;
-        public Float slow_duration;
+        //public Float stun_duration;
+        //public Float slow_duration;
         //entity fields
         public Integer gold;
         public Integer lh;
@@ -234,8 +234,8 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) {
             combatLogEntry.attackerillusion = cle.isAttackerIllusion();
             combatLogEntry.targetillusion = cle.isTargetIllusion();
             combatLogEntry.value = cle.getValue();
-            combatLogEntry.stun_duration = cle.getStunDuration();
-            combatLogEntry.slow_duration = cle.getSlowDuration();
+            //combatLogEntry.stun_duration = cle.getStunDuration();
+            //combatLogEntry.slow_duration = cle.getSlowDuration();
             //value may be out of bounds in string table, we can only get valuename if a purchase (type 11)
             if (cle.getType() == DOTA_COMBATLOG_TYPES.DOTA_COMBATLOG_PURCHASE) {
                 combatLogEntry.valuename = cle.getValueName();
diff --git a/routes/api.js b/routes/api.js
index c2afa0c3f..f8a878abf 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -134,6 +134,17 @@ module.exports = function(db, redis, cassandra)
         });
     });
     */
+    api.get('/match_logs/:match_id', function(req, res, cb)
+    {
+        db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
     api.get('/distributions', function(req, res, cb)
     {
         queries.getDistributions(redis, function(err, result)
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index d20cac76b..eb9f960a4 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -199,8 +199,6 @@ CREATE TABLE match_logs (
   gold_reason int,
   xp_reason int,
   valuename varchar(100),
-  stun_duration real,
-  slow_duration real,
   gold int,
   lh int,
   xp int,
diff --git a/store/queries.js b/store/queries.js
index cf191fab2..20e698405 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -208,7 +208,7 @@ function insertMatch(db, redis, match, options, cb)
 
     function upsertMatch(cb)
     {
-        if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE)
+        if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.leagueid)
         {
             return cb();
         }
@@ -232,6 +232,10 @@ function insertMatch(db, redis, match, options, cb)
                         player_slot: pm.player_slot
                     }, cb);
                 }, exit);
+                //TODO create table for picks/bans
+                //TODO insert picks/bans
+                //TODO remove dota_unknown to save space
+                //TODO figure out whehter to store expanded or raw data
 
                 function exit(err)
                 {

From fd63f476472a40ccbf5f4a1a9a677acca75cfdfa Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 18 Jun 2016 05:53:54 +0000
Subject: [PATCH 16/49] add todos

---
 sql/create_tables.sql | 2 +-
 store/queries.js      | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index eb9f960a4..5efdad478 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -176,7 +176,7 @@ CREATE TABLE notable_players (
 );
 
 CREATE TABLE match_logs (
-  match_id bigint,
+  match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
   time int,
   type varchar(100),
   team int,
diff --git a/store/queries.js b/store/queries.js
index 20e698405..78831b848 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -232,6 +232,8 @@ function insertMatch(db, redis, match, options, cb)
                         player_slot: pm.player_slot
                     }, cb);
                 }, exit);
+                //TODO create leagues table
+                //TODO create worker task to get leagues
                 //TODO create table for picks/bans
                 //TODO insert picks/bans
                 //TODO remove dota_unknown to save space

From 68ada3214f84dc63a0f85660feb2314a27261d8a Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 18 Jun 2016 19:04:08 +0000
Subject: [PATCH 17/49] store raw data, with slot indexes

---
 processors/processReduce.js                 | 32 +++++++++++--
 routes/api.js                               |  2 +-
 sql/create_tables.sql                       | 45 +++++++++++--------
 store/queries.js                            |  4 +-
 svc/parser.js                               |  2 +-
 tasks/getMatches.js                         | 50 +++++++++++++++++++++
 dev/allMatches.js => tasks/getMatchesSeq.js |  0
 7 files changed, 109 insertions(+), 26 deletions(-)
 create mode 100644 tasks/getMatches.js
 rename dev/allMatches.js => tasks/getMatchesSeq.js (100%)

diff --git a/processors/processReduce.js b/processors/processReduce.js
index 1b11d4bda..f8ed5bd5e 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -2,9 +2,9 @@
  * A processor to reduce the event stream by grouping similar events.
  * NOT CURRENTLY IN PRODUCTION USE
  **/
-function processReduce(entries, match)
+function processReduce(entries, match, meta)
 {
-    return entries.filter(function(e)
+    var result = entries.filter(function(e)
     {
         if (e.type === "actions")
         {
@@ -21,11 +21,35 @@ function processReduce(entries, match)
         return true;
     }).map(function(e)
     {
-        return Object.assign(
+        var e2 = Object.assign(
         {}, e,
         {
-            match_id: match.match_id
+            match_id: match.match_id,
+            attackername_slot: meta.hero_to_slot[e.attackername],
+            targetname_slot: meta.hero_to_slot[e.targetname],
+            sourcename_slot: meta.hero_to_slot[e.sourcename],
+            targetsourcename_slot: meta.hero_to_slot[e.targetname],
+            player1_slot: meta.slot_to_playerslot[e.player1],
+            player_slot: e.player_slot || meta.slot_to_playerslot[e.slot],
+            inflictor: translate(e.inflictor),
         });
+        delete e2.attackername;
+        delete e2.targetname;
+        delete e2.sourcename;
+        delete e2.targetsourcename;
+        return e2;
     });
+    var count = {};
+    result.forEach(function(r)
+    {
+        count[r.type] = (count[r.type] || 0) + 1;
+    });
+    console.log(count);
+    return result;
+}
+
+function translate(s)
+{
+    return s === "dota_unknown" ? null : s;
 }
 module.exports = processReduce;
\ No newline at end of file
diff --git a/routes/api.js b/routes/api.js
index f8a878abf..475d2eb85 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -136,7 +136,7 @@ module.exports = function(db, redis, cassandra)
     */
     api.get('/match_logs/:match_id', function(req, res, cb)
     {
-        db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result)
+        db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result)
         {
             if (err)
             {
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 5efdad478..bae99a7d2 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -179,38 +179,45 @@ CREATE TABLE match_logs (
   match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
   time int,
   type varchar(100),
-  team int,
+  team smallint,
   unit varchar(100),
-  key varchar(100),
+  key varchar(1000),
   value int,
-  slot int,
-  player_slot int,
+  slot smallint,
+  player_slot smallint,
   player1 int,
   player2 int,
-  attackername varchar(100),
-  targetname varchar(100),
-  sourcename varchar(100),
-  targetsourcename varchar(100),
   attackerhero boolean,
   targethero boolean,
   attackerillusion boolean,
   targetillusion boolean,
   inflictor varchar(100),
-  gold_reason int,
-  xp_reason int,
+  gold_reason smallint,
+  xp_reason smallint,
   valuename varchar(100),
   gold int,
   lh int,
   xp int,
-  x int,
-  y int,
+  x smallint,
+  y smallint,
   stuns real,
-  hero_id int,
-  life_state int,
-  level int,
-  kills int,
-  deaths int,
-  assists int,
-  denies int
+  hero_id smallint,
+  life_state smallint,
+  level smallint,
+  kills smallint,
+  deaths smallint,
+  assists smallint,
+  denies smallint,
+  attackername_slot smallint,
+  targetname_slot smallint,
+  sourcename_slot smallint,
+  targetsourcename_slot smallint,
+  player1_slot smallint
 );
 CREATE INDEX ON match_logs(match_id);
+CREATE INDEX ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL;
diff --git a/store/queries.js b/store/queries.js
index 78831b848..79a2c9167 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -238,7 +238,9 @@ function insertMatch(db, redis, match, options, cb)
                 //TODO insert picks/bans
                 //TODO remove dota_unknown to save space
                 //TODO figure out whehter to store expanded or raw data
-
+/*
+"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}]
+*/
                 function exit(err)
                 {
                     if (err)
diff --git a/svc/parser.js b/svc/parser.js
index 8c1a75325..5cff09a0c 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -267,7 +267,7 @@ function runParse(match, job, cb)
                 var message = "time spent on post-processing match ";
                 console.time(message);
                 var meta = processMetadata(entries);
-                var logs = match.leagueid ? processReduce(entries, match) : undefined;
+                var logs = match.leagueid ? processReduce(entries, match, meta) : undefined;
                 var res = processExpand(entries, meta);
                 var parsed_data = processParsedData(res.parsed_data);
                 var teamfights = processTeamfights(res.tf_data, meta);
diff --git a/tasks/getMatches.js b/tasks/getMatches.js
new file mode 100644
index 000000000..cf2d7fd92
--- /dev/null
+++ b/tasks/getMatches.js
@@ -0,0 +1,50 @@
+var async = require('async');
+var utility = require('../util/utility');
+var generateJob = utility.generateJob;
+var getData = utility.getData;
+var db = require('../store/db');
+var redis = require('../store/redis');
+var cassandra = require('../store/cassandra');
+var queries = require('../store/queries');
+var insertMatch = queries.insertMatch;
+var args = process.argv.slice(2);
+var match_id = Number(args[0]);
+var delay = 1000;
+var job = generateJob("api_details",
+{
+    match_id: match_id
+});
+var url = job.url;
+getData(
+{
+    url: url,
+    delay: delay
+}, function(err, body)
+{
+    if (err)
+    {
+        throw err;
+    }
+    if (body.result)
+    {
+        var match = body.result;
+        match.parse_status = 0;
+        insertMatch(db, redis, match,
+        {
+            skipCounts: true,
+            skipAbilityUpgrades: true,
+            cassandra: cassandra,
+        }, function(err)
+        {
+            if (err)
+            {
+                throw err;
+            }
+            process.exit(0);
+        });
+    }
+    else
+    {
+        throw body;
+    }
+});
\ No newline at end of file
diff --git a/dev/allMatches.js b/tasks/getMatchesSeq.js
similarity index 100%
rename from dev/allMatches.js
rename to tasks/getMatchesSeq.js

From 5530d780d0d8220dbde4ebfbab4f978e26ee2a4f Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 18 Jun 2016 23:34:48 +0000
Subject: [PATCH 18/49] remove some debugging info

---
 processors/processExpand.js | 2 +-
 processors/processReduce.js | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/processors/processExpand.js b/processors/processExpand.js
index e67b97647..c91c15ca3 100644
--- a/processors/processExpand.js
+++ b/processors/processExpand.js
@@ -449,7 +449,7 @@ function processExpand(entries, meta)
     //null means all types
     var reqs = {
         parsed_data: null,
-        expanded: null,
+        //expanded: null,
         tf_data:
         {
             killed: 1,
diff --git a/processors/processReduce.js b/processors/processReduce.js
index f8ed5bd5e..2bf9dbe00 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -1,6 +1,5 @@
 /**
- * A processor to reduce the event stream by grouping similar events.
- * NOT CURRENTLY IN PRODUCTION USE
+ * A processor to reduce the event stream to only logs we want to persist
  **/
 function processReduce(entries, match, meta)
 {
@@ -39,12 +38,14 @@ function processReduce(entries, match, meta)
         delete e2.targetsourcename;
         return e2;
     });
+    /*
     var count = {};
     result.forEach(function(r)
     {
         count[r.type] = (count[r.type] || 0) + 1;
     });
     console.log(count);
+    */
     return result;
 }
 

From 98fc634794f12e7ffa0e3eda13f8bea3c70ae3ea Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 00:43:06 +0000
Subject: [PATCH 19/49] implement league fetcher, use async for match insert
 flow

---
 sql/create_tables.sql | 17 +++++++++
 store/queries.js      | 81 +++++++++++++++++++++++++++----------------
 svc/parser.js         |  2 +-
 svc/scanner.js        |  7 +---
 svc/worker.js         | 30 ++++++++++++++++
 5 files changed, 101 insertions(+), 36 deletions(-)

diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index bae99a7d2..8d1564d77 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -221,3 +221,20 @@ CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot
 CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL;
 CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL;
 CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL;
+
+CREATE TABLE picks_bans(
+  match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
+  is_pick boolean,
+  hero_id int,
+  team smallint,
+  order smallint,
+  PRIMARY KEY (match_id, order)
+);
+
+CREATE TABLE leagues(
+  leagueid bigint PRIMARY KEY,
+  ticket varchar(255),
+  banner varchar(255),
+  tier varchar(255),
+  name varchar(255)
+)
\ No newline at end of file
diff --git a/store/queries.js b/store/queries.js
index 79a2c9167..f082aac51 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -191,6 +191,7 @@ function insertMatch(db, redis, match, options, cb)
     //we want to insert into matches, then insert into player_matches for each entry in players
     async.series(
     {
+        "ipm": isProMatch,
         "u": upsertMatch,
         "uc": upsertMatchCassandra,
         "uml": upsertMatchLogs,
@@ -206,23 +207,38 @@ function insertMatch(db, redis, match, options, cb)
         return cb(err, results.dp);
     });
 
+    function isProMatch(cb)
+    {
+        //TODO check redis/postgres for professional/premium league
+        match.isProMatch = match.leagueid >= 0;
+        cb();
+    }
+
     function upsertMatch(cb)
     {
-        if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.leagueid)
+        if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.isProMatch)
         {
             return cb();
         }
         db.transaction(function(trx)
         {
-            upsert(trx, 'matches', match,
+            async.series(
             {
-                match_id: match.match_id
-            }, function(err)
+                "im": insertMatch,
+                "ipm": insertPlayerMatches,
+                "ipb": insertPicksBans
+            }, exit);
+
+            function insertMatch(cb)
             {
-                if (err)
+                upsert(trx, 'matches', match,
                 {
-                    return exit(err);
-                }
+                    match_id: match.match_id
+                }, cb);
+            }
+
+            function insertPlayerMatches(cb)
+            {
                 async.each(players || [], function(pm, cb)
                 {
                     pm.match_id = match.match_id;
@@ -231,30 +247,37 @@ function insertMatch(db, redis, match, options, cb)
                         match_id: pm.match_id,
                         player_slot: pm.player_slot
                     }, cb);
-                }, exit);
-                //TODO create leagues table
-                //TODO create worker task to get leagues
-                //TODO create table for picks/bans
-                //TODO insert picks/bans
-                //TODO remove dota_unknown to save space
-                //TODO figure out whehter to store expanded or raw data
-/*
-"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}]
-*/
-                function exit(err)
+                }, cb);
+            }
+
+            function insertPicksBans(cb)
+            {
+                async.each(match.picks_bans || [], function(p, cb)
                 {
-                    if (err)
+                    //order is a reserved keyword
+                    p.ord = p.order;
+                    p.match_id = match.match_id;
+                    upsert(trx, 'picks_bans', p,
                     {
-                        console.error(err);
-                        trx.rollback(err);
-                    }
-                    else
-                    {
-                        trx.commit();
-                    }
-                    cb(err);
+                        match_id: p.match_id,
+                        ord: p.ord
+                    }, cb);
+                }, cb);
+            }
+
+            function exit(err)
+            {
+                if (err)
+                {
+                    console.error(err);
+                    trx.rollback(err);
                 }
-            });
+                else
+                {
+                    trx.commit();
+                }
+                cb(err);
+            }
         });
     }
 
@@ -462,7 +485,7 @@ function insertMatch(db, redis, match, options, cb)
                 duration: match.duration,
                 replay_blob_key: match.replay_blob_key,
                 pgroup: match.pgroup,
-                leagueid: match.leagueid,
+                isProMatch: match.isProMatch,
             },
             {
                 lifo: options.lifo,
diff --git a/svc/parser.js b/svc/parser.js
index 5cff09a0c..1228af883 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -267,7 +267,7 @@ function runParse(match, job, cb)
                 var message = "time spent on post-processing match ";
                 console.time(message);
                 var meta = processMetadata(entries);
-                var logs = match.leagueid ? processReduce(entries, match, meta) : undefined;
+                var logs = match.isProMatch ? processReduce(entries, match, meta) : undefined;
                 var res = processExpand(entries, meta);
                 var parsed_data = processParsedData(res.parsed_data);
                 var teamfights = processTeamfights(res.tf_data, meta);
diff --git a/svc/scanner.js b/svc/scanner.js
index 00d6e3215..fa15a8f35 100644
--- a/svc/scanner.js
+++ b/svc/scanner.js
@@ -112,12 +112,7 @@ function start()
 
             function processMatch(match, cb)
             {
-                if (config.ENABLE_PRO_PARSING && match.leagueid)
-                {
-                    //parse tournament games
-                    match.parse_status = 0;
-                }
-                else if (match.players.some(function(p)
+                if (match.players.some(function(p)
                     {
                         return (p.account_id in trackedPlayers);
                     }))
diff --git a/svc/worker.js b/svc/worker.js
index 4c492994e..ee1cd62f4 100644
--- a/svc/worker.js
+++ b/svc/worker.js
@@ -124,6 +124,36 @@ invokeInterval(function notablePlayers(cb)
         }, cb);
     });
 }, 10 * 60 * 1000);
+invokeInterval(function leagues(cb)
+{
+    utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues)
+    {
+        if (err)
+        {
+            return cb(err);
+        }
+        utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/items_game.json', function(err, items)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            var arr = [];
+            for (var key in leagues)
+            {
+                arr.push(
+                {
+                    leagueid: key,
+                    ticket: leagues[key].ticket,
+                    banner: leagues[key].banner,
+                    tier: leagues[key].tier,
+                    name: items.items_game.items[key].name
+                });
+            }
+            db('leagues').insert(arr).asCallback(cb);
+        });
+    });
+}, 10 * 60 * 1000);
 
 function invokeInterval(func, delay)
 {

From a628f581303fc556ad2bae23dbe22e86bcdd2e6b Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 01:06:27 +0000
Subject: [PATCH 20/49] add leagues endpoint

---
 routes/api.js         | 11 +++++++++++
 sql/create_tables.sql |  4 ++--
 store/queries.js      |  8 +++++---
 svc/worker.js         | 33 ++++++++++++++++++++-------------
 4 files changed, 38 insertions(+), 18 deletions(-)

diff --git a/routes/api.js b/routes/api.js
index 475d2eb85..d13e75341 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -145,6 +145,17 @@ module.exports = function(db, redis, cassandra)
             res.json(result.rows);
         });
     });
+    api.get('/leagues', function(req, res, cb)
+    {
+        db.raw(`SELECT * FROM leagues ORDER BY leagueid`).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
     api.get('/distributions', function(req, res, cb)
     {
         queries.getDistributions(redis, function(err, result)
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 8d1564d77..7eaa7571a 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -227,8 +227,8 @@ CREATE TABLE picks_bans(
   is_pick boolean,
   hero_id int,
   team smallint,
-  order smallint,
-  PRIMARY KEY (match_id, order)
+  ord smallint,
+  PRIMARY KEY (match_id, ord)
 );
 
 CREATE TABLE leagues(
diff --git a/store/queries.js b/store/queries.js
index f082aac51..50544476f 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -209,9 +209,11 @@ function insertMatch(db, redis, match, options, cb)
 
     function isProMatch(cb)
     {
-        //TODO check redis/postgres for professional/premium league
-        match.isProMatch = match.leagueid >= 0;
-        cb();
+        redis.sismember('pro_leagueids', match.league_id, function(err, result)
+        {
+            match.isProMatch = Boolean(Number(result));
+            cb(err);
+        });
     }
 
     function upsertMatch(cb)
diff --git a/svc/worker.js b/svc/worker.js
index ee1cd62f4..4e3df510e 100644
--- a/svc/worker.js
+++ b/svc/worker.js
@@ -126,31 +126,38 @@ invokeInterval(function notablePlayers(cb)
 }, 10 * 60 * 1000);
 invokeInterval(function leagues(cb)
 {
-    utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues)
+    var container = utility.generateJob("api_leagues",
+    {});
+    utility.getData(container.url, function(err, api_leagues)
     {
         if (err)
         {
             return cb(err);
         }
-        utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/items_game.json', function(err, items)
+        utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues)
         {
             if (err)
             {
                 return cb(err);
             }
-            var arr = [];
-            for (var key in leagues)
+            async.each(api_leagues.result.leagues, function(l, cb)
             {
-                arr.push(
+                if (leagues[l.leagueid])
                 {
-                    leagueid: key,
-                    ticket: leagues[key].ticket,
-                    banner: leagues[key].banner,
-                    tier: leagues[key].tier,
-                    name: items.items_game.items[key].name
-                });
-            }
-            db('leagues').insert(arr).asCallback(cb);
+                    l.tier = leagues[l.leagueid].tier;
+                    l.ticket = leagues[l.leagueid].ticket;
+                    l.banner = leagues[l.leagueid].banner;
+                }
+                l.name = l.description.substring("#DOTA_Item_Desc_".length).split('_').join(' ');
+                if (l.tier === "professional" || l.tier === "premium")
+                {
+                    redis.sadd('pro_leagueids', l.leagueid);
+                }
+                queries.upsert(db, 'leagues', l,
+                {
+                    leagueid: l.league_id
+                }, cb);
+            }, cb);
         });
     });
 }, 10 * 60 * 1000);

From 8b77717f2b0c7caa2fbc2723d93bec4af56df9df Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 02:55:24 +0000
Subject: [PATCH 21/49] fix leagueid prop name

---
 config.js        |  1 -
 store/queries.js | 15 +++++++++++----
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/config.js b/config.js
index 0ee61c505..beccd9af9 100644
--- a/config.js
+++ b/config.js
@@ -54,7 +54,6 @@ var defaults = {
     "UI_HOST": "", //The host of the UI, redirect traffic from / and /return here
     "ENABLE_RECAPTCHA": "", //set to enable the recaptcha on the Request page
     "ENABLE_ADS": "", //set to turn on ads
-    "ENABLE_PRO_PARSING": "", // set to parse pro matches from sequential API
     "ENABLE_MATCH_CACHE": "", // set to enable caching matches (Redis)
     "ENABLE_PLAYER_CACHE": "", // set to enable caching players (Cassandra)
     "ENABLE_INSERT_ALL_MATCHES": "", //set to enable inserting all matches
diff --git a/store/queries.js b/store/queries.js
index 50544476f..189535a2b 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -209,11 +209,18 @@ function insertMatch(db, redis, match, options, cb)
 
     function isProMatch(cb)
     {
-        redis.sismember('pro_leagueids', match.league_id, function(err, result)
+        if (match.leagueid)
         {
-            match.isProMatch = Boolean(Number(result));
-            cb(err);
-        });
+            redis.sismember('pro_leagueids', match.leagueid, function(err, result)
+            {
+                match.isProMatch = Boolean(Number(result));
+                cb(err);
+            });
+        }
+        else
+        {
+            cb();
+        }
     }
 
     function upsertMatch(cb)

From b1ee5ea3897e3ab1c55aea49b27cdbaef1ed7298 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 03:03:37 +0000
Subject: [PATCH 22/49] transact match log insert

---
 store/queries.js | 75 +++++++++++++++++++-----------------------------
 1 file changed, 29 insertions(+), 46 deletions(-)

diff --git a/store/queries.js b/store/queries.js
index 189535a2b..0df215242 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -194,7 +194,6 @@ function insertMatch(db, redis, match, options, cb)
         "ipm": isProMatch,
         "u": upsertMatch,
         "uc": upsertMatchCassandra,
-        "uml": upsertMatchLogs,
         "upc": updatePlayerCaches,
         "uct": updateCounts,
         "cmc": clearMatchCache,
@@ -233,12 +232,13 @@ function insertMatch(db, redis, match, options, cb)
         {
             async.series(
             {
-                "im": insertMatch,
-                "ipm": insertPlayerMatches,
-                "ipb": insertPicksBans
+                "m": upsertMatch,
+                "pm": upsertPlayerMatches,
+                "pb": upsertPicksBans,
+                "l": upsertMatchLogs,
             }, exit);
 
-            function insertMatch(cb)
+            function upsertMatch(cb)
             {
                 upsert(trx, 'matches', match,
                 {
@@ -246,7 +246,7 @@ function insertMatch(db, redis, match, options, cb)
                 }, cb);
             }
 
-            function insertPlayerMatches(cb)
+            function upsertPlayerMatches(cb)
             {
                 async.each(players || [], function(pm, cb)
                 {
@@ -259,7 +259,7 @@ function insertMatch(db, redis, match, options, cb)
                 }, cb);
             }
 
-            function insertPicksBans(cb)
+            function upsertPicksBans(cb)
             {
                 async.each(match.picks_bans || [], function(p, cb)
                 {
@@ -274,6 +274,28 @@ function insertMatch(db, redis, match, options, cb)
                 }, cb);
             }
 
+            function upsertMatchLogs(cb)
+            {
+                if (!match.logs)
+                {
+                    return cb();
+                }
+                else
+                {
+                    trx.raw(`DELETE FROM match_logs WHERE match_id = ?`, [match.match_id]).asCallback(function(err)
+                    {
+                        if (err)
+                        {
+                            return exit(err);
+                        }
+                        async.eachLimit(match.logs, 100, function(e, cb)
+                        {
+                            trx('match_logs').insert(e).asCallback(cb);
+                        }, cb);
+                    });
+                }
+            }
+
             function exit(err)
             {
                 if (err)
@@ -351,45 +373,6 @@ function insertMatch(db, redis, match, options, cb)
         });
     }
 
-    function upsertMatchLogs(cb)
-    {
-        if (!match.logs)
-        {
-            return cb();
-        }
-        else
-        {
-            db.transaction(function(trx)
-            {
-                trx.raw(`DELETE FROM match_logs where match_id = ?`, [match.match_id]).asCallback(function(err)
-                {
-                    if (err)
-                    {
-                        return exit(err);
-                    }
-                    async.eachLimit(match.logs, 10000, function(e, cb)
-                    {
-                        trx('match_logs').insert(e).asCallback(cb);
-                    }, exit);
-                });
-
-                function exit(err)
-                {
-                    if (err)
-                    {
-                        console.error(err);
-                        trx.rollback(err);
-                    }
-                    else
-                    {
-                        trx.commit();
-                    }
-                    cb(err);
-                }
-            });
-        }
-    }
-
     function updatePlayerCaches(cb)
     {
         var copy = createMatchCopy(match, players, options);

From 48b06f12f9e3b0f8f3703d476f428cc3bcfb0587 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 22:03:50 +0000
Subject: [PATCH 23/49] remove parse_status

---
 store/queries.js       | 15 ++++++++-------
 svc/parser.js          |  3 +--
 svc/requests.js        |  2 +-
 svc/scanner.js         | 12 +++++++-----
 tasks/getMatches.js    |  2 +-
 views/match/match.jade |  1 -
 6 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/store/queries.js b/store/queries.js
index 0df215242..768a9af2f 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -191,7 +191,7 @@ function insertMatch(db, redis, match, options, cb)
     //we want to insert into matches, then insert into player_matches for each entry in players
     async.series(
     {
-        "ipm": isProMatch,
+        "dlp": decideLogParse,
         "u": upsertMatch,
         "uc": upsertMatchCassandra,
         "upc": updatePlayerCaches,
@@ -205,14 +205,14 @@ function insertMatch(db, redis, match, options, cb)
     {
         return cb(err, results.dp);
     });
-
-    function isProMatch(cb)
+    
+    function decideLogParse(cb)
     {
         if (match.leagueid)
         {
             redis.sismember('pro_leagueids', match.leagueid, function(err, result)
             {
-                match.isProMatch = Boolean(Number(result));
+                options.doLogParse = options.doLogParse || Boolean(Number(result));
                 cb(err);
             });
         }
@@ -224,7 +224,7 @@ function insertMatch(db, redis, match, options, cb)
 
     function upsertMatch(cb)
     {
-        if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.isProMatch)
+        if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !options.doLogParse)
         {
             return cb();
         }
@@ -458,9 +458,10 @@ function insertMatch(db, redis, match, options, cb)
         }, cb);
     }
 
+
     function decideParse(cb)
     {
-        if (match.parse_status !== 0)
+        if (options.skipParse)
         {
             //not parsing this match
             //this isn't a error, although we want to report that we refused to parse back to user if it was a request
@@ -477,7 +478,7 @@ function insertMatch(db, redis, match, options, cb)
                 duration: match.duration,
                 replay_blob_key: match.replay_blob_key,
                 pgroup: match.pgroup,
-                isProMatch: match.isProMatch,
+                doLogParse: options.doLogParse,
             },
             {
                 lifo: options.lifo,
diff --git a/svc/parser.js b/svc/parser.js
index 1228af883..586c8773e 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -91,7 +91,6 @@ pQueue.process(config.PARSER_PARALLELISM, function(job, cb)
                 parsed_data.start_time = match.start_time;
                 parsed_data.duration = match.duration;
                 parsed_data.replay_blob_key = match.replay_blob_key;
-                parsed_data.parse_status = 2;
                 if (match.replay_blob_key)
                 {
                     insertUploadedParse(parsed_data, cb);
@@ -267,7 +266,7 @@ function runParse(match, job, cb)
                 var message = "time spent on post-processing match ";
                 console.time(message);
                 var meta = processMetadata(entries);
-                var logs = match.isProMatch ? processReduce(entries, match, meta) : undefined;
+                var logs = match.doLogParse ? processReduce(entries, match, meta) : undefined;
                 var res = processExpand(entries, meta);
                 var parsed_data = processParsedData(res.parsed_data);
                 var teamfights = processTeamfights(res.tf_data, meta);
diff --git a/svc/requests.js b/svc/requests.js
index c6007a733..08901defb 100644
--- a/svc/requests.js
+++ b/svc/requests.js
@@ -29,13 +29,13 @@ function processRequest(job, cb)
             }
             //match details response
             var match = body.result;
-            match.parse_status = 0;
             insertMatch(db, redis, match,
             {
                 type: "api",
                 attempts: 1,
                 lifo: true,
                 cassandra: cassandra,
+                skipParse: false,
             }, waitParse);
         });
     }
diff --git a/svc/scanner.js b/svc/scanner.js
index fa15a8f35..ee8d86348 100644
--- a/svc/scanner.js
+++ b/svc/scanner.js
@@ -112,21 +112,22 @@ function start()
 
             function processMatch(match, cb)
             {
+                var insert = false;
+                var skipParse = true;
                 if (match.players.some(function(p)
                     {
                         return (p.account_id in trackedPlayers);
                     }))
                 {
-                    //queued
-                    match.parse_status = 0;
+                    insert = true;
+                    skipParse = false;
                 }
                 else if (match.players.some(function(p)
                     {
                         return (config.ENABLE_INSERT_ALL_MATCHES || p.account_id in userPlayers);
                     }))
                 {
-                    //skipped
-                    match.parse_status = 3;
+                    insert = true;
                 }
                 //check if match was previously processed
                 redis.get('scanner_insert:' + match.match_id, function(err, result)
@@ -137,7 +138,7 @@ function start()
                     }
                     //don't insert this match if we already processed it recently
                     //deduplicate matches in this page set
-                    if ((match.parse_status === 0 || match.parse_status === 3) && !result && !matchBuffer[match.match_id])
+                    if (insert && !result && !matchBuffer[match.match_id])
                     {
                         matchBuffer[match.match_id] = 1;
                         insertMatch(db, redis, match,
@@ -146,6 +147,7 @@ function start()
                             origin: "scanner",
                             cassandra: cassandra,
                             userPlayers: userPlayers,
+                            skipParse: skipParse,
                         }, function(err)
                         {
                             if (!err)
diff --git a/tasks/getMatches.js b/tasks/getMatches.js
index cf2d7fd92..0362a1b95 100644
--- a/tasks/getMatches.js
+++ b/tasks/getMatches.js
@@ -28,11 +28,11 @@ getData(
     if (body.result)
     {
         var match = body.result;
-        match.parse_status = 0;
         insertMatch(db, redis, match,
         {
             skipCounts: true,
             skipAbilityUpgrades: true,
+            skipParse: false,
             cassandra: cassandra,
         }, function(err)
         {
diff --git a/views/match/match.jade b/views/match/match.jade
index 53673b8d2..6d0a345d7 100644
--- a/views/match/match.jade
+++ b/views/match/match.jade
@@ -32,7 +32,6 @@ block content
             th Percentile
         tbody
           tr
-            //td= constants.parse_status[match.parse_status]
             td= constants.game_mode[match.game_mode] ? constants.game_mode[match.game_mode].name : match.game_mode
             td= constants.region[match.region] ? constants.region[match.region] : match.region
             td= moment().startOf('day').seconds(match.duration).format("H:mm:ss")

From b8ce7b34b4502439fe6ea528a8edc96113fd72a6 Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 15:05:54 -0700
Subject: [PATCH 24/49] shorten mmr estimate to 20 matches

---
 svc/cacher.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/svc/cacher.js b/svc/cacher.js
index ffb8e45d9..b5475f0ac 100644
--- a/svc/cacher.js
+++ b/svc/cacher.js
@@ -248,7 +248,7 @@ function updateMatchRating(match, cb)
                 {
                     //push into list, limit elements
                     redis.lpush('mmr_estimates:' + player.account_id, avg);
-                    redis.ltrim('mmr_estimates:' + player.account_id, 0, 24);
+                    redis.ltrim('mmr_estimates:' + player.account_id, 0, 19);
                 }
             });
             cb();
@@ -258,4 +258,4 @@ function updateMatchRating(match, cb)
             return cb(err);
         }
     });
-}
\ No newline at end of file
+}

From a95f2c3bf06b34035e32dd47a69eb4cebb032c1d Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 22:17:43 +0000
Subject: [PATCH 25/49] add endpoint to list pro matches

---
 routes/api.js         | 20 +++++++++++++++++++-
 sql/create_tables.sql |  1 +
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/routes/api.js b/routes/api.js
index d13e75341..4a3311210 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -136,7 +136,25 @@ module.exports = function(db, redis, cassandra)
     */
     api.get('/match_logs/:match_id', function(req, res, cb)
     {
-        db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result)
+        db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time ASC`, [req.params.match_id]).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
+    api.get('/pro_matches', function(req, res, cb)
+    {
+        db.raw(`
+        SELECT match_id, start_time, duration, ma.leagueid, name
+        FROM matches ma
+        JOIN leagues le
+        ON ma.leagueid = le.leagueid
+        WHERE ma.leagueid > 0
+        ORDER BY match_id DESC
+        `).asCallback(function(err, result)
         {
             if (err)
             {
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 7eaa7571a..936d49041 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -39,6 +39,7 @@ CREATE TABLE matches (
   teamfights json[],
   version integer
   );
+CREATE INDEX on matches(leagueid) WHERE leagueid > 0;
 
 CREATE TABLE player_matches (
   PRIMARY KEY(match_id, player_slot),

From 77b4b90b89c029f6621fffce1af610cd66ed9d66 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 22:23:51 +0000
Subject: [PATCH 26/49] fix double parse

---
 svc/parser.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/svc/parser.js b/svc/parser.js
index 586c8773e..ac5aa4864 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -157,6 +157,7 @@ function insertStandardParse(match, cb)
     {
         type: "parsed",
         cassandra: cassandra,
+        skipParse: true,
     }, cb);
 }
 

From a46b6b8419988a2fd0a021605e33ae13f76d137e Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 22:50:27 +0000
Subject: [PATCH 27/49] fix insert call in test

---
 svc/fullhistory.js     | 1 +
 tasks/getMatchesSeq.js | 1 +
 test/test.js           | 3 ++-
 3 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/svc/fullhistory.js b/svc/fullhistory.js
index 312034a51..d5707ac4a 100644
--- a/svc/fullhistory.js
+++ b/svc/fullhistory.js
@@ -90,6 +90,7 @@ function processFullHistory(job, cb)
                         type: "api",
                         cassandra: cassandra,
                         skipAbilityUpgrades: true,
+                        skipParse: true,
                     }, cb);
                 });
             }, function(err)
diff --git a/tasks/getMatchesSeq.js b/tasks/getMatchesSeq.js
index ebd445e36..331c5bb1c 100644
--- a/tasks/getMatchesSeq.js
+++ b/tasks/getMatchesSeq.js
@@ -86,6 +86,7 @@ function getPage(match_seq_num, bucket)
                 {
                     skipCounts: true,
                     skipAbilityUpgrades: true,
+                    skipParse: true,
                     cassandra: cassandra,
                 }, cb);
             }, function(err)
diff --git a/test/test.js b/test/test.js
index 73af32d1d..e6e4cebd7 100644
--- a/test/test.js
+++ b/test/test.js
@@ -128,7 +128,8 @@ before(function(done)
             {
                 queries.insertMatch(db, redis, m,
                 {
-                    type: "api"
+                    type: "api",
+                    skipParse: true,
                 }, cb);
             }, cb);
         },

From afa5e6167a65b8ac5a8596b2c52970d11b6f4a85 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Mon, 20 Jun 2016 00:54:20 +0000
Subject: [PATCH 28/49] remove userplayers

---
 store/buildSets.js | 14 --------------
 store/queries.js   |  9 +--------
 svc/scanner.js     |  8 +-------
 3 files changed, 2 insertions(+), 29 deletions(-)

diff --git a/store/buildSets.js b/store/buildSets.js
index fed99d429..2c3794d77 100644
--- a/store/buildSets.js
+++ b/store/buildSets.js
@@ -18,20 +18,6 @@ module.exports = function buildSets(db, redis, cb) {
                 cb(err, t);
             });
         },
-        //users in this set have their matches added
-        "userPlayers": function(cb) {
-            db.select(['account_id']).from('players').whereNotNull('last_login').asCallback(function(err, docs) {
-                if (err) {
-                    return cb(err);
-                }
-                var t = {};
-                docs.forEach(function(player) {
-                    t[player.account_id] = true;
-                });
-                //console.log(t);
-                cb(err, t);
-            });
-        },
         //users in this set are added to the trackedPlayers set
         "donators": function(cb) {
             db.select(['account_id']).from('players').where('cheese', '>', 0).asCallback(function(err, docs) {
diff --git a/store/queries.js b/store/queries.js
index 768a9af2f..49974199e 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -32,13 +32,6 @@ function getSets(redis, cb)
                 cb(err, JSON.parse(tps || "{}"));
             });
         },
-        "userPlayers": function(cb)
-        {
-            redis.get("userPlayers", function(err, ups)
-            {
-                cb(err, JSON.parse(ups || "{}"));
-            });
-        },
         "donators": function(cb)
         {
             redis.get("donators", function(err, ds)
@@ -426,7 +419,7 @@ function insertMatch(db, redis, match, options, cb)
     {
         async.each(match.players, function(p, cb)
         {
-            if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && (p.account_id in options.userPlayers || (config.ENABLE_RANDOM_MMR_UPDATE && match.match_id % 3 === 0)))
+            if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && config.ENABLE_RANDOM_MMR_UPDATE)
             {
                 addToQueue(mQueue,
                 {
diff --git a/svc/scanner.js b/svc/scanner.js
index ee8d86348..819db762d 100644
--- a/svc/scanner.js
+++ b/svc/scanner.js
@@ -13,7 +13,6 @@ var getData = utility.getData;
 var generateJob = utility.generateJob;
 var async = require('async');
 var trackedPlayers;
-var userPlayers;
 var parallelism = config.SCANNER_PARALLELISM;
 var PAGE_SIZE = 100;
 buildSets(db, redis, function(err)
@@ -72,7 +71,6 @@ function start()
             }
             //set local vars
             trackedPlayers = result.trackedPlayers;
-            userPlayers = result.userPlayers;
             var arr = [];
             var matchBuffer = {};
             var completePages = {};
@@ -122,10 +120,7 @@ function start()
                     insert = true;
                     skipParse = false;
                 }
-                else if (match.players.some(function(p)
-                    {
-                        return (config.ENABLE_INSERT_ALL_MATCHES || p.account_id in userPlayers);
-                    }))
+                else if (config.ENABLE_INSERT_ALL_MATCHES)
                 {
                     insert = true;
                 }
@@ -146,7 +141,6 @@ function start()
                             type: "api",
                             origin: "scanner",
                             cassandra: cassandra,
-                            userPlayers: userPlayers,
                             skipParse: skipParse,
                         }, function(err)
                         {

From 41b115c7a3eb3dcc07cd193641e866e6eaa7ff44 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Mon, 20 Jun 2016 01:19:10 +0000
Subject: [PATCH 29/49] update knex, remove modifier_remove events from log

---
 package.json                | 2 +-
 processors/processReduce.js | 4 ++++
 store/queries.js            | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index b88d0ee1f..011261da3 100644
--- a/package.json
+++ b/package.json
@@ -55,7 +55,7 @@
     "http-proxy": "^1.13.1",
     "jade": "^1.11.0",
     "jquery": "^2.2.0",
-    "knex": "^0.10.0",
+    "knex": "^0.11.7",
     "moment": "^2.11.2",
     "multer": "^1.1.0",
     "numeral": "^1.5.3",
diff --git a/processors/processReduce.js b/processors/processReduce.js
index 2bf9dbe00..3fd6823b2 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -9,6 +9,10 @@ function processReduce(entries, match, meta)
         {
             return false;
         }
+        if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE")
+        {
+            return false;
+        }
         if (e.type === "interval" && e.time % 60 !== 0)
         {
             return false;
diff --git a/store/queries.js b/store/queries.js
index 49974199e..05ecb64cd 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -279,9 +279,9 @@ function insertMatch(db, redis, match, options, cb)
                     {
                         if (err)
                         {
-                            return exit(err);
+                            return cb(err);
                         }
-                        async.eachLimit(match.logs, 100, function(e, cb)
+                        async.eachLimit(match.logs, 10000, function(e, cb)
                         {
                             trx('match_logs').insert(e).asCallback(cb);
                         }, cb);

From 38f998be18e1368c135e5e8207410d1e0e29b36f Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Sun, 19 Jun 2016 21:23:17 -0700
Subject: [PATCH 30/49] Update web.js

---
 svc/web.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/svc/web.js b/svc/web.js
index 13ebd837e..c2a23f091 100644
--- a/svc/web.js
+++ b/svc/web.js
@@ -80,6 +80,7 @@ app.locals.getAggs = utility.getAggs;
 app.use(compression());
 app.use("/apps/dota2/images/:group_name/:image_name", function(req, res)
 {
+    res.header('Cache-Control', 'max-age=604800, public');
     request("http://cdn.dota2.com/apps/dota2/images/" + req.params.group_name + "/" + req.params.image_name).pipe(res);
 });
 app.use("/public", express.static(path.join(__dirname, '/../public')));

From e789d57954b832656ec2401b1dbabe1c4894826c Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Mon, 20 Jun 2016 00:58:10 -0700
Subject: [PATCH 31/49] Update cacher.js

---
 svc/cacher.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/svc/cacher.js b/svc/cacher.js
index b5475f0ac..08844f0fd 100644
--- a/svc/cacher.js
+++ b/svc/cacher.js
@@ -248,7 +248,7 @@ function updateMatchRating(match, cb)
                 {
                     //push into list, limit elements
                     redis.lpush('mmr_estimates:' + player.account_id, avg);
-                    redis.ltrim('mmr_estimates:' + player.account_id, 0, 19);
+                    redis.ltrim('mmr_estimates:' + player.account_id, 0, 14);
                 }
             });
             cb();

From c23bec81fefa54249c861c67589201d5da6309db Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Mon, 20 Jun 2016 01:55:51 -0700
Subject: [PATCH 32/49] refactor types array in queue.js

---
 store/queue.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/store/queue.js b/store/queue.js
index 1909fcaca..1598f73e4 100644
--- a/store/queue.js
+++ b/store/queue.js
@@ -6,6 +6,7 @@ var config = require('../config');
 var bull = require('bull');
 var url = require('url');
 var async = require('async');
+var types = ["request", "mmr", "parse", "cache", "fullhistory"];
 // parse the url
 var conn_info = url.parse(config.REDIS_URL, true /* parse query string */ );
 if (conn_info.protocol !== 'redis:')
@@ -50,7 +51,6 @@ function addToQueue(queue, payload, options, cb)
 
 function getCounts(redis, cb)
 {
-    var types = ["request", "mmr", "parse", "cache"];
     async.map(types, getQueueCounts, function(err, result)
     {
         var obj = {};
@@ -91,7 +91,6 @@ function getCounts(redis, cb)
 
 function cleanup(redis, cb)
 {
-    var types = ["request", "mmr", "parse", "cache"];
     async.each(types, function(key, cb)
     {
         var queue = getQueue(key);
@@ -111,4 +110,4 @@ module.exports = {
     addToQueue: addToQueue,
     getCounts: getCounts,
     cleanup: cleanup
-};
\ No newline at end of file
+};

From e3cfeac44948108b70503739cf4eb272a1b35e6f Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Mon, 20 Jun 2016 03:25:39 -0700
Subject: [PATCH 33/49] Update worker.js

---
 svc/worker.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/svc/worker.js b/svc/worker.js
index 4c492994e..a064db05e 100644
--- a/svc/worker.js
+++ b/svc/worker.js
@@ -60,7 +60,7 @@ invokeInterval(function buildDistributions(cb)
                 });
                 results.rows = results.rows.map(function(r, i)
                 {
-                    r.cumulative_sum = results.rows.slice(0, i).reduce(function(prev, current)
+                    r.cumulative_sum = results.rows.slice(0, i + 1).reduce(function(prev, current)
                     {
                         return {
                             count: prev.count + current.count

From b60f4b48bb37dd3e5f692b3f3307f62dd28b23da Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Tue, 21 Jun 2016 23:40:44 -0700
Subject: [PATCH 34/49] add slowlog to find slow web reqs

---
 svc/web.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/svc/web.js b/svc/web.js
index c2a23f091..5a4a490b2 100644
--- a/svc/web.js
+++ b/svc/web.js
@@ -136,13 +136,18 @@ app.use(function telemetry(req, res, cb)
     res.once('finish', function()
     {
         var timeEnd = new Date();
+        var elapsed = timeEnd - timeStart;
+        if (elapsed > 1000)
+        {
+            console.log("[SLOWLOG] %s, %s", req.path, elapsed);
+        }
         /*
         var obj = JSON.stringify({
             path: req.path,
             time: timeEnd - timeStart
         };
         */
-        redis.lpush("load_times", timeEnd - timeStart);
+        redis.lpush("load_times", elapsed);
         redis.ltrim("load_times", 0, 9999);
     });
     cb();

From 0165e1e4fa563a05e7bf41c12c4e3dadc31ad789 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Wed, 22 Jun 2016 08:10:10 +0000
Subject: [PATCH 35/49] remove non-hero logs

---
 processors/processReduce.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/processors/processReduce.js b/processors/processReduce.js
index 3fd6823b2..434c032e9 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -13,6 +13,13 @@ function processReduce(entries, match, meta)
         {
             return false;
         }
+        if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL")
+        {
+            if (!e.targethero || e.targetillusion)
+            {
+                return false;
+            }
+        }
         if (e.type === "interval" && e.time % 60 !== 0)
         {
             return false;

From 74f2640382a9fb07f67c82d674bd8863eda7dbde Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Wed, 22 Jun 2016 09:07:43 +0000
Subject: [PATCH 36/49] extract ward entityLeft events

---
 java_parser/src/main/java/yasp/Main.java | 66 +++++++++++++++---------
 processors/populate.js                   |  3 ++
 processors/processExpand.js              | 24 +++++----
 sql/cassandra.cql                        |  1 +
 sql/create_tables.sql                    |  3 +-
 util/analysis.js                         | 20 +++++--
 util/utility.js                          |  1 +
 views/match/match_performances.jade      |  4 --
 views/match/match_vision.jade            |  4 +-
 9 files changed, 84 insertions(+), 42 deletions(-)

diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java
index f507f938f..b4ad1c54d 100644
--- a/java_parser/src/main/java/yasp/Main.java
+++ b/java_parser/src/main/java/yasp/Main.java
@@ -10,6 +10,7 @@
 import skadistats.clarity.model.s1.GameRulesStateType;
 import skadistats.clarity.processor.entities.Entities;
 import skadistats.clarity.processor.entities.OnEntityEntered;
+import skadistats.clarity.processor.entities.OnEntityLeft;
 import skadistats.clarity.processor.entities.UsesEntities;
 import skadistats.clarity.processor.gameevents.CombatLog;
 import skadistats.clarity.processor.gameevents.OnCombatLogEntry;
@@ -78,6 +79,7 @@ private class Entry {
         public Integer xp;
         public Integer x;
         public Integer y;
+        public Integer z;
         public Float stuns;
         public Integer hero_id;
         public Integer life_state;
@@ -87,7 +89,9 @@ private class Entry {
         public Integer assists;
         public Integer denies;
         //public Boolean hasPredictedVictory;
-    
+        public Boolean entityleft;
+        public Integer ehandle;
+        
         public Entry() {
         }
         
@@ -255,28 +259,12 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) {
 
     @OnEntityEntered
     public void onEntityEntered(Context ctx, Entity e) {
-        //CDOTA_NPC_Observer_Ward
-        //CDOTA_NPC_Observer_Ward_TrueSight
-        //s1 "DT_DOTA_NPC_Observer_Ward"
-        //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight"
-        boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward");
-        boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight");
-        if (isObserver || isSentry) {
-            //System.err.println(e);
-            Entry entry = new Entry(time);
-            Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null);
-            Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null);
-            Integer[] pos = {x, y};
-            entry.type = isObserver ? "obs" : "sen";
-            entry.key = Arrays.toString(pos);
-            //System.err.println(entry.key);
-            Integer owner = getEntityProperty(e, "m_hOwnerEntity", null);
-            Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner);
-            entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null;
-            //2/3 radiant/dire
-            //entry.team = e.getProperty("m_iTeamNum");
-            output(entry);
-        }
+        processWardEntity(ctx, e, false);
+    }
+    
+    @OnEntityLeft
+    public void onEntityLeft(Context ctx, Entity e) {
+        processWardEntity(ctx, e, true);
     }
 
     @UsesEntities
@@ -448,6 +436,38 @@ public <T> T getEntityProperty(Entity e, String property, Integer idx) {
         FieldPath fp = e.getDtClass().getFieldPathForName(property);
         return e.getPropertyForFieldPath(fp);
     }
+    
+    public void processWardEntity(Context ctx, Entity e, boolean entityLeft)
+    {
+        //CDOTA_NPC_Observer_Ward
+        //CDOTA_NPC_Observer_Ward_TrueSight
+        //s1 "DT_DOTA_NPC_Observer_Ward"
+        //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight"
+        boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward");
+        boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight");
+        if (isObserver || isSentry) {
+            //System.err.println(e);
+            Entry entry = new Entry(time);
+            Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null);
+            Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null);
+            Integer z = getEntityProperty(e, "CBodyComponent.m_cellZ", null);
+            Integer[] pos = {x, y};
+            entry.x = x;
+            entry.y = y;
+            entry.z = z;
+            entry.type = isObserver ? "obs" : "sen";
+            entry.key = Arrays.toString(pos);
+            entry.entityleft = entityLeft;
+            entry.ehandle = e.getHandle();
+            //System.err.println(entry.key);
+            Integer owner = getEntityProperty(e, "m_hOwnerEntity", null);
+            Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner);
+            entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null;
+            //2/3 radiant/dire
+            //entry.team = e.getProperty("m_iTeamNum");
+            output(entry);
+        }
+    }
 
     public void run(String[] args) throws Exception {
         long tStart = System.currentTimeMillis();
diff --git a/processors/populate.js b/processors/populate.js
index 39d61bc47..b1eda8df9 100644
--- a/processors/populate.js
+++ b/processors/populate.js
@@ -7,6 +7,9 @@ function populate(e, container)
         case 'player_slot':
             container.players[e.key].player_slot = e.value;
             break;
+        case 'log':
+            container.log.push(JSON.parse(JSON.stringify(e)));
+            break;
         case 'chat':
             container.chat.push(JSON.parse(JSON.stringify(e)));
             break;
diff --git a/processors/processExpand.js b/processors/processExpand.js
index c91c15ca3..ef10789b0 100644
--- a/processors/processExpand.js
+++ b/processors/processExpand.js
@@ -425,24 +425,30 @@ function processExpand(entries, meta)
         },
         "obs": function(e)
         {
-            //key is a JSON array of position data
-            e.key = JSON.parse(e.key);
-            e.posData = true;
-            expand(e);
             var e2 = JSON.parse(JSON.stringify(e));
-            e2.posData = false;
             e2.type = "obs_log";
             expand(e2);
+            var e4 = JSON.parse(JSON.stringify(e));
+            e4.type = "log";
+            expand(e4);
+            var e3 = JSON.parse(JSON.stringify(e));
+            //key is a JSON array of position data
+            e3.key = JSON.parse(e3.key);
+            e3.posData = true;
+            expand(e3);
         },
         "sen": function(e)
         {
-            e.key = JSON.parse(e.key);
-            e.posData = true;
-            expand(e);
             var e2 = JSON.parse(JSON.stringify(e));
-            e2.posData = false;
             e2.type = "sen_log";
             expand(e2);
+            var e4 = JSON.parse(JSON.stringify(e));
+            e4.type = "log";
+            expand(e4);
+            var e3 = JSON.parse(JSON.stringify(e));
+            e3.key = JSON.parse(e3.key);
+            e3.posData = true;
+            expand(e3);
         }
     };
     //define the types we want to put into each array
diff --git a/sql/cassandra.cql b/sql/cassandra.cql
index 4c1572049..6de8d3110 100644
--- a/sql/cassandra.cql
+++ b/sql/cassandra.cql
@@ -26,6 +26,7 @@ CREATE TABLE matches (
   radiant_xp_adv text,
   teamfights text,
   version int,
+  log text,
 );
 
 CREATE TABLE player_matches (
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 134bda3f7..8c23cdad4 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -37,7 +37,8 @@ CREATE TABLE matches (
   radiant_gold_adv integer[],
   radiant_xp_adv integer[],
   teamfights json[],
-  version integer
+  version integer,
+  log json[]
   );
 
 CREATE TABLE player_matches (
diff --git a/util/analysis.js b/util/analysis.js
index 8f9f2f163..6aca06a63 100644
--- a/util/analysis.js
+++ b/util/analysis.js
@@ -4,6 +4,7 @@
  **/
 var util = require('util');
 var constants = require('../constants.js');
+
 function generatePlayerAnalysis(match, pm)
 {
     //define condition check for each advice point
@@ -150,7 +151,8 @@ function generatePlayerAnalysis(match, pm)
         {
             var flying_available = 180;
             var time;
-            if (pm.purchase && pm.first_purchase_time && pm.first_purchase_time.flying_courier) {
+            if (pm.purchase && pm.first_purchase_time && pm.first_purchase_time.flying_courier)
+            {
                 time = pm.first_purchase_time.flying_courier;
             }
             return {
@@ -172,7 +174,7 @@ function generatePlayerAnalysis(match, pm)
         wards: function(m, pm)
         {
             var ward_cooldown = 60 * 7;
-            var wards = pm.obs_log ? pm.obs_log.length : 0;
+            var wards = getObsWardsPlaced(pm);
             //divide game length by ward cooldown
             //2 wards respawn every interval
             //split responsibility between 2 supports
@@ -304,7 +306,19 @@ function generatePlayerAnalysis(match, pm)
 
     function isSupport(pm)
     {
-        return pm.obs_log && pm.obs_log.length >= 2 && pm.lh_t && pm.lh_t[10] < 20;
+        return getObsWardsPlaced(pm) >= 2 && pm.lh_t && pm.lh_t[10] < 20;
+    }
+
+    function getObsWardsPlaced(pm)
+    {
+        if (!pm.obs_log)
+        {
+            return 0;
+        }
+        return pm.obs_log.filter(function(l)
+        {
+            return !l.entityleft;
+        }).length;
     }
 
     function isRoshHero(pm)
diff --git a/util/utility.js b/util/utility.js
index ba4d11fb1..3d436b320 100644
--- a/util/utility.js
+++ b/util/utility.js
@@ -357,6 +357,7 @@ function getParseSchema()
         "teamfights": [],
         "objectives": [],
         "chat": [],
+        "log": [],
         "radiant_gold_adv": [],
         "radiant_xp_adv": [],
         "players": Array.apply(null, new Array(10)).map(function()
diff --git a/views/match/match_performances.jade b/views/match/match_performances.jade
index b78029092..2edb420df 100644
--- a/views/match/match_performances.jade
+++ b/views/match/match_performances.jade
@@ -9,8 +9,6 @@ block match_content
             tr
               th: abbr(title=tooltips.hero_id) Hero
               th: abbr(title=tooltips.lane) Lane
-              //th: abbr(title=tooltips.used_ward_observer) Observer
-              //th: abbr(title=tooltips.used_ward_sentry) Sentry
               th: abbr(title=tooltips.lane_efficiency) EFF@10
               th: abbr(title=tooltips.lhten) LH@10
               th: abbr(title=tooltips.stuns) Stuns
@@ -19,8 +17,6 @@ block match_content
           tr.activate(data-index=i, data-type="lane_pos", class = player.isRadiant ? "radiant" : "dire")
             +hero_td(player)
             td #{constants.lane_ids[player.lane] || "-"} (#{constants.lane_role[player.lane_role] || "-"})
-            //td.activate(data-index=i, data-type="obs") #{player.obs_log.length || "-"}
-            //td.activate(data-index=i, data-type="sen") #{player.sen_log.length || "-"}
             td.rankable #{(Number(player.lane_efficiency)*100).toFixed(2)}
             td.rankable #{player.lh_t && player.lh_t[10] ? player.lh_t[10] : "-"}
             td.rankable #{Number(Math.max(player.stuns, 0)).toFixed(2)}
diff --git a/views/match/match_vision.jade b/views/match/match_vision.jade
index 34d66137a..fb9e9a569 100644
--- a/views/match/match_vision.jade
+++ b/views/match/match_vision.jade
@@ -14,7 +14,7 @@ block match_content
           tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="obs").activate
             +hero_td(player)
             td.rankable.lane-position #{player.purchase_ward_observer || "-"}
-            td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.length : "-"}
+            td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.filter(function(l){return !l.entityleft;}).length : "-"}
             td.rankable.lane-position #{player.observer_kills || "-"}
     
     .col-md-4
@@ -29,7 +29,7 @@ block match_content
           tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="sen").activate
             +hero_td(player)
             td.rankable.lane-position #{player.purchase_ward_sentry || "-"}
-            td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.length : "-"}
+            td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.filter(function(l){return !l.entityleft;}).length : "-"}
             td.rankable.lane-position #{player.sentry_kills || "-"}
         
     .col-md-4

From 254cd9947b34840f694658f28e6f8b52a2a2c4cd Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 23 Jun 2016 03:22:36 +0000
Subject: [PATCH 37/49] reuse log

---
 java_parser/src/main/java/yasp/Main.java | 9 ++++++++-
 processors/populate.js                   | 4 ----
 processors/processExpand.js              | 8 +-------
 sql/cassandra.cql                        | 3 +--
 sql/create_tables.sql                    | 3 +--
 util/utility.js                          | 1 -
 6 files changed, 11 insertions(+), 17 deletions(-)

diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java
index b4ad1c54d..2c520f46f 100644
--- a/java_parser/src/main/java/yasp/Main.java
+++ b/java_parser/src/main/java/yasp/Main.java
@@ -455,7 +455,14 @@ public void processWardEntity(Context ctx, Entity e, boolean entityLeft)
             entry.x = x;
             entry.y = y;
             entry.z = z;
-            entry.type = isObserver ? "obs" : "sen";
+            if (entityLeft)
+            {
+                entry.type = isObserver ? "obs_left" : "sen_left";
+            }
+            else
+            {
+                entry.type = isObserver ? "obs" : "sen";
+            }
             entry.key = Arrays.toString(pos);
             entry.entityleft = entityLeft;
             entry.ehandle = e.getHandle();
diff --git a/processors/populate.js b/processors/populate.js
index b1eda8df9..35581c816 100644
--- a/processors/populate.js
+++ b/processors/populate.js
@@ -7,9 +7,6 @@ function populate(e, container)
         case 'player_slot':
             container.players[e.key].player_slot = e.value;
             break;
-        case 'log':
-            container.log.push(JSON.parse(JSON.stringify(e)));
-            break;
         case 'chat':
             container.chat.push(JSON.parse(JSON.stringify(e)));
             break;
@@ -92,5 +89,4 @@ function populate(e, container)
             break;
     }
 }
-
 module.exports = populate;
\ No newline at end of file
diff --git a/processors/processExpand.js b/processors/processExpand.js
index ef10789b0..f8395d753 100644
--- a/processors/processExpand.js
+++ b/processors/processExpand.js
@@ -428,9 +428,6 @@ function processExpand(entries, meta)
             var e2 = JSON.parse(JSON.stringify(e));
             e2.type = "obs_log";
             expand(e2);
-            var e4 = JSON.parse(JSON.stringify(e));
-            e4.type = "log";
-            expand(e4);
             var e3 = JSON.parse(JSON.stringify(e));
             //key is a JSON array of position data
             e3.key = JSON.parse(e3.key);
@@ -442,14 +439,11 @@ function processExpand(entries, meta)
             var e2 = JSON.parse(JSON.stringify(e));
             e2.type = "sen_log";
             expand(e2);
-            var e4 = JSON.parse(JSON.stringify(e));
-            e4.type = "log";
-            expand(e4);
             var e3 = JSON.parse(JSON.stringify(e));
             e3.key = JSON.parse(e3.key);
             e3.posData = true;
             expand(e3);
-        }
+        },
     };
     //define the types we want to put into each array
     //null means all types
diff --git a/sql/cassandra.cql b/sql/cassandra.cql
index 6de8d3110..5b075cbf6 100644
--- a/sql/cassandra.cql
+++ b/sql/cassandra.cql
@@ -25,8 +25,7 @@ CREATE TABLE matches (
   radiant_gold_adv text,
   radiant_xp_adv text,
   teamfights text,
-  version int,
-  log text,
+  version int
 );
 
 CREATE TABLE player_matches (
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 8c23cdad4..134bda3f7 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -37,8 +37,7 @@ CREATE TABLE matches (
   radiant_gold_adv integer[],
   radiant_xp_adv integer[],
   teamfights json[],
-  version integer,
-  log json[]
+  version integer
   );
 
 CREATE TABLE player_matches (
diff --git a/util/utility.js b/util/utility.js
index 3d436b320..ba4d11fb1 100644
--- a/util/utility.js
+++ b/util/utility.js
@@ -357,7 +357,6 @@ function getParseSchema()
         "teamfights": [],
         "objectives": [],
         "chat": [],
-        "log": [],
         "radiant_gold_adv": [],
         "radiant_xp_adv": [],
         "players": Array.apply(null, new Array(10)).map(function()

From e1eeb51e31fd1fe7fc2bab75d731c6693b6f330f Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 23 Jun 2016 03:32:36 +0000
Subject: [PATCH 38/49] add new columns to log schema

---
 processors/processReduce.js | 47 +++++++++++++++++++++++--------------
 sql/create_tables.sql       |  3 +++
 svc/parser.js               |  4 ++--
 3 files changed, 35 insertions(+), 19 deletions(-)

diff --git a/processors/processReduce.js b/processors/processReduce.js
index 434c032e9..d0865c998 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -3,32 +3,45 @@
  **/
 function processReduce(entries, match, meta)
 {
+    var basicLogTypes = {
+        "obs": 1,
+        "sen": 1,
+        "obs_left": 1,
+        "sen_left": 1,
+    };
     var result = entries.filter(function(e)
     {
-        if (e.type === "actions")
+        if (match.doLogParse)
         {
-            return false;
-        }
-        if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE")
-        {
-            return false;
-        }
-        if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL")
-        {
-            if (!e.targethero || e.targetillusion)
+            if (e.type === "actions")
             {
                 return false;
             }
+            if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE")
+            {
+                return false;
+            }
+            if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL")
+            {
+                if (!e.targethero || e.targetillusion)
+                {
+                    return false;
+                }
+            }
+            if (e.type === "interval" && e.time % 60 !== 0)
+            {
+                return false;
+            }
+            if (!e.time)
+            {
+                return false;
+            }
+            return true;
         }
-        if (e.type === "interval" && e.time % 60 !== 0)
-        {
-            return false;
-        }
-        if (!e.time)
+        else
         {
-            return false;
+            return (e.type in basicLogTypes);
         }
-        return true;
     }).map(function(e)
     {
         var e2 = Object.assign(
diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 936d49041..2588c01b6 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -201,6 +201,9 @@ CREATE TABLE match_logs (
   xp int,
   x smallint,
   y smallint,
+  z smallint,
+  entityleft boolean,
+  ehandle int,
   stuns real,
   hero_id smallint,
   life_state smallint,
diff --git a/svc/parser.js b/svc/parser.js
index ac5aa4864..906fa5069 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -245,7 +245,7 @@ function runParse(match, job, cb)
         }
         entries.push(e);
     });
-    request.debug = true
+    request.debug = true;
 
     function exit(err)
     {
@@ -267,7 +267,7 @@ function runParse(match, job, cb)
                 var message = "time spent on post-processing match ";
                 console.time(message);
                 var meta = processMetadata(entries);
-                var logs = match.doLogParse ? processReduce(entries, match, meta) : undefined;
+                var logs = processReduce(entries, match, meta);
                 var res = processExpand(entries, meta);
                 var parsed_data = processParsedData(res.parsed_data);
                 var teamfights = processTeamfights(res.tf_data, meta);

From 0cdda2a47648c70109b2bc4d5edecd0b7e0a4084 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 23 Jun 2016 04:31:52 +0000
Subject: [PATCH 39/49] use null for undefined account_ids in pgroup

---
 store/queries.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/store/queries.js b/store/queries.js
index 05ecb64cd..d6fd4282b 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -124,7 +124,7 @@ function upsert(db, table, row, conflict, cb)
             return util.format("%s=%s", key, "EXCLUDED." + key);
         });
         var query = util.format("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (%s) DO UPDATE SET %s", table, Object.keys(row).join(','), values, Object.keys(conflict).join(','), update.join(','));
-        //require('fs').writeFileSync('output.json', query);
+        //console.log(query.toString());
         db.raw(query, Object.keys(row).map(function(key)
         {
             return row[key];
@@ -161,7 +161,7 @@ function insertMatch(db, redis, match, options, cb)
         players.forEach(function(p, i)
         {
             match.pgroup[p.player_slot] = {
-                account_id: p.account_id,
+                account_id: p.account_id || null,
                 hero_id: p.hero_id,
                 player_slot: p.player_slot
             };
@@ -198,7 +198,7 @@ function insertMatch(db, redis, match, options, cb)
     {
         return cb(err, results.dp);
     });
-    
+
     function decideLogParse(cb)
     {
         if (match.leagueid)
@@ -451,7 +451,6 @@ function insertMatch(db, redis, match, options, cb)
         }, cb);
     }
 
-
     function decideParse(cb)
     {
         if (options.skipParse)

From a294c3de53ded1bbfbeea30df3e982404f7c6bb9 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 23 Jun 2016 04:39:55 +0000
Subject: [PATCH 40/49] undo match_vision change

---
 views/match/match_vision.jade | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/views/match/match_vision.jade b/views/match/match_vision.jade
index fb9e9a569..34d66137a 100644
--- a/views/match/match_vision.jade
+++ b/views/match/match_vision.jade
@@ -14,7 +14,7 @@ block match_content
           tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="obs").activate
             +hero_td(player)
             td.rankable.lane-position #{player.purchase_ward_observer || "-"}
-            td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.filter(function(l){return !l.entityleft;}).length : "-"}
+            td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.length : "-"}
             td.rankable.lane-position #{player.observer_kills || "-"}
     
     .col-md-4
@@ -29,7 +29,7 @@ block match_content
           tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="sen").activate
             +hero_td(player)
             td.rankable.lane-position #{player.purchase_ward_sentry || "-"}
-            td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.filter(function(l){return !l.entityleft;}).length : "-"}
+            td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.length : "-"}
             td.rankable.lane-position #{player.sentry_kills || "-"}
         
     .col-md-4

From 31399c82a111951184b0c1dd7313d5e1f94ab7b9 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Thu, 23 Jun 2016 04:46:33 +0000
Subject: [PATCH 41/49] disable log parsing for regular matches

---
 processors/processReduce.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/processors/processReduce.js b/processors/processReduce.js
index d0865c998..2d7f709a1 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -3,6 +3,11 @@
  **/
 function processReduce(entries, match, meta)
 {
+    //for now, disable log parsing for regular matches
+    if (!match.doLogParse)
+    {
+        return;
+    }
     var basicLogTypes = {
         "obs": 1,
         "sen": 1,

From 82cfdf05f4fb66b67f948e079b110eeacb9c3dd7 Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Thu, 23 Jun 2016 21:30:27 -0700
Subject: [PATCH 42/49] Update processExpand.js

---
 processors/processExpand.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/processors/processExpand.js b/processors/processExpand.js
index c91c15ca3..2e97f7bd1 100644
--- a/processors/processExpand.js
+++ b/processors/processExpand.js
@@ -226,7 +226,8 @@ function processExpand(entries, meta)
             //multikill
             e.unit = e.attackername;
             //add the "minimum value", as of 2016-02-06
-            e.key = e.value + 2;
+            //remove the "minimum value", as of 2016-06-23
+            e.key = e.value;
             e.value = 1;
             e.type = "multi_kills";
             expand(e);
@@ -236,7 +237,8 @@ function processExpand(entries, meta)
             //killstreak
             e.unit = e.attackername;
             //add the "minimum value", as of 2016-02-06
-            e.key = e.value + 3;
+            //remove the "minimum value", as of 2016-06-23
+            e.key = e.value;
             e.value = 1;
             e.type = "kill_streaks";
             expand(e);

From 690a2e9e8e3ec8c86e89f793842ebcde1b656d24 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Fri, 24 Jun 2016 08:47:22 +0000
Subject: [PATCH 43/49] add drafts/pro_players endpoints

---
 json/navbar_pages.json |  4 ----
 routes/api.js          | 49 +++++++++++++++++++++++++++++++++++++++++-
 store/queries.js       | 16 ++++----------
 svc/web.js             | 12 +----------
 4 files changed, 53 insertions(+), 28 deletions(-)

diff --git a/json/navbar_pages.json b/json/navbar_pages.json
index 6ddb4efd9..d57cb79fa 100644
--- a/json/navbar_pages.json
+++ b/json/navbar_pages.json
@@ -9,10 +9,6 @@
   "benchmarks": {
     "name": "Benchmarks"
   },
-  "top": {
-    "name": "Top",
-    "hide": true
-  },
   "distributions": {
     "name": "Distributions"
   },
diff --git a/routes/api.js b/routes/api.js
index 4a3311210..e901914fd 100644
--- a/routes/api.js
+++ b/routes/api.js
@@ -134,6 +134,7 @@ module.exports = function(db, redis, cassandra)
         });
     });
     */
+    /*
     api.get('/match_logs/:match_id', function(req, res, cb)
     {
         db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time ASC`, [req.params.match_id]).asCallback(function(err, result)
@@ -145,6 +146,7 @@ module.exports = function(db, redis, cassandra)
             res.json(result.rows);
         });
     });
+    */
     api.get('/pro_matches', function(req, res, cb)
     {
         db.raw(`
@@ -163,9 +165,54 @@ module.exports = function(db, redis, cassandra)
             res.json(result.rows);
         });
     });
+    api.get('/pro_players', function(req, res, cb)
+    {
+        queries.getProPlayers(db, redis, function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result);
+        });
+    });
+    api.get('/drafts', function(req, res, cb)
+    {
+        db.raw(`
+        SELECT pb.hero_id,
+        sum(case when ((pm.player_slot < 128) = m.radiant_win) then 1 else 0 end) wins, 
+        sum(case when is_pick is true then 1 else 0 end) picks,
+        sum(case when is_pick is false then 1 else 0 end) bans
+        FROM picks_bans pb
+        LEFT JOIN matches m
+        ON pb.match_id = m.match_id
+        LEFT JOIN player_matches pm
+        ON pb.hero_id = pm.hero_id
+        AND pm.match_id = m.match_id
+        GROUP BY pb.hero_id;
+        `).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
+    api.get('/pick_order', function(req, res, cb)
+    {
+        db.raw(`SELECT hero_id, ord, count( * ) FROM picks_bans WHERE is_pick is true GROUP BY hero_id, ord;`).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
     api.get('/leagues', function(req, res, cb)
     {
-        db.raw(`SELECT * FROM leagues ORDER BY leagueid`).asCallback(function(err, result)
+        db.raw(`SELECT * FROM leagues ORDER BY leagueid DESC`).asCallback(function(err, result)
         {
             if (err)
             {
diff --git a/store/queries.js b/store/queries.js
index 05ecb64cd..f2f59a8c6 100644
--- a/store/queries.js
+++ b/store/queries.js
@@ -198,7 +198,7 @@ function insertMatch(db, redis, match, options, cb)
     {
         return cb(err, results.dp);
     });
-    
+
     function decideLogParse(cb)
     {
         if (match.leagueid)
@@ -451,7 +451,6 @@ function insertMatch(db, redis, match, options, cb)
         }, cb);
     }
 
-
     function decideParse(cb)
     {
         if (options.skipParse)
@@ -725,7 +724,7 @@ function expectedWin(rates)
     return 1 - rates.reduce((prev, curr) => (100 - curr * 100) * prev, 1) / (Math.pow(50, rates.length - 1) * 100);
 }
 
-function getTop(db, redis, cb)
+function getProPlayers(db, redis, cb)
 {
     db.raw(`
     SELECT * from notable_players
@@ -735,14 +734,7 @@ function getTop(db, redis, cb)
         {
             return cb(err);
         }
-        getLeaderboard(db, redis, 'solo_competitive_rank', 500, function(err, result2)
-        {
-            return cb(err,
-            {
-                notables: result.rows,
-                leaderboard: result2
-            });
-        });
+        return cb(err, result.rows);
     });
 }
 
@@ -964,7 +956,7 @@ module.exports = {
     insertMatchSkill,
     getDistributions,
     getPicks,
-    getTop,
+    getProPlayers,
     getHeroRankings,
     getBenchmarks,
     benchmarkMatch,
diff --git a/svc/web.js b/svc/web.js
index 13ebd837e..e3182328f 100644
--- a/svc/web.js
+++ b/svc/web.js
@@ -288,17 +288,6 @@ app.get('/picks/:n?', function(req, res, cb)
         });
     });
 });
-app.get('/top', function(req, res, cb)
-{
-    queries.getTop(db, redis, function(err, result)
-    {
-        if (err)
-        {
-            return cb(err);
-        }
-        res.render('top', result);
-    });
-});
 app.get('/rankings/:hero_id?', function(req, res, cb)
 {
     if (!req.params.hero_id)
@@ -388,6 +377,7 @@ app.use(function(req, res, next)
 {
     if (config.UI_HOST)
     {
+        //route not found, redirect to SPA
         return res.redirect(config.UI_HOST + req.url);
     }
     var err = new Error("Not Found");

From 0d958f77515f21b0841d088a89fca2de12e39873 Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Fri, 24 Jun 2016 09:30:11 -0700
Subject: [PATCH 44/49] Update web.js

---
 svc/web.js | 20 +++++---------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/svc/web.js b/svc/web.js
index 5a4a490b2..bd493a5ad 100644
--- a/svc/web.js
+++ b/svc/web.js
@@ -98,7 +98,7 @@ app.use(function rateLimit(req, res, cb)
     var ip = req.headers['x-forwarded-for'] || req.connection.remoteAddress || "";
     ip = ip.replace(/^.*:/, '').split(',')[0];
     var key = 'rate_limit:' + ip;
-    console.log("%s visit %s, ip %s", req.user ? req.user.account_id : "anonymous", req.path, ip);
+    console.log("%s visit %s, ip %s", req.user ? req.user.account_id : "anonymous", req.originalUrl, ip);
     redis.multi().incr(key).expire(key, 1).exec(function(err, resp)
     {
         if (err)
@@ -121,13 +121,9 @@ app.use(function rateLimit(req, res, cb)
 app.use(function telemetry(req, res, cb)
 {
     var timeStart = new Date();
-    if (req.path.indexOf('/names') === 0)
+    if (req.originalUrl.indexOf('/api') === 0)
     {
-        redis.zadd("alias_hits", moment().format('X'), moment().valueOf() + req.path);
-    }
-    if (req.path.indexOf('/api') === 0)
-    {
-        redis.zadd("api_hits", moment().format('X'), moment().valueOf() + req.path);
+        redis.zadd("api_hits", moment().format('X'), req.originalUrl);
     }
     if (req.user)
     {
@@ -139,14 +135,8 @@ app.use(function telemetry(req, res, cb)
         var elapsed = timeEnd - timeStart;
         if (elapsed > 1000)
         {
-            console.log("[SLOWLOG] %s, %s", req.path, elapsed);
+            console.log("[SLOWLOG] %s, %s", req.originalUrl, elapsed);
         }
-        /*
-        var obj = JSON.stringify({
-            path: req.path,
-            time: timeEnd - timeStart
-        };
-        */
         redis.lpush("load_times", elapsed);
         redis.ltrim("load_times", 0, 9999);
     });
@@ -404,7 +394,7 @@ app.use(function(err, req, res, next)
 {
     res.status(err.status || 500);
     console.log(err);
-    redis.zadd("error_500", moment().format('X'), req.path);
+    redis.zadd("error_500", moment().format('X'), req.originalUrl);
     if (config.NODE_ENV !== "development")
     {
         return res.render('error/' + (err.status === 404 ? '404' : '500'),

From c76de71fb3251644bbed8ed2b7e79e42947508eb Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Fri, 24 Jun 2016 19:50:38 -0700
Subject: [PATCH 45/49] Update create_tables.sql

---
 sql/create_tables.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index 2588c01b6..cd234ab73 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -241,4 +241,4 @@ CREATE TABLE leagues(
   banner varchar(255),
   tier varchar(255),
   name varchar(255)
-)
\ No newline at end of file
+);

From 5baa18f727e3852f52a903befa84864515f4c43f Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 25 Jun 2016 03:02:36 +0000
Subject: [PATCH 46/49] fix sql schema

---
 sql/create_tables.sql | 24 ++++++++++--------------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/sql/create_tables.sql b/sql/create_tables.sql
index cd234ab73..6093d6c3b 100644
--- a/sql/create_tables.sql
+++ b/sql/create_tables.sql
@@ -21,17 +21,16 @@ CREATE TABLE matches (
   game_mode integer,
   engine integer,
   picks_bans json[],
-  --radiant_team_name varchar(255),
-  --dire_team_name varchar(255),
-  --radiant_captain integer,
-  --dire_captain integer,
-  --radiant_logo integer
-  --dire_logo integer,
-  --radiant_team_complete integer,
-  --dire_team_complete integer,
-  --radiant_team_id integer,
-  --dire_team_id integer,
-  --parsed data below
+  radiant_team_id integer,
+  dire_team_id integer,
+  radiant_team_name varchar(255),
+  dire_team_name varchar(255),
+  radiant_team_complete smallint,
+  dire_team_complete smallint,
+  radiant_captain bigint,
+  dire_captain bigint,
+  radiant_logo bigint,
+  dire_logo bigint,
   chat json[],
   objectives json[],
   radiant_gold_adv integer[],
@@ -102,9 +101,6 @@ CREATE TABLE player_matches (
   multi_kills json,
   life_state json,
   damage_inflictor_received json
-  --disabled due to incompatibility
-  --kill_streaks_log json[][], --an array of kill streak values
-  --multi_kill_id_vals integer[] --an array of multi kill values (the length of each multi kill)
 );
 CREATE INDEX on player_matches(account_id) WHERE account_id IS NOT NULL;
 

From e771bc27a1a1c206dd8f158a4a1c243bb92061cc Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 25 Jun 2016 04:42:18 +0000
Subject: [PATCH 47/49] update deploy script

---
 scripts/deploy.sh | 29 +----------------------------
 1 file changed, 1 insertion(+), 28 deletions(-)

diff --git a/scripts/deploy.sh b/scripts/deploy.sh
index e8204b539..a511a5d14 100644
--- a/scripts/deploy.sh
+++ b/scripts/deploy.sh
@@ -1,29 +1,2 @@
 #!/bin/bash
-#deploy
-DATETIME=$(date +%s)
-
-if [ "$1" = "parser" ] || [[ $# -eq 0 ]]; then
-gcloud compute instance-templates create parser-$DATETIME --machine-type n1-highcpu-2 --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --metadata startup-script='#!/bin/bash
-sudo docker run -d --name=parser --restart=always -e PROVIDER=gce -e ROLE=parser yasp/yasp:latest sh -c "npm start"
-sudo docker start parser
-'
-gcloud alpha compute rolling-updates start --group parser-group-1 --template parser-$DATETIME
-fi
-
-if [ "$1" = "backend" ] || [[ $# -eq 0 ]]; then
-gcloud compute instance-templates create backend-$DATETIME --machine-type n1-highcpu-4 --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --tags "http-server" --metadata startup-script='#!/bin/bash
-sudo docker run -d --name yasp --restart=always --net=host -e PROVIDER=gce -e GROUP=core yasp/yasp:latest sh -c "npm start"
-sudo docker start yasp
-'
-gcloud alpha compute rolling-updates start --group backend-group-1 --template backend-$DATETIME
-fi
-
-if [ "$1" = "web" ] || [[ $# -eq 0 ]]; then
-gcloud compute instance-templates create web-$DATETIME --machine-type g1-small --image container-vm --preemptible --boot-disk-size 10GB --boot-disk-type pd-ssd --tags "http-server" --metadata startup-script='#!/bin/bash
-sudo docker run -d --name=web --restart=always --net=host -e FRONTEND_PORT=80 -e PROVIDER=gce -e ROLE=web yasp/yasp:latest sh -c "npm start"
-sudo docker start web
-'
-gcloud alpha compute rolling-updates start --group web-group-1 --template web-$DATETIME --min-instance-update-time 180
-fi
-
-gcloud alpha compute rolling-updates list
\ No newline at end of file
+gcloud compute instance-groups managed list-instances $1-group-1 --format "value(NAME)" | xargs -n1 gcloud compute instance-groups managed recreate-instances $1-group-1 --instances
\ No newline at end of file

From 9c33cfc5a3e71a0e51a2230c0ffc93408b2bcaf7 Mon Sep 17 00:00:00 2001
From: Ubuntu <howardc93@gmail.com>
Date: Sat, 25 Jun 2016 06:34:39 +0000
Subject: [PATCH 48/49] properly pass dologparse option

---
 processors/processReduce.js | 10 +++++-----
 svc/parser.js               |  1 +
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/processors/processReduce.js b/processors/processReduce.js
index 2d7f709a1..682bf2a38 100644
--- a/processors/processReduce.js
+++ b/processors/processReduce.js
@@ -16,7 +16,11 @@ function processReduce(entries, match, meta)
     };
     var result = entries.filter(function(e)
     {
-        if (match.doLogParse)
+        if (!match.doLogParse)
+        {
+            return (e.type in basicLogTypes);
+        }
+        else
         {
             if (e.type === "actions")
             {
@@ -43,10 +47,6 @@ function processReduce(entries, match, meta)
             }
             return true;
         }
-        else
-        {
-            return (e.type in basicLogTypes);
-        }
     }).map(function(e)
     {
         var e2 = Object.assign(
diff --git a/svc/parser.js b/svc/parser.js
index 906fa5069..3242c70c6 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -158,6 +158,7 @@ function insertStandardParse(match, cb)
         type: "parsed",
         cassandra: cassandra,
         skipParse: true,
+        doLogParse: match.doLogParse,
     }, cb);
 }
 

From 5ed309c3414d473062f68f6996aae5fcef832beb Mon Sep 17 00:00:00 2001
From: Howard Chung <howardc93@gmail.com>
Date: Sat, 25 Jun 2016 04:05:21 -0700
Subject: [PATCH 49/49] Update parser.js

---
 svc/parser.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/svc/parser.js b/svc/parser.js
index 3242c70c6..51d2b0151 100644
--- a/svc/parser.js
+++ b/svc/parser.js
@@ -91,6 +91,7 @@ pQueue.process(config.PARSER_PARALLELISM, function(job, cb)
                 parsed_data.start_time = match.start_time;
                 parsed_data.duration = match.duration;
                 parsed_data.replay_blob_key = match.replay_blob_key;
+                parsed_data.doLogParse = match.doLogParse;
                 if (match.replay_blob_key)
                 {
                     insertUploadedParse(parsed_data, cb);