Merge branch 'match_logs' into ward_log

config.js

@@ -54,7 +54,6 @@ var defaults = {
     "UI_HOST": "", //The host of the UI, redirect traffic from / and /return here
     "ENABLE_RECAPTCHA": "", //set to enable the recaptcha on the Request page
     "ENABLE_ADS": "", //set to turn on ads
-    "ENABLE_PRO_PARSING": "", // set to parse pro matches from sequential API
     "ENABLE_MATCH_CACHE": "", // set to enable caching matches (Redis)
     "ENABLE_PLAYER_CACHE": "", // set to enable caching players (Cassandra)
     "ENABLE_INSERT_ALL_MATCHES": "", //set to enable inserting all matches

java_parser/src/main/java/yasp/Main.java

@@ -71,8 +71,8 @@ private class Entry {
         public Integer gold_reason;
         public Integer xp_reason;
         public String valuename;
-        public Float stun_duration;
-        public Float slow_duration;
+        //public Float stun_duration;
+        //public Float slow_duration;
         //entity fields
         public Integer gold;
         public Integer lh;
@@ -238,8 +238,8 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) {
             combatLogEntry.attackerillusion = cle.isAttackerIllusion();
             combatLogEntry.targetillusion = cle.isTargetIllusion();
             combatLogEntry.value = cle.getValue();
-            combatLogEntry.stun_duration = cle.getStunDuration();
-            combatLogEntry.slow_duration = cle.getSlowDuration();
+            //combatLogEntry.stun_duration = cle.getStunDuration();
+            //combatLogEntry.slow_duration = cle.getSlowDuration();
             //value may be out of bounds in string table, we can only get valuename if a purchase (type 11)
             if (cle.getType() == DOTA_COMBATLOG_TYPES.DOTA_COMBATLOG_PURCHASE) {
                 combatLogEntry.valuename = cle.getValueName();

package.json

@@ -55,7 +55,7 @@
     "http-proxy": "^1.13.1",
     "jade": "^1.11.0",
     "jquery": "^2.2.0",
-    "knex": "^0.10.0",
+    "knex": "^0.11.7",
     "moment": "^2.11.2",
     "multer": "^1.1.0",
     "numeral": "^1.5.3",

processors/processReduce.js

@@ -1,22 +1,67 @@
 /**
- * A processor to reduce the event stream by grouping similar events.
- * NOT CURRENTLY IN PRODUCTION USE
+ * A processor to reduce the event stream to only logs we want to persist
  **/
-function processReduce(entries)
+function processReduce(entries, match, meta)
 {
-    var reduceMap = {};
-    //group by player_slot, type, targethero, targetillusion
-    for (var i = 0; i < entries.length; i++)
+    var result = entries.filter(function(e)
     {
-        //group big categories: actions, combat log damage
-        var e = entries[i];
-        var identifier = [e.player_slot, e.type, e.key].join(":");
-        e.value = e.value || 1;
-        //var identifier = e.type;
-        //e.value = 1;
-        reduceMap[identifier] = reduceMap[identifier] ? reduceMap[identifier] + e.value : e.value || 1;
-    }
-    //var fs = require('fs');
-    //fs.writeFileSync('./output3.json', JSON.stringify(reduceMap, null , 2));
+        if (e.type === "actions")
+        {
+            return false;
+        }
+        if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE")
+        {
+            return false;
+        }
+        if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL")
+        {
+            if (!e.targethero || e.targetillusion)
+            {
+                return false;
+            }
+        }
+        if (e.type === "interval" && e.time % 60 !== 0)
+        {
+            return false;
+        }
+        if (!e.time)
+        {
+            return false;
+        }
+        return true;
+    }).map(function(e)
+    {
+        var e2 = Object.assign(
+        {}, e,
+        {
+            match_id: match.match_id,
+            attackername_slot: meta.hero_to_slot[e.attackername],
+            targetname_slot: meta.hero_to_slot[e.targetname],
+            sourcename_slot: meta.hero_to_slot[e.sourcename],
+            targetsourcename_slot: meta.hero_to_slot[e.targetname],
+            player1_slot: meta.slot_to_playerslot[e.player1],
+            player_slot: e.player_slot || meta.slot_to_playerslot[e.slot],
+            inflictor: translate(e.inflictor),
+        });
+        delete e2.attackername;
+        delete e2.targetname;
+        delete e2.sourcename;
+        delete e2.targetsourcename;
+        return e2;
+    });
+    /*
+    var count = {};
+    result.forEach(function(r)
+    {
+        count[r.type] = (count[r.type] || 0) + 1;
+    });
+    console.log(count);
+    */
+    return result;
+}
+
+function translate(s)
+{
+    return s === "dota_unknown" ? null : s;
 }
 module.exports = processReduce;

routes/api.js

@@ -134,6 +134,46 @@ module.exports = function(db, redis, cassandra)
         });
     });
     */
+    api.get('/match_logs/:match_id', function(req, res, cb)
+    {
+        db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time ASC`, [req.params.match_id]).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
+    api.get('/pro_matches', function(req, res, cb)
+    {
+        db.raw(`
+        SELECT match_id, start_time, duration, ma.leagueid, name
+        FROM matches ma
+        JOIN leagues le
+        ON ma.leagueid = le.leagueid
+        WHERE ma.leagueid > 0
+        ORDER BY match_id DESC
+        `).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
+    api.get('/leagues', function(req, res, cb)
+    {
+        db.raw(`SELECT * FROM leagues ORDER BY leagueid`).asCallback(function(err, result)
+        {
+            if (err)
+            {
+                return cb(err);
+            }
+            res.json(result.rows);
+        });
+    });
     api.get('/distributions', function(req, res, cb)
     {
         queries.getDistributions(redis, function(err, result)

sql/create_tables.sql

@@ -39,6 +39,7 @@ CREATE TABLE matches (
   teamfights json[],
   version integer
   );
+CREATE INDEX on matches(leagueid) WHERE leagueid > 0;
 
 CREATE TABLE player_matches (
   PRIMARY KEY(match_id, player_slot),
@@ -174,3 +175,67 @@ CREATE TABLE notable_players (
   is_pro boolean,
   locked_until integer
 );
+
+CREATE TABLE match_logs (
+  match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
+  time int,
+  type varchar(100),
+  team smallint,
+  unit varchar(100),
+  key varchar(1000),
+  value int,
+  slot smallint,
+  player_slot smallint,
+  player1 int,
+  player2 int,
+  attackerhero boolean,
+  targethero boolean,
+  attackerillusion boolean,
+  targetillusion boolean,
+  inflictor varchar(100),
+  gold_reason smallint,
+  xp_reason smallint,
+  valuename varchar(100),
+  gold int,
+  lh int,
+  xp int,
+  x smallint,
+  y smallint,
+  stuns real,
+  hero_id smallint,
+  life_state smallint,
+  level smallint,
+  kills smallint,
+  deaths smallint,
+  assists smallint,
+  denies smallint,
+  attackername_slot smallint,
+  targetname_slot smallint,
+  sourcename_slot smallint,
+  targetsourcename_slot smallint,
+  player1_slot smallint
+);
+CREATE INDEX ON match_logs(match_id);
+CREATE INDEX ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL;
+
+CREATE TABLE picks_bans(
+  match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
+  is_pick boolean,
+  hero_id int,
+  team smallint,
+  ord smallint,
+  PRIMARY KEY (match_id, ord)
+);
+
+CREATE TABLE leagues(
+  leagueid bigint PRIMARY KEY,
+  ticket varchar(255),
+  banner varchar(255),
+  tier varchar(255),
+  name varchar(255)
+)

store/buildSets.js

@@ -18,20 +18,6 @@ module.exports = function buildSets(db, redis, cb) {
                 cb(err, t);
             });
         },
-        //users in this set have their matches added
-        "userPlayers": function(cb) {
-            db.select(['account_id']).from('players').whereNotNull('last_login').asCallback(function(err, docs) {
-                if (err) {
-                    return cb(err);
-                }
-                var t = {};
-                docs.forEach(function(player) {
-                    t[player.account_id] = true;
-                });
-                //console.log(t);
-                cb(err, t);
-            });
-        },
         //users in this set are added to the trackedPlayers set
         "donators": function(cb) {
             db.select(['account_id']).from('players').where('cheese', '>', 0).asCallback(function(err, docs) {