store raw data, with slot indexes

processors/processReduce.js

@@ -2,9 +2,9 @@
  * A processor to reduce the event stream by grouping similar events.
  * NOT CURRENTLY IN PRODUCTION USE
  **/
-function processReduce(entries, match)
+function processReduce(entries, match, meta)
 {
-    return entries.filter(function(e)
+    var result = entries.filter(function(e)
     {
         if (e.type === "actions")
         {
@@ -21,11 +21,35 @@ function processReduce(entries, match)
         return true;
     }).map(function(e)
     {
-        return Object.assign(
+        var e2 = Object.assign(
         {}, e,
         {
-            match_id: match.match_id
+            match_id: match.match_id,
+            attackername_slot: meta.hero_to_slot[e.attackername],
+            targetname_slot: meta.hero_to_slot[e.targetname],
+            sourcename_slot: meta.hero_to_slot[e.sourcename],
+            targetsourcename_slot: meta.hero_to_slot[e.targetname],
+            player1_slot: meta.slot_to_playerslot[e.player1],
+            player_slot: e.player_slot || meta.slot_to_playerslot[e.slot],
+            inflictor: translate(e.inflictor),
         });
+        delete e2.attackername;
+        delete e2.targetname;
+        delete e2.sourcename;
+        delete e2.targetsourcename;
+        return e2;
     });
+    var count = {};
+    result.forEach(function(r)
+    {
+        count[r.type] = (count[r.type] || 0) + 1;
+    });
+    console.log(count);
+    return result;
+}
+
+function translate(s)
+{
+    return s === "dota_unknown" ? null : s;
 }
 module.exports = processReduce;

routes/api.js

@@ -136,7 +136,7 @@ module.exports = function(db, redis, cassandra)
     */
     api.get('/match_logs/:match_id', function(req, res, cb)
     {
-        db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result)
+        db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result)
         {
             if (err)
             {

sql/create_tables.sql

@@ -179,38 +179,45 @@ CREATE TABLE match_logs (
   match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
   time int,
   type varchar(100),
-  team int,
+  team smallint,
   unit varchar(100),
-  key varchar(100),
+  key varchar(1000),
   value int,
-  slot int,
-  player_slot int,
+  slot smallint,
+  player_slot smallint,
   player1 int,
   player2 int,
-  attackername varchar(100),
-  targetname varchar(100),
-  sourcename varchar(100),
-  targetsourcename varchar(100),
   attackerhero boolean,
   targethero boolean,
   attackerillusion boolean,
   targetillusion boolean,
   inflictor varchar(100),
-  gold_reason int,
-  xp_reason int,
+  gold_reason smallint,
+  xp_reason smallint,
   valuename varchar(100),
   gold int,
   lh int,
   xp int,
-  x int,
-  y int,
+  x smallint,
+  y smallint,
   stuns real,
-  hero_id int,
-  life_state int,
-  level int,
-  kills int,
-  deaths int,
-  assists int,
-  denies int
+  hero_id smallint,
+  life_state smallint,
+  level smallint,
+  kills smallint,
+  deaths smallint,
+  assists smallint,
+  denies smallint,
+  attackername_slot smallint,
+  targetname_slot smallint,
+  sourcename_slot smallint,
+  targetsourcename_slot smallint,
+  player1_slot smallint
 );
 CREATE INDEX ON match_logs(match_id);
+CREATE INDEX ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL;
+CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL;

store/queries.js

@@ -238,7 +238,9 @@ function insertMatch(db, redis, match, options, cb)
                 //TODO insert picks/bans
                 //TODO remove dota_unknown to save space
                 //TODO figure out whehter to store expanded or raw data
-
+/*
+"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}]
+*/
                 function exit(err)
                 {
                     if (err)

svc/parser.js

@@ -267,7 +267,7 @@ function runParse(match, job, cb)
                 var message = "time spent on post-processing match ";
                 console.time(message);
                 var meta = processMetadata(entries);
-                var logs = match.leagueid ? processReduce(entries, match) : undefined;
+                var logs = match.leagueid ? processReduce(entries, match, meta) : undefined;
                 var res = processExpand(entries, meta);
                 var parsed_data = processParsedData(res.parsed_data);
                 var teamfights = processTeamfights(res.tf_data, meta);

tasks/getMatches.js

@@ -0,0 +1,50 @@
+var async = require('async');
+var utility = require('../util/utility');
+var generateJob = utility.generateJob;
+var getData = utility.getData;
+var db = require('../store/db');
+var redis = require('../store/redis');
+var cassandra = require('../store/cassandra');
+var queries = require('../store/queries');
+var insertMatch = queries.insertMatch;
+var args = process.argv.slice(2);
+var match_id = Number(args[0]);
+var delay = 1000;
+var job = generateJob("api_details",
+{
+    match_id: match_id
+});
+var url = job.url;
+getData(
+{
+    url: url,
+    delay: delay
+}, function(err, body)
+{
+    if (err)
+    {
+        throw err;
+    }
+    if (body.result)
+    {
+        var match = body.result;
+        match.parse_status = 0;
+        insertMatch(db, redis, match,
+        {
+            skipCounts: true,
+            skipAbilityUpgrades: true,
+            cassandra: cassandra,
+        }, function(err)
+        {
+            if (err)
+            {
+                throw err;
+            }
+            process.exit(0);
+        });
+    }
+    else
+    {
+        throw body;
+    }
+});