From 68ada3214f84dc63a0f85660feb2314a27261d8a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 19:04:08 +0000 Subject: [PATCH] store raw data, with slot indexes --- processors/processReduce.js | 32 +++++++++++-- routes/api.js | 2 +- sql/create_tables.sql | 45 +++++++++++-------- store/queries.js | 4 +- svc/parser.js | 2 +- tasks/getMatches.js | 50 +++++++++++++++++++++ dev/allMatches.js => tasks/getMatchesSeq.js | 0 7 files changed, 109 insertions(+), 26 deletions(-) create mode 100644 tasks/getMatches.js rename dev/allMatches.js => tasks/getMatchesSeq.js (100%) diff --git a/processors/processReduce.js b/processors/processReduce.js index 1b11d4bda..f8ed5bd5e 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -2,9 +2,9 @@ * A processor to reduce the event stream by grouping similar events. * NOT CURRENTLY IN PRODUCTION USE **/ -function processReduce(entries, match) +function processReduce(entries, match, meta) { - return entries.filter(function(e) + var result = entries.filter(function(e) { if (e.type === "actions") { @@ -21,11 +21,35 @@ function processReduce(entries, match) return true; }).map(function(e) { - return Object.assign( + var e2 = Object.assign( {}, e, { - match_id: match.match_id + match_id: match.match_id, + attackername_slot: meta.hero_to_slot[e.attackername], + targetname_slot: meta.hero_to_slot[e.targetname], + sourcename_slot: meta.hero_to_slot[e.sourcename], + targetsourcename_slot: meta.hero_to_slot[e.targetname], + player1_slot: meta.slot_to_playerslot[e.player1], + player_slot: e.player_slot || meta.slot_to_playerslot[e.slot], + inflictor: translate(e.inflictor), }); + delete e2.attackername; + delete e2.targetname; + delete e2.sourcename; + delete e2.targetsourcename; + return e2; }); + var count = {}; + result.forEach(function(r) + { + count[r.type] = (count[r.type] || 0) + 1; + }); + console.log(count); + return result; +} + +function translate(s) +{ + return s === "dota_unknown" ? null : s; } module.exports = processReduce; \ No newline at end of file diff --git a/routes/api.js b/routes/api.js index f8a878abf..475d2eb85 100644 --- a/routes/api.js +++ b/routes/api.js @@ -136,7 +136,7 @@ module.exports = function(db, redis, cassandra) */ api.get('/match_logs/:match_id', function(req, res, cb) { - db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result) + db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result) { if (err) { diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 5efdad478..bae99a7d2 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -179,38 +179,45 @@ CREATE TABLE match_logs ( match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, time int, type varchar(100), - team int, + team smallint, unit varchar(100), - key varchar(100), + key varchar(1000), value int, - slot int, - player_slot int, + slot smallint, + player_slot smallint, player1 int, player2 int, - attackername varchar(100), - targetname varchar(100), - sourcename varchar(100), - targetsourcename varchar(100), attackerhero boolean, targethero boolean, attackerillusion boolean, targetillusion boolean, inflictor varchar(100), - gold_reason int, - xp_reason int, + gold_reason smallint, + xp_reason smallint, valuename varchar(100), gold int, lh int, xp int, - x int, - y int, + x smallint, + y smallint, stuns real, - hero_id int, - life_state int, - level int, - kills int, - deaths int, - assists int, - denies int + hero_id smallint, + life_state smallint, + level smallint, + kills smallint, + deaths smallint, + assists smallint, + denies smallint, + attackername_slot smallint, + targetname_slot smallint, + sourcename_slot smallint, + targetsourcename_slot smallint, + player1_slot smallint ); CREATE INDEX ON match_logs(match_id); +CREATE INDEX ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL; diff --git a/store/queries.js b/store/queries.js index 78831b848..79a2c9167 100644 --- a/store/queries.js +++ b/store/queries.js @@ -238,7 +238,9 @@ function insertMatch(db, redis, match, options, cb) //TODO insert picks/bans //TODO remove dota_unknown to save space //TODO figure out whehter to store expanded or raw data - +/* +"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}] +*/ function exit(err) { if (err) diff --git a/svc/parser.js b/svc/parser.js index 8c1a75325..5cff09a0c 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -267,7 +267,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.leagueid ? processReduce(entries, match) : undefined; + var logs = match.leagueid ? processReduce(entries, match, meta) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); diff --git a/tasks/getMatches.js b/tasks/getMatches.js new file mode 100644 index 000000000..cf2d7fd92 --- /dev/null +++ b/tasks/getMatches.js @@ -0,0 +1,50 @@ +var async = require('async'); +var utility = require('../util/utility'); +var generateJob = utility.generateJob; +var getData = utility.getData; +var db = require('../store/db'); +var redis = require('../store/redis'); +var cassandra = require('../store/cassandra'); +var queries = require('../store/queries'); +var insertMatch = queries.insertMatch; +var args = process.argv.slice(2); +var match_id = Number(args[0]); +var delay = 1000; +var job = generateJob("api_details", +{ + match_id: match_id +}); +var url = job.url; +getData( +{ + url: url, + delay: delay +}, function(err, body) +{ + if (err) + { + throw err; + } + if (body.result) + { + var match = body.result; + match.parse_status = 0; + insertMatch(db, redis, match, + { + skipCounts: true, + skipAbilityUpgrades: true, + cassandra: cassandra, + }, function(err) + { + if (err) + { + throw err; + } + process.exit(0); + }); + } + else + { + throw body; + } +}); \ No newline at end of file diff --git a/dev/allMatches.js b/tasks/getMatchesSeq.js similarity index 100% rename from dev/allMatches.js rename to tasks/getMatchesSeq.js