From dc8d48bf528da1e6fd588146e8002edec81c4c20 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 11 Jun 2016 03:41:40 +0000 Subject: [PATCH 01/25] start implementation of log parsing --- processors/processExpand.js | 2 +- processors/processReduce.js | 22 ++++++++++++------- sql/create_tables.sql | 44 ++++++++++++++++++++++++++++++++++++- svc/parser.js | 3 ++- 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/processors/processExpand.js b/processors/processExpand.js index c91c15ca3..e67b97647 100644 --- a/processors/processExpand.js +++ b/processors/processExpand.js @@ -449,7 +449,7 @@ function processExpand(entries, meta) //null means all types var reqs = { parsed_data: null, - //expanded: null, + expanded: null, tf_data: { killed: 1, diff --git a/processors/processReduce.js b/processors/processReduce.js index 41493a8fa..f35890898 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -8,15 +8,21 @@ function processReduce(entries) //group by player_slot, type, targethero, targetillusion for (var i = 0; i < entries.length; i++) { - //group big categories: actions, combat log damage var e = entries[i]; - var identifier = [e.player_slot, e.type, e.key].join(":"); - e.value = e.value || 1; - //var identifier = e.type; - //e.value = 1; - reduceMap[identifier] = reduceMap[identifier] ? reduceMap[identifier] + e.value : e.value || 1; + reduceMap[e.type] = reduceMap[e.type] ? reduceMap[e.type] + 1 : 1; } - //var fs = require('fs'); - //fs.writeFileSync('./output3.json', JSON.stringify(reduceMap, null , 2)); + console.log(reduceMap); + return entries.filter(function(e) + { + if (e.type === "actions") + { + return false; + } + if (e.type === "interval" && e.time % 60 !== 0) + { + return false; + } + return true; + }); } module.exports = processReduce; \ No newline at end of file diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 80f3fec9d..72e5d6370 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -172,4 +172,46 @@ CREATE TABLE notable_players ( is_locked boolean, is_pro boolean, locked_until integer -); \ No newline at end of file +); + +CREATE TABLE match_logs ( + match_id bigint, + time int, + type varchar(100), + team int, + unit varchar(100), + key varchar(100), + value int, + slot int, + player_slot int, + player1 int, + player2 int, + attackername string, + targetname string, + sourcename string, + targetsourcename string, + attackerhero boolean, + targethero boolean, + attackerillusion boolean, + targetillusion boolean, + inflictor varchar(100), + gold_reason int, + xp_reason int, + valuename varchar(100), + stun_duration real, + slow_duration real, + gold int, + lh int, + xp int, + x int, + y int, + stuns real, + hero_id int, + life_state int, + level int, + kills int, + deaths int, + assists int, + denies int +); +CREATE INDEX ON match_logs(match_id); \ No newline at end of file diff --git a/svc/parser.js b/svc/parser.js index 7843e0609..8384ab224 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -263,6 +263,8 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); + var logs = processReduce(entries); + console.log('logs: %s', logs.length); var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); @@ -273,7 +275,6 @@ function runParse(match, job, cb) parsed_data.radiant_xp_adv = ap.radiant_xp_adv; parsed_data.upload = upload; //processMultiKillStreaks(); - //processReduce(res.expanded); console.timeEnd(message); return cb(err, parsed_data); } From 8774cd487b583d34758ac2a9a7129f8d949731b2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 11 Jun 2016 08:33:22 +0000 Subject: [PATCH 02/25] update schema --- processors/processReduce.js | 21 +++++++++++-------- sql/create_tables.sql | 8 ++++---- store/queries.js | 41 +++++++++++++++++++++++++++++++++++++ svc/parser.js | 4 ++-- 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/processors/processReduce.js b/processors/processReduce.js index f35890898..1b11d4bda 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -2,16 +2,8 @@ * A processor to reduce the event stream by grouping similar events. * NOT CURRENTLY IN PRODUCTION USE **/ -function processReduce(entries) +function processReduce(entries, match) { - var reduceMap = {}; - //group by player_slot, type, targethero, targetillusion - for (var i = 0; i < entries.length; i++) - { - var e = entries[i]; - reduceMap[e.type] = reduceMap[e.type] ? reduceMap[e.type] + 1 : 1; - } - console.log(reduceMap); return entries.filter(function(e) { if (e.type === "actions") @@ -22,7 +14,18 @@ function processReduce(entries) { return false; } + if (!e.time) + { + return false; + } return true; + }).map(function(e) + { + return Object.assign( + {}, e, + { + match_id: match.match_id + }); }); } module.exports = processReduce; \ No newline at end of file diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 47e7b4e3d..7a7b418b3 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -187,10 +187,10 @@ CREATE TABLE match_logs ( player_slot int, player1 int, player2 int, - attackername string, - targetname string, - sourcename string, - targetsourcename string, + attackername varchar(100), + targetname varchar(100), + sourcename varchar(100), + targetsourcename varchar(100), attackerhero boolean, targethero boolean, attackerillusion boolean, diff --git a/store/queries.js b/store/queries.js index c4952d6ed..bd6e3446c 100644 --- a/store/queries.js +++ b/store/queries.js @@ -193,6 +193,7 @@ function insertMatch(db, redis, match, options, cb) { "u": upsertMatch, "uc": upsertMatchCassandra, + "uml": upsertMatchLogs, "upc": updatePlayerCaches, "uct": updateCounts, "cmc": clearMatchCache, @@ -309,6 +310,45 @@ function insertMatch(db, redis, match, options, cb) }); } + function upsertMatchLogs(cb) + { + if (!match.logs) + { + return cb(); + } + else + { + db.transaction(function(trx) + { + trx.raw(`DELETE FROM match_logs where match_id = ?`, [match.match_id]).asCallback(function(err) + { + if (err) + { + return exit(err); + } + async.eachLimit(match.logs, 10000, function(e, cb) + { + trx('match_logs').insert(e).asCallback(cb); + }, exit); + }); + + function exit(err) + { + if (err) + { + console.error(err); + trx.rollback(err); + } + else + { + trx.commit(); + } + cb(err); + } + }); + } + } + function updatePlayerCaches(cb) { var copy = createMatchCopy(match, players, options); @@ -400,6 +440,7 @@ function insertMatch(db, redis, match, options, cb) duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, + leagueid: match.leagueid, }, { lifo: options.lifo, diff --git a/svc/parser.js b/svc/parser.js index 8384ab224..3355508fd 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -263,8 +263,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = processReduce(entries); - console.log('logs: %s', logs.length); + var logs = match.leagueid ? processReduce(entries, match) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); @@ -274,6 +273,7 @@ function runParse(match, job, cb) parsed_data.radiant_gold_adv = ap.radiant_gold_adv; parsed_data.radiant_xp_adv = ap.radiant_xp_adv; parsed_data.upload = upload; + parsed_data.logs = logs; //processMultiKillStreaks(); console.timeEnd(message); return cb(err, parsed_data); From 63207e4aa7b44ae17e254291c5ddc00dbf5d0efb Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 05:51:30 +0000 Subject: [PATCH 03/25] add api --- java_parser/src/main/java/yasp/Main.java | 8 ++++---- routes/api.js | 11 +++++++++++ sql/create_tables.sql | 2 -- store/queries.js | 6 +++++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java index f507f938f..edca25d14 100644 --- a/java_parser/src/main/java/yasp/Main.java +++ b/java_parser/src/main/java/yasp/Main.java @@ -70,8 +70,8 @@ private class Entry { public Integer gold_reason; public Integer xp_reason; public String valuename; - public Float stun_duration; - public Float slow_duration; + //public Float stun_duration; + //public Float slow_duration; //entity fields public Integer gold; public Integer lh; @@ -234,8 +234,8 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) { combatLogEntry.attackerillusion = cle.isAttackerIllusion(); combatLogEntry.targetillusion = cle.isTargetIllusion(); combatLogEntry.value = cle.getValue(); - combatLogEntry.stun_duration = cle.getStunDuration(); - combatLogEntry.slow_duration = cle.getSlowDuration(); + //combatLogEntry.stun_duration = cle.getStunDuration(); + //combatLogEntry.slow_duration = cle.getSlowDuration(); //value may be out of bounds in string table, we can only get valuename if a purchase (type 11) if (cle.getType() == DOTA_COMBATLOG_TYPES.DOTA_COMBATLOG_PURCHASE) { combatLogEntry.valuename = cle.getValueName(); diff --git a/routes/api.js b/routes/api.js index c2afa0c3f..f8a878abf 100644 --- a/routes/api.js +++ b/routes/api.js @@ -134,6 +134,17 @@ module.exports = function(db, redis, cassandra) }); }); */ + api.get('/match_logs/:match_id', function(req, res, cb) + { + db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); api.get('/distributions', function(req, res, cb) { queries.getDistributions(redis, function(err, result) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index d20cac76b..eb9f960a4 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -199,8 +199,6 @@ CREATE TABLE match_logs ( gold_reason int, xp_reason int, valuename varchar(100), - stun_duration real, - slow_duration real, gold int, lh int, xp int, diff --git a/store/queries.js b/store/queries.js index cf191fab2..20e698405 100644 --- a/store/queries.js +++ b/store/queries.js @@ -208,7 +208,7 @@ function insertMatch(db, redis, match, options, cb) function upsertMatch(cb) { - if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE) + if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.leagueid) { return cb(); } @@ -232,6 +232,10 @@ function insertMatch(db, redis, match, options, cb) player_slot: pm.player_slot }, cb); }, exit); + //TODO create table for picks/bans + //TODO insert picks/bans + //TODO remove dota_unknown to save space + //TODO figure out whehter to store expanded or raw data function exit(err) { From fd63f476472a40ccbf5f4a1a9a677acca75cfdfa Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 05:53:54 +0000 Subject: [PATCH 04/25] add todos --- sql/create_tables.sql | 2 +- store/queries.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index eb9f960a4..5efdad478 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -176,7 +176,7 @@ CREATE TABLE notable_players ( ); CREATE TABLE match_logs ( - match_id bigint, + match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, time int, type varchar(100), team int, diff --git a/store/queries.js b/store/queries.js index 20e698405..78831b848 100644 --- a/store/queries.js +++ b/store/queries.js @@ -232,6 +232,8 @@ function insertMatch(db, redis, match, options, cb) player_slot: pm.player_slot }, cb); }, exit); + //TODO create leagues table + //TODO create worker task to get leagues //TODO create table for picks/bans //TODO insert picks/bans //TODO remove dota_unknown to save space From 68ada3214f84dc63a0f85660feb2314a27261d8a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 19:04:08 +0000 Subject: [PATCH 05/25] store raw data, with slot indexes --- processors/processReduce.js | 32 +++++++++++-- routes/api.js | 2 +- sql/create_tables.sql | 45 +++++++++++-------- store/queries.js | 4 +- svc/parser.js | 2 +- tasks/getMatches.js | 50 +++++++++++++++++++++ dev/allMatches.js => tasks/getMatchesSeq.js | 0 7 files changed, 109 insertions(+), 26 deletions(-) create mode 100644 tasks/getMatches.js rename dev/allMatches.js => tasks/getMatchesSeq.js (100%) diff --git a/processors/processReduce.js b/processors/processReduce.js index 1b11d4bda..f8ed5bd5e 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -2,9 +2,9 @@ * A processor to reduce the event stream by grouping similar events. * NOT CURRENTLY IN PRODUCTION USE **/ -function processReduce(entries, match) +function processReduce(entries, match, meta) { - return entries.filter(function(e) + var result = entries.filter(function(e) { if (e.type === "actions") { @@ -21,11 +21,35 @@ function processReduce(entries, match) return true; }).map(function(e) { - return Object.assign( + var e2 = Object.assign( {}, e, { - match_id: match.match_id + match_id: match.match_id, + attackername_slot: meta.hero_to_slot[e.attackername], + targetname_slot: meta.hero_to_slot[e.targetname], + sourcename_slot: meta.hero_to_slot[e.sourcename], + targetsourcename_slot: meta.hero_to_slot[e.targetname], + player1_slot: meta.slot_to_playerslot[e.player1], + player_slot: e.player_slot || meta.slot_to_playerslot[e.slot], + inflictor: translate(e.inflictor), }); + delete e2.attackername; + delete e2.targetname; + delete e2.sourcename; + delete e2.targetsourcename; + return e2; }); + var count = {}; + result.forEach(function(r) + { + count[r.type] = (count[r.type] || 0) + 1; + }); + console.log(count); + return result; +} + +function translate(s) +{ + return s === "dota_unknown" ? null : s; } module.exports = processReduce; \ No newline at end of file diff --git a/routes/api.js b/routes/api.js index f8a878abf..475d2eb85 100644 --- a/routes/api.js +++ b/routes/api.js @@ -136,7 +136,7 @@ module.exports = function(db, redis, cassandra) */ api.get('/match_logs/:match_id', function(req, res, cb) { - db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result) + db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result) { if (err) { diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 5efdad478..bae99a7d2 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -179,38 +179,45 @@ CREATE TABLE match_logs ( match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, time int, type varchar(100), - team int, + team smallint, unit varchar(100), - key varchar(100), + key varchar(1000), value int, - slot int, - player_slot int, + slot smallint, + player_slot smallint, player1 int, player2 int, - attackername varchar(100), - targetname varchar(100), - sourcename varchar(100), - targetsourcename varchar(100), attackerhero boolean, targethero boolean, attackerillusion boolean, targetillusion boolean, inflictor varchar(100), - gold_reason int, - xp_reason int, + gold_reason smallint, + xp_reason smallint, valuename varchar(100), gold int, lh int, xp int, - x int, - y int, + x smallint, + y smallint, stuns real, - hero_id int, - life_state int, - level int, - kills int, - deaths int, - assists int, - denies int + hero_id smallint, + life_state smallint, + level smallint, + kills smallint, + deaths smallint, + assists smallint, + denies smallint, + attackername_slot smallint, + targetname_slot smallint, + sourcename_slot smallint, + targetsourcename_slot smallint, + player1_slot smallint ); CREATE INDEX ON match_logs(match_id); +CREATE INDEX ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL; diff --git a/store/queries.js b/store/queries.js index 78831b848..79a2c9167 100644 --- a/store/queries.js +++ b/store/queries.js @@ -238,7 +238,9 @@ function insertMatch(db, redis, match, options, cb) //TODO insert picks/bans //TODO remove dota_unknown to save space //TODO figure out whehter to store expanded or raw data - +/* +"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}] +*/ function exit(err) { if (err) diff --git a/svc/parser.js b/svc/parser.js index 8c1a75325..5cff09a0c 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -267,7 +267,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.leagueid ? processReduce(entries, match) : undefined; + var logs = match.leagueid ? processReduce(entries, match, meta) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); diff --git a/tasks/getMatches.js b/tasks/getMatches.js new file mode 100644 index 000000000..cf2d7fd92 --- /dev/null +++ b/tasks/getMatches.js @@ -0,0 +1,50 @@ +var async = require('async'); +var utility = require('../util/utility'); +var generateJob = utility.generateJob; +var getData = utility.getData; +var db = require('../store/db'); +var redis = require('../store/redis'); +var cassandra = require('../store/cassandra'); +var queries = require('../store/queries'); +var insertMatch = queries.insertMatch; +var args = process.argv.slice(2); +var match_id = Number(args[0]); +var delay = 1000; +var job = generateJob("api_details", +{ + match_id: match_id +}); +var url = job.url; +getData( +{ + url: url, + delay: delay +}, function(err, body) +{ + if (err) + { + throw err; + } + if (body.result) + { + var match = body.result; + match.parse_status = 0; + insertMatch(db, redis, match, + { + skipCounts: true, + skipAbilityUpgrades: true, + cassandra: cassandra, + }, function(err) + { + if (err) + { + throw err; + } + process.exit(0); + }); + } + else + { + throw body; + } +}); \ No newline at end of file diff --git a/dev/allMatches.js b/tasks/getMatchesSeq.js similarity index 100% rename from dev/allMatches.js rename to tasks/getMatchesSeq.js From 5530d780d0d8220dbde4ebfbab4f978e26ee2a4f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 23:34:48 +0000 Subject: [PATCH 06/25] remove some debugging info --- processors/processExpand.js | 2 +- processors/processReduce.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/processors/processExpand.js b/processors/processExpand.js index e67b97647..c91c15ca3 100644 --- a/processors/processExpand.js +++ b/processors/processExpand.js @@ -449,7 +449,7 @@ function processExpand(entries, meta) //null means all types var reqs = { parsed_data: null, - expanded: null, + //expanded: null, tf_data: { killed: 1, diff --git a/processors/processReduce.js b/processors/processReduce.js index f8ed5bd5e..2bf9dbe00 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -1,6 +1,5 @@ /** - * A processor to reduce the event stream by grouping similar events. - * NOT CURRENTLY IN PRODUCTION USE + * A processor to reduce the event stream to only logs we want to persist **/ function processReduce(entries, match, meta) { @@ -39,12 +38,14 @@ function processReduce(entries, match, meta) delete e2.targetsourcename; return e2; }); + /* var count = {}; result.forEach(function(r) { count[r.type] = (count[r.type] || 0) + 1; }); console.log(count); + */ return result; } From 98fc634794f12e7ffa0e3eda13f8bea3c70ae3ea Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 00:43:06 +0000 Subject: [PATCH 07/25] implement league fetcher, use async for match insert flow --- sql/create_tables.sql | 17 +++++++++ store/queries.js | 81 +++++++++++++++++++++++++++---------------- svc/parser.js | 2 +- svc/scanner.js | 7 +--- svc/worker.js | 30 ++++++++++++++++ 5 files changed, 101 insertions(+), 36 deletions(-) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index bae99a7d2..8d1564d77 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -221,3 +221,20 @@ CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL; CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL; CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL; + +CREATE TABLE picks_bans( + match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, + is_pick boolean, + hero_id int, + team smallint, + order smallint, + PRIMARY KEY (match_id, order) +); + +CREATE TABLE leagues( + leagueid bigint PRIMARY KEY, + ticket varchar(255), + banner varchar(255), + tier varchar(255), + name varchar(255) +) \ No newline at end of file diff --git a/store/queries.js b/store/queries.js index 79a2c9167..f082aac51 100644 --- a/store/queries.js +++ b/store/queries.js @@ -191,6 +191,7 @@ function insertMatch(db, redis, match, options, cb) //we want to insert into matches, then insert into player_matches for each entry in players async.series( { + "ipm": isProMatch, "u": upsertMatch, "uc": upsertMatchCassandra, "uml": upsertMatchLogs, @@ -206,23 +207,38 @@ function insertMatch(db, redis, match, options, cb) return cb(err, results.dp); }); + function isProMatch(cb) + { + //TODO check redis/postgres for professional/premium league + match.isProMatch = match.leagueid >= 0; + cb(); + } + function upsertMatch(cb) { - if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.leagueid) + if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.isProMatch) { return cb(); } db.transaction(function(trx) { - upsert(trx, 'matches', match, + async.series( { - match_id: match.match_id - }, function(err) + "im": insertMatch, + "ipm": insertPlayerMatches, + "ipb": insertPicksBans + }, exit); + + function insertMatch(cb) { - if (err) + upsert(trx, 'matches', match, { - return exit(err); - } + match_id: match.match_id + }, cb); + } + + function insertPlayerMatches(cb) + { async.each(players || [], function(pm, cb) { pm.match_id = match.match_id; @@ -231,30 +247,37 @@ function insertMatch(db, redis, match, options, cb) match_id: pm.match_id, player_slot: pm.player_slot }, cb); - }, exit); - //TODO create leagues table - //TODO create worker task to get leagues - //TODO create table for picks/bans - //TODO insert picks/bans - //TODO remove dota_unknown to save space - //TODO figure out whehter to store expanded or raw data -/* -"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}] -*/ - function exit(err) + }, cb); + } + + function insertPicksBans(cb) + { + async.each(match.picks_bans || [], function(p, cb) { - if (err) + //order is a reserved keyword + p.ord = p.order; + p.match_id = match.match_id; + upsert(trx, 'picks_bans', p, { - console.error(err); - trx.rollback(err); - } - else - { - trx.commit(); - } - cb(err); + match_id: p.match_id, + ord: p.ord + }, cb); + }, cb); + } + + function exit(err) + { + if (err) + { + console.error(err); + trx.rollback(err); } - }); + else + { + trx.commit(); + } + cb(err); + } }); } @@ -462,7 +485,7 @@ function insertMatch(db, redis, match, options, cb) duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, - leagueid: match.leagueid, + isProMatch: match.isProMatch, }, { lifo: options.lifo, diff --git a/svc/parser.js b/svc/parser.js index 5cff09a0c..1228af883 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -267,7 +267,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.leagueid ? processReduce(entries, match, meta) : undefined; + var logs = match.isProMatch ? processReduce(entries, match, meta) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); diff --git a/svc/scanner.js b/svc/scanner.js index 00d6e3215..fa15a8f35 100644 --- a/svc/scanner.js +++ b/svc/scanner.js @@ -112,12 +112,7 @@ function start() function processMatch(match, cb) { - if (config.ENABLE_PRO_PARSING && match.leagueid) - { - //parse tournament games - match.parse_status = 0; - } - else if (match.players.some(function(p) + if (match.players.some(function(p) { return (p.account_id in trackedPlayers); })) diff --git a/svc/worker.js b/svc/worker.js index 4c492994e..ee1cd62f4 100644 --- a/svc/worker.js +++ b/svc/worker.js @@ -124,6 +124,36 @@ invokeInterval(function notablePlayers(cb) }, cb); }); }, 10 * 60 * 1000); +invokeInterval(function leagues(cb) +{ + utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues) + { + if (err) + { + return cb(err); + } + utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/items_game.json', function(err, items) + { + if (err) + { + return cb(err); + } + var arr = []; + for (var key in leagues) + { + arr.push( + { + leagueid: key, + ticket: leagues[key].ticket, + banner: leagues[key].banner, + tier: leagues[key].tier, + name: items.items_game.items[key].name + }); + } + db('leagues').insert(arr).asCallback(cb); + }); + }); +}, 10 * 60 * 1000); function invokeInterval(func, delay) { From a628f581303fc556ad2bae23dbe22e86bcdd2e6b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 01:06:27 +0000 Subject: [PATCH 08/25] add leagues endpoint --- routes/api.js | 11 +++++++++++ sql/create_tables.sql | 4 ++-- store/queries.js | 8 +++++--- svc/worker.js | 33 ++++++++++++++++++++------------- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/routes/api.js b/routes/api.js index 475d2eb85..d13e75341 100644 --- a/routes/api.js +++ b/routes/api.js @@ -145,6 +145,17 @@ module.exports = function(db, redis, cassandra) res.json(result.rows); }); }); + api.get('/leagues', function(req, res, cb) + { + db.raw(`SELECT * FROM leagues ORDER BY leagueid`).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); api.get('/distributions', function(req, res, cb) { queries.getDistributions(redis, function(err, result) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 8d1564d77..7eaa7571a 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -227,8 +227,8 @@ CREATE TABLE picks_bans( is_pick boolean, hero_id int, team smallint, - order smallint, - PRIMARY KEY (match_id, order) + ord smallint, + PRIMARY KEY (match_id, ord) ); CREATE TABLE leagues( diff --git a/store/queries.js b/store/queries.js index f082aac51..50544476f 100644 --- a/store/queries.js +++ b/store/queries.js @@ -209,9 +209,11 @@ function insertMatch(db, redis, match, options, cb) function isProMatch(cb) { - //TODO check redis/postgres for professional/premium league - match.isProMatch = match.leagueid >= 0; - cb(); + redis.sismember('pro_leagueids', match.league_id, function(err, result) + { + match.isProMatch = Boolean(Number(result)); + cb(err); + }); } function upsertMatch(cb) diff --git a/svc/worker.js b/svc/worker.js index ee1cd62f4..4e3df510e 100644 --- a/svc/worker.js +++ b/svc/worker.js @@ -126,31 +126,38 @@ invokeInterval(function notablePlayers(cb) }, 10 * 60 * 1000); invokeInterval(function leagues(cb) { - utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues) + var container = utility.generateJob("api_leagues", + {}); + utility.getData(container.url, function(err, api_leagues) { if (err) { return cb(err); } - utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/items_game.json', function(err, items) + utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues) { if (err) { return cb(err); } - var arr = []; - for (var key in leagues) + async.each(api_leagues.result.leagues, function(l, cb) { - arr.push( + if (leagues[l.leagueid]) { - leagueid: key, - ticket: leagues[key].ticket, - banner: leagues[key].banner, - tier: leagues[key].tier, - name: items.items_game.items[key].name - }); - } - db('leagues').insert(arr).asCallback(cb); + l.tier = leagues[l.leagueid].tier; + l.ticket = leagues[l.leagueid].ticket; + l.banner = leagues[l.leagueid].banner; + } + l.name = l.description.substring("#DOTA_Item_Desc_".length).split('_').join(' '); + if (l.tier === "professional" || l.tier === "premium") + { + redis.sadd('pro_leagueids', l.leagueid); + } + queries.upsert(db, 'leagues', l, + { + leagueid: l.league_id + }, cb); + }, cb); }); }); }, 10 * 60 * 1000); From 8b77717f2b0c7caa2fbc2723d93bec4af56df9df Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 02:55:24 +0000 Subject: [PATCH 09/25] fix leagueid prop name --- config.js | 1 - store/queries.js | 15 +++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/config.js b/config.js index 0ee61c505..beccd9af9 100644 --- a/config.js +++ b/config.js @@ -54,7 +54,6 @@ var defaults = { "UI_HOST": "", //The host of the UI, redirect traffic from / and /return here "ENABLE_RECAPTCHA": "", //set to enable the recaptcha on the Request page "ENABLE_ADS": "", //set to turn on ads - "ENABLE_PRO_PARSING": "", // set to parse pro matches from sequential API "ENABLE_MATCH_CACHE": "", // set to enable caching matches (Redis) "ENABLE_PLAYER_CACHE": "", // set to enable caching players (Cassandra) "ENABLE_INSERT_ALL_MATCHES": "", //set to enable inserting all matches diff --git a/store/queries.js b/store/queries.js index 50544476f..189535a2b 100644 --- a/store/queries.js +++ b/store/queries.js @@ -209,11 +209,18 @@ function insertMatch(db, redis, match, options, cb) function isProMatch(cb) { - redis.sismember('pro_leagueids', match.league_id, function(err, result) + if (match.leagueid) { - match.isProMatch = Boolean(Number(result)); - cb(err); - }); + redis.sismember('pro_leagueids', match.leagueid, function(err, result) + { + match.isProMatch = Boolean(Number(result)); + cb(err); + }); + } + else + { + cb(); + } } function upsertMatch(cb) From b1ee5ea3897e3ab1c55aea49b27cdbaef1ed7298 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 03:03:37 +0000 Subject: [PATCH 10/25] transact match log insert --- store/queries.js | 75 +++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/store/queries.js b/store/queries.js index 189535a2b..0df215242 100644 --- a/store/queries.js +++ b/store/queries.js @@ -194,7 +194,6 @@ function insertMatch(db, redis, match, options, cb) "ipm": isProMatch, "u": upsertMatch, "uc": upsertMatchCassandra, - "uml": upsertMatchLogs, "upc": updatePlayerCaches, "uct": updateCounts, "cmc": clearMatchCache, @@ -233,12 +232,13 @@ function insertMatch(db, redis, match, options, cb) { async.series( { - "im": insertMatch, - "ipm": insertPlayerMatches, - "ipb": insertPicksBans + "m": upsertMatch, + "pm": upsertPlayerMatches, + "pb": upsertPicksBans, + "l": upsertMatchLogs, }, exit); - function insertMatch(cb) + function upsertMatch(cb) { upsert(trx, 'matches', match, { @@ -246,7 +246,7 @@ function insertMatch(db, redis, match, options, cb) }, cb); } - function insertPlayerMatches(cb) + function upsertPlayerMatches(cb) { async.each(players || [], function(pm, cb) { @@ -259,7 +259,7 @@ function insertMatch(db, redis, match, options, cb) }, cb); } - function insertPicksBans(cb) + function upsertPicksBans(cb) { async.each(match.picks_bans || [], function(p, cb) { @@ -274,6 +274,28 @@ function insertMatch(db, redis, match, options, cb) }, cb); } + function upsertMatchLogs(cb) + { + if (!match.logs) + { + return cb(); + } + else + { + trx.raw(`DELETE FROM match_logs WHERE match_id = ?`, [match.match_id]).asCallback(function(err) + { + if (err) + { + return exit(err); + } + async.eachLimit(match.logs, 100, function(e, cb) + { + trx('match_logs').insert(e).asCallback(cb); + }, cb); + }); + } + } + function exit(err) { if (err) @@ -351,45 +373,6 @@ function insertMatch(db, redis, match, options, cb) }); } - function upsertMatchLogs(cb) - { - if (!match.logs) - { - return cb(); - } - else - { - db.transaction(function(trx) - { - trx.raw(`DELETE FROM match_logs where match_id = ?`, [match.match_id]).asCallback(function(err) - { - if (err) - { - return exit(err); - } - async.eachLimit(match.logs, 10000, function(e, cb) - { - trx('match_logs').insert(e).asCallback(cb); - }, exit); - }); - - function exit(err) - { - if (err) - { - console.error(err); - trx.rollback(err); - } - else - { - trx.commit(); - } - cb(err); - } - }); - } - } - function updatePlayerCaches(cb) { var copy = createMatchCopy(match, players, options); From 48b06f12f9e3b0f8f3703d476f428cc3bcfb0587 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:03:50 +0000 Subject: [PATCH 11/25] remove parse_status --- store/queries.js | 15 ++++++++------- svc/parser.js | 3 +-- svc/requests.js | 2 +- svc/scanner.js | 12 +++++++----- tasks/getMatches.js | 2 +- views/match/match.jade | 1 - 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/store/queries.js b/store/queries.js index 0df215242..768a9af2f 100644 --- a/store/queries.js +++ b/store/queries.js @@ -191,7 +191,7 @@ function insertMatch(db, redis, match, options, cb) //we want to insert into matches, then insert into player_matches for each entry in players async.series( { - "ipm": isProMatch, + "dlp": decideLogParse, "u": upsertMatch, "uc": upsertMatchCassandra, "upc": updatePlayerCaches, @@ -205,14 +205,14 @@ function insertMatch(db, redis, match, options, cb) { return cb(err, results.dp); }); - - function isProMatch(cb) + + function decideLogParse(cb) { if (match.leagueid) { redis.sismember('pro_leagueids', match.leagueid, function(err, result) { - match.isProMatch = Boolean(Number(result)); + options.doLogParse = options.doLogParse || Boolean(Number(result)); cb(err); }); } @@ -224,7 +224,7 @@ function insertMatch(db, redis, match, options, cb) function upsertMatch(cb) { - if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.isProMatch) + if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !options.doLogParse) { return cb(); } @@ -458,9 +458,10 @@ function insertMatch(db, redis, match, options, cb) }, cb); } + function decideParse(cb) { - if (match.parse_status !== 0) + if (options.skipParse) { //not parsing this match //this isn't a error, although we want to report that we refused to parse back to user if it was a request @@ -477,7 +478,7 @@ function insertMatch(db, redis, match, options, cb) duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, - isProMatch: match.isProMatch, + doLogParse: options.doLogParse, }, { lifo: options.lifo, diff --git a/svc/parser.js b/svc/parser.js index 1228af883..586c8773e 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -91,7 +91,6 @@ pQueue.process(config.PARSER_PARALLELISM, function(job, cb) parsed_data.start_time = match.start_time; parsed_data.duration = match.duration; parsed_data.replay_blob_key = match.replay_blob_key; - parsed_data.parse_status = 2; if (match.replay_blob_key) { insertUploadedParse(parsed_data, cb); @@ -267,7 +266,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.isProMatch ? processReduce(entries, match, meta) : undefined; + var logs = match.doLogParse ? processReduce(entries, match, meta) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); diff --git a/svc/requests.js b/svc/requests.js index c6007a733..08901defb 100644 --- a/svc/requests.js +++ b/svc/requests.js @@ -29,13 +29,13 @@ function processRequest(job, cb) } //match details response var match = body.result; - match.parse_status = 0; insertMatch(db, redis, match, { type: "api", attempts: 1, lifo: true, cassandra: cassandra, + skipParse: false, }, waitParse); }); } diff --git a/svc/scanner.js b/svc/scanner.js index fa15a8f35..ee8d86348 100644 --- a/svc/scanner.js +++ b/svc/scanner.js @@ -112,21 +112,22 @@ function start() function processMatch(match, cb) { + var insert = false; + var skipParse = true; if (match.players.some(function(p) { return (p.account_id in trackedPlayers); })) { - //queued - match.parse_status = 0; + insert = true; + skipParse = false; } else if (match.players.some(function(p) { return (config.ENABLE_INSERT_ALL_MATCHES || p.account_id in userPlayers); })) { - //skipped - match.parse_status = 3; + insert = true; } //check if match was previously processed redis.get('scanner_insert:' + match.match_id, function(err, result) @@ -137,7 +138,7 @@ function start() } //don't insert this match if we already processed it recently //deduplicate matches in this page set - if ((match.parse_status === 0 || match.parse_status === 3) && !result && !matchBuffer[match.match_id]) + if (insert && !result && !matchBuffer[match.match_id]) { matchBuffer[match.match_id] = 1; insertMatch(db, redis, match, @@ -146,6 +147,7 @@ function start() origin: "scanner", cassandra: cassandra, userPlayers: userPlayers, + skipParse: skipParse, }, function(err) { if (!err) diff --git a/tasks/getMatches.js b/tasks/getMatches.js index cf2d7fd92..0362a1b95 100644 --- a/tasks/getMatches.js +++ b/tasks/getMatches.js @@ -28,11 +28,11 @@ getData( if (body.result) { var match = body.result; - match.parse_status = 0; insertMatch(db, redis, match, { skipCounts: true, skipAbilityUpgrades: true, + skipParse: false, cassandra: cassandra, }, function(err) { diff --git a/views/match/match.jade b/views/match/match.jade index 53673b8d2..6d0a345d7 100644 --- a/views/match/match.jade +++ b/views/match/match.jade @@ -32,7 +32,6 @@ block content th Percentile tbody tr - //td= constants.parse_status[match.parse_status] td= constants.game_mode[match.game_mode] ? constants.game_mode[match.game_mode].name : match.game_mode td= constants.region[match.region] ? constants.region[match.region] : match.region td= moment().startOf('day').seconds(match.duration).format("H:mm:ss") From a95f2c3bf06b34035e32dd47a69eb4cebb032c1d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:17:43 +0000 Subject: [PATCH 12/25] add endpoint to list pro matches --- routes/api.js | 20 +++++++++++++++++++- sql/create_tables.sql | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index d13e75341..4a3311210 100644 --- a/routes/api.js +++ b/routes/api.js @@ -136,7 +136,25 @@ module.exports = function(db, redis, cassandra) */ api.get('/match_logs/:match_id', function(req, res, cb) { - db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result) + db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time ASC`, [req.params.match_id]).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); + api.get('/pro_matches', function(req, res, cb) + { + db.raw(` + SELECT match_id, start_time, duration, ma.leagueid, name + FROM matches ma + JOIN leagues le + ON ma.leagueid = le.leagueid + WHERE ma.leagueid > 0 + ORDER BY match_id DESC + `).asCallback(function(err, result) { if (err) { diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 7eaa7571a..936d49041 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -39,6 +39,7 @@ CREATE TABLE matches ( teamfights json[], version integer ); +CREATE INDEX on matches(leagueid) WHERE leagueid > 0; CREATE TABLE player_matches ( PRIMARY KEY(match_id, player_slot), From 77b4b90b89c029f6621fffce1af610cd66ed9d66 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:23:51 +0000 Subject: [PATCH 13/25] fix double parse --- svc/parser.js | 1 + 1 file changed, 1 insertion(+) diff --git a/svc/parser.js b/svc/parser.js index 586c8773e..ac5aa4864 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -157,6 +157,7 @@ function insertStandardParse(match, cb) { type: "parsed", cassandra: cassandra, + skipParse: true, }, cb); } From a46b6b8419988a2fd0a021605e33ae13f76d137e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:50:27 +0000 Subject: [PATCH 14/25] fix insert call in test --- svc/fullhistory.js | 1 + tasks/getMatchesSeq.js | 1 + test/test.js | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/svc/fullhistory.js b/svc/fullhistory.js index 312034a51..d5707ac4a 100644 --- a/svc/fullhistory.js +++ b/svc/fullhistory.js @@ -90,6 +90,7 @@ function processFullHistory(job, cb) type: "api", cassandra: cassandra, skipAbilityUpgrades: true, + skipParse: true, }, cb); }); }, function(err) diff --git a/tasks/getMatchesSeq.js b/tasks/getMatchesSeq.js index ebd445e36..331c5bb1c 100644 --- a/tasks/getMatchesSeq.js +++ b/tasks/getMatchesSeq.js @@ -86,6 +86,7 @@ function getPage(match_seq_num, bucket) { skipCounts: true, skipAbilityUpgrades: true, + skipParse: true, cassandra: cassandra, }, cb); }, function(err) diff --git a/test/test.js b/test/test.js index 73af32d1d..e6e4cebd7 100644 --- a/test/test.js +++ b/test/test.js @@ -128,7 +128,8 @@ before(function(done) { queries.insertMatch(db, redis, m, { - type: "api" + type: "api", + skipParse: true, }, cb); }, cb); }, From afa5e6167a65b8ac5a8596b2c52970d11b6f4a85 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 20 Jun 2016 00:54:20 +0000 Subject: [PATCH 15/25] remove userplayers --- store/buildSets.js | 14 -------------- store/queries.js | 9 +-------- svc/scanner.js | 8 +------- 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/store/buildSets.js b/store/buildSets.js index fed99d429..2c3794d77 100644 --- a/store/buildSets.js +++ b/store/buildSets.js @@ -18,20 +18,6 @@ module.exports = function buildSets(db, redis, cb) { cb(err, t); }); }, - //users in this set have their matches added - "userPlayers": function(cb) { - db.select(['account_id']).from('players').whereNotNull('last_login').asCallback(function(err, docs) { - if (err) { - return cb(err); - } - var t = {}; - docs.forEach(function(player) { - t[player.account_id] = true; - }); - //console.log(t); - cb(err, t); - }); - }, //users in this set are added to the trackedPlayers set "donators": function(cb) { db.select(['account_id']).from('players').where('cheese', '>', 0).asCallback(function(err, docs) { diff --git a/store/queries.js b/store/queries.js index 768a9af2f..49974199e 100644 --- a/store/queries.js +++ b/store/queries.js @@ -32,13 +32,6 @@ function getSets(redis, cb) cb(err, JSON.parse(tps || "{}")); }); }, - "userPlayers": function(cb) - { - redis.get("userPlayers", function(err, ups) - { - cb(err, JSON.parse(ups || "{}")); - }); - }, "donators": function(cb) { redis.get("donators", function(err, ds) @@ -426,7 +419,7 @@ function insertMatch(db, redis, match, options, cb) { async.each(match.players, function(p, cb) { - if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && (p.account_id in options.userPlayers || (config.ENABLE_RANDOM_MMR_UPDATE && match.match_id % 3 === 0))) + if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && config.ENABLE_RANDOM_MMR_UPDATE) { addToQueue(mQueue, { diff --git a/svc/scanner.js b/svc/scanner.js index ee8d86348..819db762d 100644 --- a/svc/scanner.js +++ b/svc/scanner.js @@ -13,7 +13,6 @@ var getData = utility.getData; var generateJob = utility.generateJob; var async = require('async'); var trackedPlayers; -var userPlayers; var parallelism = config.SCANNER_PARALLELISM; var PAGE_SIZE = 100; buildSets(db, redis, function(err) @@ -72,7 +71,6 @@ function start() } //set local vars trackedPlayers = result.trackedPlayers; - userPlayers = result.userPlayers; var arr = []; var matchBuffer = {}; var completePages = {}; @@ -122,10 +120,7 @@ function start() insert = true; skipParse = false; } - else if (match.players.some(function(p) - { - return (config.ENABLE_INSERT_ALL_MATCHES || p.account_id in userPlayers); - })) + else if (config.ENABLE_INSERT_ALL_MATCHES) { insert = true; } @@ -146,7 +141,6 @@ function start() type: "api", origin: "scanner", cassandra: cassandra, - userPlayers: userPlayers, skipParse: skipParse, }, function(err) { From 41b115c7a3eb3dcc07cd193641e866e6eaa7ff44 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 20 Jun 2016 01:19:10 +0000 Subject: [PATCH 16/25] update knex, remove modifier_remove events from log --- package.json | 2 +- processors/processReduce.js | 4 ++++ store/queries.js | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index b88d0ee1f..011261da3 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "http-proxy": "^1.13.1", "jade": "^1.11.0", "jquery": "^2.2.0", - "knex": "^0.10.0", + "knex": "^0.11.7", "moment": "^2.11.2", "multer": "^1.1.0", "numeral": "^1.5.3", diff --git a/processors/processReduce.js b/processors/processReduce.js index 2bf9dbe00..3fd6823b2 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -9,6 +9,10 @@ function processReduce(entries, match, meta) { return false; } + if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE") + { + return false; + } if (e.type === "interval" && e.time % 60 !== 0) { return false; diff --git a/store/queries.js b/store/queries.js index 49974199e..05ecb64cd 100644 --- a/store/queries.js +++ b/store/queries.js @@ -279,9 +279,9 @@ function insertMatch(db, redis, match, options, cb) { if (err) { - return exit(err); + return cb(err); } - async.eachLimit(match.logs, 100, function(e, cb) + async.eachLimit(match.logs, 10000, function(e, cb) { trx('match_logs').insert(e).asCallback(cb); }, cb); From 0165e1e4fa563a05e7bf41c12c4e3dadc31ad789 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 22 Jun 2016 08:10:10 +0000 Subject: [PATCH 17/25] remove non-hero logs --- processors/processReduce.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/processors/processReduce.js b/processors/processReduce.js index 3fd6823b2..434c032e9 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -13,6 +13,13 @@ function processReduce(entries, match, meta) { return false; } + if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL") + { + if (!e.targethero || e.targetillusion) + { + return false; + } + } if (e.type === "interval" && e.time % 60 !== 0) { return false; From 74f2640382a9fb07f67c82d674bd8863eda7dbde Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 22 Jun 2016 09:07:43 +0000 Subject: [PATCH 18/25] extract ward entityLeft events --- java_parser/src/main/java/yasp/Main.java | 66 +++++++++++++++--------- processors/populate.js | 3 ++ processors/processExpand.js | 24 +++++---- sql/cassandra.cql | 1 + sql/create_tables.sql | 3 +- util/analysis.js | 20 +++++-- util/utility.js | 1 + views/match/match_performances.jade | 4 -- views/match/match_vision.jade | 4 +- 9 files changed, 84 insertions(+), 42 deletions(-) diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java index f507f938f..b4ad1c54d 100644 --- a/java_parser/src/main/java/yasp/Main.java +++ b/java_parser/src/main/java/yasp/Main.java @@ -10,6 +10,7 @@ import skadistats.clarity.model.s1.GameRulesStateType; import skadistats.clarity.processor.entities.Entities; import skadistats.clarity.processor.entities.OnEntityEntered; +import skadistats.clarity.processor.entities.OnEntityLeft; import skadistats.clarity.processor.entities.UsesEntities; import skadistats.clarity.processor.gameevents.CombatLog; import skadistats.clarity.processor.gameevents.OnCombatLogEntry; @@ -78,6 +79,7 @@ private class Entry { public Integer xp; public Integer x; public Integer y; + public Integer z; public Float stuns; public Integer hero_id; public Integer life_state; @@ -87,7 +89,9 @@ private class Entry { public Integer assists; public Integer denies; //public Boolean hasPredictedVictory; - + public Boolean entityleft; + public Integer ehandle; + public Entry() { } @@ -255,28 +259,12 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) { @OnEntityEntered public void onEntityEntered(Context ctx, Entity e) { - //CDOTA_NPC_Observer_Ward - //CDOTA_NPC_Observer_Ward_TrueSight - //s1 "DT_DOTA_NPC_Observer_Ward" - //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight" - boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward"); - boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight"); - if (isObserver || isSentry) { - //System.err.println(e); - Entry entry = new Entry(time); - Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null); - Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null); - Integer[] pos = {x, y}; - entry.type = isObserver ? "obs" : "sen"; - entry.key = Arrays.toString(pos); - //System.err.println(entry.key); - Integer owner = getEntityProperty(e, "m_hOwnerEntity", null); - Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner); - entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null; - //2/3 radiant/dire - //entry.team = e.getProperty("m_iTeamNum"); - output(entry); - } + processWardEntity(ctx, e, false); + } + + @OnEntityLeft + public void onEntityLeft(Context ctx, Entity e) { + processWardEntity(ctx, e, true); } @UsesEntities @@ -448,6 +436,38 @@ public T getEntityProperty(Entity e, String property, Integer idx) { FieldPath fp = e.getDtClass().getFieldPathForName(property); return e.getPropertyForFieldPath(fp); } + + public void processWardEntity(Context ctx, Entity e, boolean entityLeft) + { + //CDOTA_NPC_Observer_Ward + //CDOTA_NPC_Observer_Ward_TrueSight + //s1 "DT_DOTA_NPC_Observer_Ward" + //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight" + boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward"); + boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight"); + if (isObserver || isSentry) { + //System.err.println(e); + Entry entry = new Entry(time); + Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null); + Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null); + Integer z = getEntityProperty(e, "CBodyComponent.m_cellZ", null); + Integer[] pos = {x, y}; + entry.x = x; + entry.y = y; + entry.z = z; + entry.type = isObserver ? "obs" : "sen"; + entry.key = Arrays.toString(pos); + entry.entityleft = entityLeft; + entry.ehandle = e.getHandle(); + //System.err.println(entry.key); + Integer owner = getEntityProperty(e, "m_hOwnerEntity", null); + Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner); + entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null; + //2/3 radiant/dire + //entry.team = e.getProperty("m_iTeamNum"); + output(entry); + } + } public void run(String[] args) throws Exception { long tStart = System.currentTimeMillis(); diff --git a/processors/populate.js b/processors/populate.js index 39d61bc47..b1eda8df9 100644 --- a/processors/populate.js +++ b/processors/populate.js @@ -7,6 +7,9 @@ function populate(e, container) case 'player_slot': container.players[e.key].player_slot = e.value; break; + case 'log': + container.log.push(JSON.parse(JSON.stringify(e))); + break; case 'chat': container.chat.push(JSON.parse(JSON.stringify(e))); break; diff --git a/processors/processExpand.js b/processors/processExpand.js index c91c15ca3..ef10789b0 100644 --- a/processors/processExpand.js +++ b/processors/processExpand.js @@ -425,24 +425,30 @@ function processExpand(entries, meta) }, "obs": function(e) { - //key is a JSON array of position data - e.key = JSON.parse(e.key); - e.posData = true; - expand(e); var e2 = JSON.parse(JSON.stringify(e)); - e2.posData = false; e2.type = "obs_log"; expand(e2); + var e4 = JSON.parse(JSON.stringify(e)); + e4.type = "log"; + expand(e4); + var e3 = JSON.parse(JSON.stringify(e)); + //key is a JSON array of position data + e3.key = JSON.parse(e3.key); + e3.posData = true; + expand(e3); }, "sen": function(e) { - e.key = JSON.parse(e.key); - e.posData = true; - expand(e); var e2 = JSON.parse(JSON.stringify(e)); - e2.posData = false; e2.type = "sen_log"; expand(e2); + var e4 = JSON.parse(JSON.stringify(e)); + e4.type = "log"; + expand(e4); + var e3 = JSON.parse(JSON.stringify(e)); + e3.key = JSON.parse(e3.key); + e3.posData = true; + expand(e3); } }; //define the types we want to put into each array diff --git a/sql/cassandra.cql b/sql/cassandra.cql index 4c1572049..6de8d3110 100644 --- a/sql/cassandra.cql +++ b/sql/cassandra.cql @@ -26,6 +26,7 @@ CREATE TABLE matches ( radiant_xp_adv text, teamfights text, version int, + log text, ); CREATE TABLE player_matches ( diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 134bda3f7..8c23cdad4 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -37,7 +37,8 @@ CREATE TABLE matches ( radiant_gold_adv integer[], radiant_xp_adv integer[], teamfights json[], - version integer + version integer, + log json[] ); CREATE TABLE player_matches ( diff --git a/util/analysis.js b/util/analysis.js index 8f9f2f163..6aca06a63 100644 --- a/util/analysis.js +++ b/util/analysis.js @@ -4,6 +4,7 @@ **/ var util = require('util'); var constants = require('../constants.js'); + function generatePlayerAnalysis(match, pm) { //define condition check for each advice point @@ -150,7 +151,8 @@ function generatePlayerAnalysis(match, pm) { var flying_available = 180; var time; - if (pm.purchase && pm.first_purchase_time && pm.first_purchase_time.flying_courier) { + if (pm.purchase && pm.first_purchase_time && pm.first_purchase_time.flying_courier) + { time = pm.first_purchase_time.flying_courier; } return { @@ -172,7 +174,7 @@ function generatePlayerAnalysis(match, pm) wards: function(m, pm) { var ward_cooldown = 60 * 7; - var wards = pm.obs_log ? pm.obs_log.length : 0; + var wards = getObsWardsPlaced(pm); //divide game length by ward cooldown //2 wards respawn every interval //split responsibility between 2 supports @@ -304,7 +306,19 @@ function generatePlayerAnalysis(match, pm) function isSupport(pm) { - return pm.obs_log && pm.obs_log.length >= 2 && pm.lh_t && pm.lh_t[10] < 20; + return getObsWardsPlaced(pm) >= 2 && pm.lh_t && pm.lh_t[10] < 20; + } + + function getObsWardsPlaced(pm) + { + if (!pm.obs_log) + { + return 0; + } + return pm.obs_log.filter(function(l) + { + return !l.entityleft; + }).length; } function isRoshHero(pm) diff --git a/util/utility.js b/util/utility.js index ba4d11fb1..3d436b320 100644 --- a/util/utility.js +++ b/util/utility.js @@ -357,6 +357,7 @@ function getParseSchema() "teamfights": [], "objectives": [], "chat": [], + "log": [], "radiant_gold_adv": [], "radiant_xp_adv": [], "players": Array.apply(null, new Array(10)).map(function() diff --git a/views/match/match_performances.jade b/views/match/match_performances.jade index b78029092..2edb420df 100644 --- a/views/match/match_performances.jade +++ b/views/match/match_performances.jade @@ -9,8 +9,6 @@ block match_content tr th: abbr(title=tooltips.hero_id) Hero th: abbr(title=tooltips.lane) Lane - //th: abbr(title=tooltips.used_ward_observer) Observer - //th: abbr(title=tooltips.used_ward_sentry) Sentry th: abbr(title=tooltips.lane_efficiency) EFF@10 th: abbr(title=tooltips.lhten) LH@10 th: abbr(title=tooltips.stuns) Stuns @@ -19,8 +17,6 @@ block match_content tr.activate(data-index=i, data-type="lane_pos", class = player.isRadiant ? "radiant" : "dire") +hero_td(player) td #{constants.lane_ids[player.lane] || "-"} (#{constants.lane_role[player.lane_role] || "-"}) - //td.activate(data-index=i, data-type="obs") #{player.obs_log.length || "-"} - //td.activate(data-index=i, data-type="sen") #{player.sen_log.length || "-"} td.rankable #{(Number(player.lane_efficiency)*100).toFixed(2)} td.rankable #{player.lh_t && player.lh_t[10] ? player.lh_t[10] : "-"} td.rankable #{Number(Math.max(player.stuns, 0)).toFixed(2)} diff --git a/views/match/match_vision.jade b/views/match/match_vision.jade index 34d66137a..fb9e9a569 100644 --- a/views/match/match_vision.jade +++ b/views/match/match_vision.jade @@ -14,7 +14,7 @@ block match_content tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="obs").activate +hero_td(player) td.rankable.lane-position #{player.purchase_ward_observer || "-"} - td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.length : "-"} + td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.filter(function(l){return !l.entityleft;}).length : "-"} td.rankable.lane-position #{player.observer_kills || "-"} .col-md-4 @@ -29,7 +29,7 @@ block match_content tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="sen").activate +hero_td(player) td.rankable.lane-position #{player.purchase_ward_sentry || "-"} - td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.length : "-"} + td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.filter(function(l){return !l.entityleft;}).length : "-"} td.rankable.lane-position #{player.sentry_kills || "-"} .col-md-4 From 254cd9947b34840f694658f28e6f8b52a2a2c4cd Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Jun 2016 03:22:36 +0000 Subject: [PATCH 19/25] reuse log --- java_parser/src/main/java/yasp/Main.java | 9 ++++++++- processors/populate.js | 4 ---- processors/processExpand.js | 8 +------- sql/cassandra.cql | 3 +-- sql/create_tables.sql | 3 +-- util/utility.js | 1 - 6 files changed, 11 insertions(+), 17 deletions(-) diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java index b4ad1c54d..2c520f46f 100644 --- a/java_parser/src/main/java/yasp/Main.java +++ b/java_parser/src/main/java/yasp/Main.java @@ -455,7 +455,14 @@ public void processWardEntity(Context ctx, Entity e, boolean entityLeft) entry.x = x; entry.y = y; entry.z = z; - entry.type = isObserver ? "obs" : "sen"; + if (entityLeft) + { + entry.type = isObserver ? "obs_left" : "sen_left"; + } + else + { + entry.type = isObserver ? "obs" : "sen"; + } entry.key = Arrays.toString(pos); entry.entityleft = entityLeft; entry.ehandle = e.getHandle(); diff --git a/processors/populate.js b/processors/populate.js index b1eda8df9..35581c816 100644 --- a/processors/populate.js +++ b/processors/populate.js @@ -7,9 +7,6 @@ function populate(e, container) case 'player_slot': container.players[e.key].player_slot = e.value; break; - case 'log': - container.log.push(JSON.parse(JSON.stringify(e))); - break; case 'chat': container.chat.push(JSON.parse(JSON.stringify(e))); break; @@ -92,5 +89,4 @@ function populate(e, container) break; } } - module.exports = populate; \ No newline at end of file diff --git a/processors/processExpand.js b/processors/processExpand.js index ef10789b0..f8395d753 100644 --- a/processors/processExpand.js +++ b/processors/processExpand.js @@ -428,9 +428,6 @@ function processExpand(entries, meta) var e2 = JSON.parse(JSON.stringify(e)); e2.type = "obs_log"; expand(e2); - var e4 = JSON.parse(JSON.stringify(e)); - e4.type = "log"; - expand(e4); var e3 = JSON.parse(JSON.stringify(e)); //key is a JSON array of position data e3.key = JSON.parse(e3.key); @@ -442,14 +439,11 @@ function processExpand(entries, meta) var e2 = JSON.parse(JSON.stringify(e)); e2.type = "sen_log"; expand(e2); - var e4 = JSON.parse(JSON.stringify(e)); - e4.type = "log"; - expand(e4); var e3 = JSON.parse(JSON.stringify(e)); e3.key = JSON.parse(e3.key); e3.posData = true; expand(e3); - } + }, }; //define the types we want to put into each array //null means all types diff --git a/sql/cassandra.cql b/sql/cassandra.cql index 6de8d3110..5b075cbf6 100644 --- a/sql/cassandra.cql +++ b/sql/cassandra.cql @@ -25,8 +25,7 @@ CREATE TABLE matches ( radiant_gold_adv text, radiant_xp_adv text, teamfights text, - version int, - log text, + version int ); CREATE TABLE player_matches ( diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 8c23cdad4..134bda3f7 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -37,8 +37,7 @@ CREATE TABLE matches ( radiant_gold_adv integer[], radiant_xp_adv integer[], teamfights json[], - version integer, - log json[] + version integer ); CREATE TABLE player_matches ( diff --git a/util/utility.js b/util/utility.js index 3d436b320..ba4d11fb1 100644 --- a/util/utility.js +++ b/util/utility.js @@ -357,7 +357,6 @@ function getParseSchema() "teamfights": [], "objectives": [], "chat": [], - "log": [], "radiant_gold_adv": [], "radiant_xp_adv": [], "players": Array.apply(null, new Array(10)).map(function() From e1eeb51e31fd1fe7fc2bab75d731c6693b6f330f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Jun 2016 03:32:36 +0000 Subject: [PATCH 20/25] add new columns to log schema --- processors/processReduce.js | 47 +++++++++++++++++++++++-------------- sql/create_tables.sql | 3 +++ svc/parser.js | 4 ++-- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/processors/processReduce.js b/processors/processReduce.js index 434c032e9..d0865c998 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -3,32 +3,45 @@ **/ function processReduce(entries, match, meta) { + var basicLogTypes = { + "obs": 1, + "sen": 1, + "obs_left": 1, + "sen_left": 1, + }; var result = entries.filter(function(e) { - if (e.type === "actions") + if (match.doLogParse) { - return false; - } - if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE") - { - return false; - } - if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL") - { - if (!e.targethero || e.targetillusion) + if (e.type === "actions") { return false; } + if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE") + { + return false; + } + if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL") + { + if (!e.targethero || e.targetillusion) + { + return false; + } + } + if (e.type === "interval" && e.time % 60 !== 0) + { + return false; + } + if (!e.time) + { + return false; + } + return true; } - if (e.type === "interval" && e.time % 60 !== 0) - { - return false; - } - if (!e.time) + else { - return false; + return (e.type in basicLogTypes); } - return true; }).map(function(e) { var e2 = Object.assign( diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 936d49041..2588c01b6 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -201,6 +201,9 @@ CREATE TABLE match_logs ( xp int, x smallint, y smallint, + z smallint, + entityleft boolean, + ehandle int, stuns real, hero_id smallint, life_state smallint, diff --git a/svc/parser.js b/svc/parser.js index ac5aa4864..906fa5069 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -245,7 +245,7 @@ function runParse(match, job, cb) } entries.push(e); }); - request.debug = true + request.debug = true; function exit(err) { @@ -267,7 +267,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.doLogParse ? processReduce(entries, match, meta) : undefined; + var logs = processReduce(entries, match, meta); var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); From 0cdda2a47648c70109b2bc4d5edecd0b7e0a4084 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Jun 2016 04:31:52 +0000 Subject: [PATCH 21/25] use null for undefined account_ids in pgroup --- store/queries.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/store/queries.js b/store/queries.js index 05ecb64cd..d6fd4282b 100644 --- a/store/queries.js +++ b/store/queries.js @@ -124,7 +124,7 @@ function upsert(db, table, row, conflict, cb) return util.format("%s=%s", key, "EXCLUDED." + key); }); var query = util.format("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (%s) DO UPDATE SET %s", table, Object.keys(row).join(','), values, Object.keys(conflict).join(','), update.join(',')); - //require('fs').writeFileSync('output.json', query); + //console.log(query.toString()); db.raw(query, Object.keys(row).map(function(key) { return row[key]; @@ -161,7 +161,7 @@ function insertMatch(db, redis, match, options, cb) players.forEach(function(p, i) { match.pgroup[p.player_slot] = { - account_id: p.account_id, + account_id: p.account_id || null, hero_id: p.hero_id, player_slot: p.player_slot }; @@ -198,7 +198,7 @@ function insertMatch(db, redis, match, options, cb) { return cb(err, results.dp); }); - + function decideLogParse(cb) { if (match.leagueid) @@ -451,7 +451,6 @@ function insertMatch(db, redis, match, options, cb) }, cb); } - function decideParse(cb) { if (options.skipParse) From a294c3de53ded1bbfbeea30df3e982404f7c6bb9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Jun 2016 04:39:55 +0000 Subject: [PATCH 22/25] undo match_vision change --- views/match/match_vision.jade | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/views/match/match_vision.jade b/views/match/match_vision.jade index fb9e9a569..34d66137a 100644 --- a/views/match/match_vision.jade +++ b/views/match/match_vision.jade @@ -14,7 +14,7 @@ block match_content tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="obs").activate +hero_td(player) td.rankable.lane-position #{player.purchase_ward_observer || "-"} - td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.filter(function(l){return !l.entityleft;}).length : "-"} + td.rankable.lane-position #{player.obs_log && player.obs_log.length ? player.obs_log.length : "-"} td.rankable.lane-position #{player.observer_kills || "-"} .col-md-4 @@ -29,7 +29,7 @@ block match_content tr(class = player.isRadiant ? "radiant" : "dire", data-index=ind, data-type="sen").activate +hero_td(player) td.rankable.lane-position #{player.purchase_ward_sentry || "-"} - td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.filter(function(l){return !l.entityleft;}).length : "-"} + td.rankable.lane-position #{player.sen_log && player.sen_log.length ? player.sen_log.length : "-"} td.rankable.lane-position #{player.sentry_kills || "-"} .col-md-4 From 31399c82a111951184b0c1dd7313d5e1f94ab7b9 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 23 Jun 2016 04:46:33 +0000 Subject: [PATCH 23/25] disable log parsing for regular matches --- processors/processReduce.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/processors/processReduce.js b/processors/processReduce.js index d0865c998..2d7f709a1 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -3,6 +3,11 @@ **/ function processReduce(entries, match, meta) { + //for now, disable log parsing for regular matches + if (!match.doLogParse) + { + return; + } var basicLogTypes = { "obs": 1, "sen": 1, From 690a2e9e8e3ec8c86e89f793842ebcde1b656d24 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 24 Jun 2016 08:47:22 +0000 Subject: [PATCH 24/25] add drafts/pro_players endpoints --- json/navbar_pages.json | 4 ---- routes/api.js | 49 +++++++++++++++++++++++++++++++++++++++++- store/queries.js | 16 ++++---------- svc/web.js | 12 +---------- 4 files changed, 53 insertions(+), 28 deletions(-) diff --git a/json/navbar_pages.json b/json/navbar_pages.json index 6ddb4efd9..d57cb79fa 100644 --- a/json/navbar_pages.json +++ b/json/navbar_pages.json @@ -9,10 +9,6 @@ "benchmarks": { "name": "Benchmarks" }, - "top": { - "name": "Top", - "hide": true - }, "distributions": { "name": "Distributions" }, diff --git a/routes/api.js b/routes/api.js index 4a3311210..e901914fd 100644 --- a/routes/api.js +++ b/routes/api.js @@ -134,6 +134,7 @@ module.exports = function(db, redis, cassandra) }); }); */ + /* api.get('/match_logs/:match_id', function(req, res, cb) { db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time ASC`, [req.params.match_id]).asCallback(function(err, result) @@ -145,6 +146,7 @@ module.exports = function(db, redis, cassandra) res.json(result.rows); }); }); + */ api.get('/pro_matches', function(req, res, cb) { db.raw(` @@ -163,9 +165,54 @@ module.exports = function(db, redis, cassandra) res.json(result.rows); }); }); + api.get('/pro_players', function(req, res, cb) + { + queries.getProPlayers(db, redis, function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result); + }); + }); + api.get('/drafts', function(req, res, cb) + { + db.raw(` + SELECT pb.hero_id, + sum(case when ((pm.player_slot < 128) = m.radiant_win) then 1 else 0 end) wins, + sum(case when is_pick is true then 1 else 0 end) picks, + sum(case when is_pick is false then 1 else 0 end) bans + FROM picks_bans pb + LEFT JOIN matches m + ON pb.match_id = m.match_id + LEFT JOIN player_matches pm + ON pb.hero_id = pm.hero_id + AND pm.match_id = m.match_id + GROUP BY pb.hero_id; + `).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); + api.get('/pick_order', function(req, res, cb) + { + db.raw(`SELECT hero_id, ord, count( * ) FROM picks_bans WHERE is_pick is true GROUP BY hero_id, ord;`).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); api.get('/leagues', function(req, res, cb) { - db.raw(`SELECT * FROM leagues ORDER BY leagueid`).asCallback(function(err, result) + db.raw(`SELECT * FROM leagues ORDER BY leagueid DESC`).asCallback(function(err, result) { if (err) { diff --git a/store/queries.js b/store/queries.js index 05ecb64cd..f2f59a8c6 100644 --- a/store/queries.js +++ b/store/queries.js @@ -198,7 +198,7 @@ function insertMatch(db, redis, match, options, cb) { return cb(err, results.dp); }); - + function decideLogParse(cb) { if (match.leagueid) @@ -451,7 +451,6 @@ function insertMatch(db, redis, match, options, cb) }, cb); } - function decideParse(cb) { if (options.skipParse) @@ -725,7 +724,7 @@ function expectedWin(rates) return 1 - rates.reduce((prev, curr) => (100 - curr * 100) * prev, 1) / (Math.pow(50, rates.length - 1) * 100); } -function getTop(db, redis, cb) +function getProPlayers(db, redis, cb) { db.raw(` SELECT * from notable_players @@ -735,14 +734,7 @@ function getTop(db, redis, cb) { return cb(err); } - getLeaderboard(db, redis, 'solo_competitive_rank', 500, function(err, result2) - { - return cb(err, - { - notables: result.rows, - leaderboard: result2 - }); - }); + return cb(err, result.rows); }); } @@ -964,7 +956,7 @@ module.exports = { insertMatchSkill, getDistributions, getPicks, - getTop, + getProPlayers, getHeroRankings, getBenchmarks, benchmarkMatch, diff --git a/svc/web.js b/svc/web.js index 13ebd837e..e3182328f 100644 --- a/svc/web.js +++ b/svc/web.js @@ -288,17 +288,6 @@ app.get('/picks/:n?', function(req, res, cb) }); }); }); -app.get('/top', function(req, res, cb) -{ - queries.getTop(db, redis, function(err, result) - { - if (err) - { - return cb(err); - } - res.render('top', result); - }); -}); app.get('/rankings/:hero_id?', function(req, res, cb) { if (!req.params.hero_id) @@ -388,6 +377,7 @@ app.use(function(req, res, next) { if (config.UI_HOST) { + //route not found, redirect to SPA return res.redirect(config.UI_HOST + req.url); } var err = new Error("Not Found"); From c76de71fb3251644bbed8ed2b7e79e42947508eb Mon Sep 17 00:00:00 2001 From: Howard Chung Date: Fri, 24 Jun 2016 19:50:38 -0700 Subject: [PATCH 25/25] Update create_tables.sql --- sql/create_tables.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 2588c01b6..cd234ab73 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -241,4 +241,4 @@ CREATE TABLE leagues( banner varchar(255), tier varchar(255), name varchar(255) -) \ No newline at end of file +);