From dc8d48bf528da1e6fd588146e8002edec81c4c20 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 11 Jun 2016 03:41:40 +0000 Subject: [PATCH 01/17] start implementation of log parsing --- processors/processExpand.js | 2 +- processors/processReduce.js | 22 ++++++++++++------- sql/create_tables.sql | 44 ++++++++++++++++++++++++++++++++++++- svc/parser.js | 3 ++- 4 files changed, 60 insertions(+), 11 deletions(-) diff --git a/processors/processExpand.js b/processors/processExpand.js index c91c15ca3..e67b97647 100644 --- a/processors/processExpand.js +++ b/processors/processExpand.js @@ -449,7 +449,7 @@ function processExpand(entries, meta) //null means all types var reqs = { parsed_data: null, - //expanded: null, + expanded: null, tf_data: { killed: 1, diff --git a/processors/processReduce.js b/processors/processReduce.js index 41493a8fa..f35890898 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -8,15 +8,21 @@ function processReduce(entries) //group by player_slot, type, targethero, targetillusion for (var i = 0; i < entries.length; i++) { - //group big categories: actions, combat log damage var e = entries[i]; - var identifier = [e.player_slot, e.type, e.key].join(":"); - e.value = e.value || 1; - //var identifier = e.type; - //e.value = 1; - reduceMap[identifier] = reduceMap[identifier] ? reduceMap[identifier] + e.value : e.value || 1; + reduceMap[e.type] = reduceMap[e.type] ? reduceMap[e.type] + 1 : 1; } - //var fs = require('fs'); - //fs.writeFileSync('./output3.json', JSON.stringify(reduceMap, null , 2)); + console.log(reduceMap); + return entries.filter(function(e) + { + if (e.type === "actions") + { + return false; + } + if (e.type === "interval" && e.time % 60 !== 0) + { + return false; + } + return true; + }); } module.exports = processReduce; \ No newline at end of file diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 80f3fec9d..72e5d6370 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -172,4 +172,46 @@ CREATE TABLE notable_players ( is_locked boolean, is_pro boolean, locked_until integer -); \ No newline at end of file +); + +CREATE TABLE match_logs ( + match_id bigint, + time int, + type varchar(100), + team int, + unit varchar(100), + key varchar(100), + value int, + slot int, + player_slot int, + player1 int, + player2 int, + attackername string, + targetname string, + sourcename string, + targetsourcename string, + attackerhero boolean, + targethero boolean, + attackerillusion boolean, + targetillusion boolean, + inflictor varchar(100), + gold_reason int, + xp_reason int, + valuename varchar(100), + stun_duration real, + slow_duration real, + gold int, + lh int, + xp int, + x int, + y int, + stuns real, + hero_id int, + life_state int, + level int, + kills int, + deaths int, + assists int, + denies int +); +CREATE INDEX ON match_logs(match_id); \ No newline at end of file diff --git a/svc/parser.js b/svc/parser.js index 7843e0609..8384ab224 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -263,6 +263,8 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); + var logs = processReduce(entries); + console.log('logs: %s', logs.length); var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); @@ -273,7 +275,6 @@ function runParse(match, job, cb) parsed_data.radiant_xp_adv = ap.radiant_xp_adv; parsed_data.upload = upload; //processMultiKillStreaks(); - //processReduce(res.expanded); console.timeEnd(message); return cb(err, parsed_data); } From 8774cd487b583d34758ac2a9a7129f8d949731b2 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 11 Jun 2016 08:33:22 +0000 Subject: [PATCH 02/17] update schema --- processors/processReduce.js | 21 +++++++++++-------- sql/create_tables.sql | 8 ++++---- store/queries.js | 41 +++++++++++++++++++++++++++++++++++++ svc/parser.js | 4 ++-- 4 files changed, 59 insertions(+), 15 deletions(-) diff --git a/processors/processReduce.js b/processors/processReduce.js index f35890898..1b11d4bda 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -2,16 +2,8 @@ * A processor to reduce the event stream by grouping similar events. * NOT CURRENTLY IN PRODUCTION USE **/ -function processReduce(entries) +function processReduce(entries, match) { - var reduceMap = {}; - //group by player_slot, type, targethero, targetillusion - for (var i = 0; i < entries.length; i++) - { - var e = entries[i]; - reduceMap[e.type] = reduceMap[e.type] ? reduceMap[e.type] + 1 : 1; - } - console.log(reduceMap); return entries.filter(function(e) { if (e.type === "actions") @@ -22,7 +14,18 @@ function processReduce(entries) { return false; } + if (!e.time) + { + return false; + } return true; + }).map(function(e) + { + return Object.assign( + {}, e, + { + match_id: match.match_id + }); }); } module.exports = processReduce; \ No newline at end of file diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 47e7b4e3d..7a7b418b3 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -187,10 +187,10 @@ CREATE TABLE match_logs ( player_slot int, player1 int, player2 int, - attackername string, - targetname string, - sourcename string, - targetsourcename string, + attackername varchar(100), + targetname varchar(100), + sourcename varchar(100), + targetsourcename varchar(100), attackerhero boolean, targethero boolean, attackerillusion boolean, diff --git a/store/queries.js b/store/queries.js index c4952d6ed..bd6e3446c 100644 --- a/store/queries.js +++ b/store/queries.js @@ -193,6 +193,7 @@ function insertMatch(db, redis, match, options, cb) { "u": upsertMatch, "uc": upsertMatchCassandra, + "uml": upsertMatchLogs, "upc": updatePlayerCaches, "uct": updateCounts, "cmc": clearMatchCache, @@ -309,6 +310,45 @@ function insertMatch(db, redis, match, options, cb) }); } + function upsertMatchLogs(cb) + { + if (!match.logs) + { + return cb(); + } + else + { + db.transaction(function(trx) + { + trx.raw(`DELETE FROM match_logs where match_id = ?`, [match.match_id]).asCallback(function(err) + { + if (err) + { + return exit(err); + } + async.eachLimit(match.logs, 10000, function(e, cb) + { + trx('match_logs').insert(e).asCallback(cb); + }, exit); + }); + + function exit(err) + { + if (err) + { + console.error(err); + trx.rollback(err); + } + else + { + trx.commit(); + } + cb(err); + } + }); + } + } + function updatePlayerCaches(cb) { var copy = createMatchCopy(match, players, options); @@ -400,6 +440,7 @@ function insertMatch(db, redis, match, options, cb) duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, + leagueid: match.leagueid, }, { lifo: options.lifo, diff --git a/svc/parser.js b/svc/parser.js index 8384ab224..3355508fd 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -263,8 +263,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = processReduce(entries); - console.log('logs: %s', logs.length); + var logs = match.leagueid ? processReduce(entries, match) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); @@ -274,6 +273,7 @@ function runParse(match, job, cb) parsed_data.radiant_gold_adv = ap.radiant_gold_adv; parsed_data.radiant_xp_adv = ap.radiant_xp_adv; parsed_data.upload = upload; + parsed_data.logs = logs; //processMultiKillStreaks(); console.timeEnd(message); return cb(err, parsed_data); From 63207e4aa7b44ae17e254291c5ddc00dbf5d0efb Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 05:51:30 +0000 Subject: [PATCH 03/17] add api --- java_parser/src/main/java/yasp/Main.java | 8 ++++---- routes/api.js | 11 +++++++++++ sql/create_tables.sql | 2 -- store/queries.js | 6 +++++- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java index f507f938f..edca25d14 100644 --- a/java_parser/src/main/java/yasp/Main.java +++ b/java_parser/src/main/java/yasp/Main.java @@ -70,8 +70,8 @@ private class Entry { public Integer gold_reason; public Integer xp_reason; public String valuename; - public Float stun_duration; - public Float slow_duration; + //public Float stun_duration; + //public Float slow_duration; //entity fields public Integer gold; public Integer lh; @@ -234,8 +234,8 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) { combatLogEntry.attackerillusion = cle.isAttackerIllusion(); combatLogEntry.targetillusion = cle.isTargetIllusion(); combatLogEntry.value = cle.getValue(); - combatLogEntry.stun_duration = cle.getStunDuration(); - combatLogEntry.slow_duration = cle.getSlowDuration(); + //combatLogEntry.stun_duration = cle.getStunDuration(); + //combatLogEntry.slow_duration = cle.getSlowDuration(); //value may be out of bounds in string table, we can only get valuename if a purchase (type 11) if (cle.getType() == DOTA_COMBATLOG_TYPES.DOTA_COMBATLOG_PURCHASE) { combatLogEntry.valuename = cle.getValueName(); diff --git a/routes/api.js b/routes/api.js index c2afa0c3f..f8a878abf 100644 --- a/routes/api.js +++ b/routes/api.js @@ -134,6 +134,17 @@ module.exports = function(db, redis, cassandra) }); }); */ + api.get('/match_logs/:match_id', function(req, res, cb) + { + db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); api.get('/distributions', function(req, res, cb) { queries.getDistributions(redis, function(err, result) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index d20cac76b..eb9f960a4 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -199,8 +199,6 @@ CREATE TABLE match_logs ( gold_reason int, xp_reason int, valuename varchar(100), - stun_duration real, - slow_duration real, gold int, lh int, xp int, diff --git a/store/queries.js b/store/queries.js index cf191fab2..20e698405 100644 --- a/store/queries.js +++ b/store/queries.js @@ -208,7 +208,7 @@ function insertMatch(db, redis, match, options, cb) function upsertMatch(cb) { - if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE) + if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.leagueid) { return cb(); } @@ -232,6 +232,10 @@ function insertMatch(db, redis, match, options, cb) player_slot: pm.player_slot }, cb); }, exit); + //TODO create table for picks/bans + //TODO insert picks/bans + //TODO remove dota_unknown to save space + //TODO figure out whehter to store expanded or raw data function exit(err) { From fd63f476472a40ccbf5f4a1a9a677acca75cfdfa Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 05:53:54 +0000 Subject: [PATCH 04/17] add todos --- sql/create_tables.sql | 2 +- store/queries.js | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index eb9f960a4..5efdad478 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -176,7 +176,7 @@ CREATE TABLE notable_players ( ); CREATE TABLE match_logs ( - match_id bigint, + match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, time int, type varchar(100), team int, diff --git a/store/queries.js b/store/queries.js index 20e698405..78831b848 100644 --- a/store/queries.js +++ b/store/queries.js @@ -232,6 +232,8 @@ function insertMatch(db, redis, match, options, cb) player_slot: pm.player_slot }, cb); }, exit); + //TODO create leagues table + //TODO create worker task to get leagues //TODO create table for picks/bans //TODO insert picks/bans //TODO remove dota_unknown to save space From 68ada3214f84dc63a0f85660feb2314a27261d8a Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 19:04:08 +0000 Subject: [PATCH 05/17] store raw data, with slot indexes --- processors/processReduce.js | 32 +++++++++++-- routes/api.js | 2 +- sql/create_tables.sql | 45 +++++++++++-------- store/queries.js | 4 +- svc/parser.js | 2 +- tasks/getMatches.js | 50 +++++++++++++++++++++ dev/allMatches.js => tasks/getMatchesSeq.js | 0 7 files changed, 109 insertions(+), 26 deletions(-) create mode 100644 tasks/getMatches.js rename dev/allMatches.js => tasks/getMatchesSeq.js (100%) diff --git a/processors/processReduce.js b/processors/processReduce.js index 1b11d4bda..f8ed5bd5e 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -2,9 +2,9 @@ * A processor to reduce the event stream by grouping similar events. * NOT CURRENTLY IN PRODUCTION USE **/ -function processReduce(entries, match) +function processReduce(entries, match, meta) { - return entries.filter(function(e) + var result = entries.filter(function(e) { if (e.type === "actions") { @@ -21,11 +21,35 @@ function processReduce(entries, match) return true; }).map(function(e) { - return Object.assign( + var e2 = Object.assign( {}, e, { - match_id: match.match_id + match_id: match.match_id, + attackername_slot: meta.hero_to_slot[e.attackername], + targetname_slot: meta.hero_to_slot[e.targetname], + sourcename_slot: meta.hero_to_slot[e.sourcename], + targetsourcename_slot: meta.hero_to_slot[e.targetname], + player1_slot: meta.slot_to_playerslot[e.player1], + player_slot: e.player_slot || meta.slot_to_playerslot[e.slot], + inflictor: translate(e.inflictor), }); + delete e2.attackername; + delete e2.targetname; + delete e2.sourcename; + delete e2.targetsourcename; + return e2; }); + var count = {}; + result.forEach(function(r) + { + count[r.type] = (count[r.type] || 0) + 1; + }); + console.log(count); + return result; +} + +function translate(s) +{ + return s === "dota_unknown" ? null : s; } module.exports = processReduce; \ No newline at end of file diff --git a/routes/api.js b/routes/api.js index f8a878abf..475d2eb85 100644 --- a/routes/api.js +++ b/routes/api.js @@ -136,7 +136,7 @@ module.exports = function(db, redis, cassandra) */ api.get('/match_logs/:match_id', function(req, res, cb) { - db.raw(`SELECT * FROM match_logs WHERE match_id = ?`, [req.params.match_id]).asCallback(function(err, result) + db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result) { if (err) { diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 5efdad478..bae99a7d2 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -179,38 +179,45 @@ CREATE TABLE match_logs ( match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, time int, type varchar(100), - team int, + team smallint, unit varchar(100), - key varchar(100), + key varchar(1000), value int, - slot int, - player_slot int, + slot smallint, + player_slot smallint, player1 int, player2 int, - attackername varchar(100), - targetname varchar(100), - sourcename varchar(100), - targetsourcename varchar(100), attackerhero boolean, targethero boolean, attackerillusion boolean, targetillusion boolean, inflictor varchar(100), - gold_reason int, - xp_reason int, + gold_reason smallint, + xp_reason smallint, valuename varchar(100), gold int, lh int, xp int, - x int, - y int, + x smallint, + y smallint, stuns real, - hero_id int, - life_state int, - level int, - kills int, - deaths int, - assists int, - denies int + hero_id smallint, + life_state smallint, + level smallint, + kills smallint, + deaths smallint, + assists smallint, + denies smallint, + attackername_slot smallint, + targetname_slot smallint, + sourcename_slot smallint, + targetsourcename_slot smallint, + player1_slot smallint ); CREATE INDEX ON match_logs(match_id); +CREATE INDEX ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL; diff --git a/store/queries.js b/store/queries.js index 78831b848..79a2c9167 100644 --- a/store/queries.js +++ b/store/queries.js @@ -238,7 +238,9 @@ function insertMatch(db, redis, match, options, cb) //TODO insert picks/bans //TODO remove dota_unknown to save space //TODO figure out whehter to store expanded or raw data - +/* +"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}] +*/ function exit(err) { if (err) diff --git a/svc/parser.js b/svc/parser.js index 8c1a75325..5cff09a0c 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -267,7 +267,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.leagueid ? processReduce(entries, match) : undefined; + var logs = match.leagueid ? processReduce(entries, match, meta) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); diff --git a/tasks/getMatches.js b/tasks/getMatches.js new file mode 100644 index 000000000..cf2d7fd92 --- /dev/null +++ b/tasks/getMatches.js @@ -0,0 +1,50 @@ +var async = require('async'); +var utility = require('../util/utility'); +var generateJob = utility.generateJob; +var getData = utility.getData; +var db = require('../store/db'); +var redis = require('../store/redis'); +var cassandra = require('../store/cassandra'); +var queries = require('../store/queries'); +var insertMatch = queries.insertMatch; +var args = process.argv.slice(2); +var match_id = Number(args[0]); +var delay = 1000; +var job = generateJob("api_details", +{ + match_id: match_id +}); +var url = job.url; +getData( +{ + url: url, + delay: delay +}, function(err, body) +{ + if (err) + { + throw err; + } + if (body.result) + { + var match = body.result; + match.parse_status = 0; + insertMatch(db, redis, match, + { + skipCounts: true, + skipAbilityUpgrades: true, + cassandra: cassandra, + }, function(err) + { + if (err) + { + throw err; + } + process.exit(0); + }); + } + else + { + throw body; + } +}); \ No newline at end of file diff --git a/dev/allMatches.js b/tasks/getMatchesSeq.js similarity index 100% rename from dev/allMatches.js rename to tasks/getMatchesSeq.js From 5530d780d0d8220dbde4ebfbab4f978e26ee2a4f Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sat, 18 Jun 2016 23:34:48 +0000 Subject: [PATCH 06/17] remove some debugging info --- processors/processExpand.js | 2 +- processors/processReduce.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/processors/processExpand.js b/processors/processExpand.js index e67b97647..c91c15ca3 100644 --- a/processors/processExpand.js +++ b/processors/processExpand.js @@ -449,7 +449,7 @@ function processExpand(entries, meta) //null means all types var reqs = { parsed_data: null, - expanded: null, + //expanded: null, tf_data: { killed: 1, diff --git a/processors/processReduce.js b/processors/processReduce.js index f8ed5bd5e..2bf9dbe00 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -1,6 +1,5 @@ /** - * A processor to reduce the event stream by grouping similar events. - * NOT CURRENTLY IN PRODUCTION USE + * A processor to reduce the event stream to only logs we want to persist **/ function processReduce(entries, match, meta) { @@ -39,12 +38,14 @@ function processReduce(entries, match, meta) delete e2.targetsourcename; return e2; }); + /* var count = {}; result.forEach(function(r) { count[r.type] = (count[r.type] || 0) + 1; }); console.log(count); + */ return result; } From 98fc634794f12e7ffa0e3eda13f8bea3c70ae3ea Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 00:43:06 +0000 Subject: [PATCH 07/17] implement league fetcher, use async for match insert flow --- sql/create_tables.sql | 17 +++++++++ store/queries.js | 81 +++++++++++++++++++++++++++---------------- svc/parser.js | 2 +- svc/scanner.js | 7 +--- svc/worker.js | 30 ++++++++++++++++ 5 files changed, 101 insertions(+), 36 deletions(-) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index bae99a7d2..8d1564d77 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -221,3 +221,20 @@ CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL; CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL; CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL; + +CREATE TABLE picks_bans( + match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, + is_pick boolean, + hero_id int, + team smallint, + order smallint, + PRIMARY KEY (match_id, order) +); + +CREATE TABLE leagues( + leagueid bigint PRIMARY KEY, + ticket varchar(255), + banner varchar(255), + tier varchar(255), + name varchar(255) +) \ No newline at end of file diff --git a/store/queries.js b/store/queries.js index 79a2c9167..f082aac51 100644 --- a/store/queries.js +++ b/store/queries.js @@ -191,6 +191,7 @@ function insertMatch(db, redis, match, options, cb) //we want to insert into matches, then insert into player_matches for each entry in players async.series( { + "ipm": isProMatch, "u": upsertMatch, "uc": upsertMatchCassandra, "uml": upsertMatchLogs, @@ -206,23 +207,38 @@ function insertMatch(db, redis, match, options, cb) return cb(err, results.dp); }); + function isProMatch(cb) + { + //TODO check redis/postgres for professional/premium league + match.isProMatch = match.leagueid >= 0; + cb(); + } + function upsertMatch(cb) { - if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.leagueid) + if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.isProMatch) { return cb(); } db.transaction(function(trx) { - upsert(trx, 'matches', match, + async.series( { - match_id: match.match_id - }, function(err) + "im": insertMatch, + "ipm": insertPlayerMatches, + "ipb": insertPicksBans + }, exit); + + function insertMatch(cb) { - if (err) + upsert(trx, 'matches', match, { - return exit(err); - } + match_id: match.match_id + }, cb); + } + + function insertPlayerMatches(cb) + { async.each(players || [], function(pm, cb) { pm.match_id = match.match_id; @@ -231,30 +247,37 @@ function insertMatch(db, redis, match, options, cb) match_id: pm.match_id, player_slot: pm.player_slot }, cb); - }, exit); - //TODO create leagues table - //TODO create worker task to get leagues - //TODO create table for picks/bans - //TODO insert picks/bans - //TODO remove dota_unknown to save space - //TODO figure out whehter to store expanded or raw data -/* -"picks_bans":[{"is_pick":false,"hero_id":41,"team":1,"order":0},{"is_pick":false,"hero_id":38,"team":0,"order":1},{"is_pick":false,"hero_id":6,"team":1,"order":2},{"is_pick":false,"hero_id":62,"team":0,"order":3},{"is_pick":true,"hero_id":29,"team":1,"order":4},{"is_pick":true,"hero_id":20,"team":0,"order":5},{"is_pick":true,"hero_id":65,"team":0,"order":6},{"is_pick":true,"hero_id":87,"team":1,"order":7},{"is_pick":false,"hero_id":93,"team":0,"order":8},{"is_pick":false,"hero_id":74,"team":1,"order":9},{"is_pick":false,"hero_id":110,"team":0,"order":10},{"is_pick":false,"hero_id":54,"team":1,"order":11},{"is_pick":true,"hero_id":18,"team":0,"order":12},{"is_pick":true,"hero_id":107,"team":1,"order":13},{"is_pick":true,"hero_id":103,"team":0,"order":14},{"is_pick":true,"hero_id":49,"team":1,"order":15},{"is_pick":false,"hero_id":63,"team":0,"order":16},{"is_pick":false,"hero_id":98,"team":1,"order":17},{"is_pick":true,"hero_id":67,"team":1,"order":18},{"is_pick":true,"hero_id":8,"team":0,"order":19}] -*/ - function exit(err) + }, cb); + } + + function insertPicksBans(cb) + { + async.each(match.picks_bans || [], function(p, cb) { - if (err) + //order is a reserved keyword + p.ord = p.order; + p.match_id = match.match_id; + upsert(trx, 'picks_bans', p, { - console.error(err); - trx.rollback(err); - } - else - { - trx.commit(); - } - cb(err); + match_id: p.match_id, + ord: p.ord + }, cb); + }, cb); + } + + function exit(err) + { + if (err) + { + console.error(err); + trx.rollback(err); } - }); + else + { + trx.commit(); + } + cb(err); + } }); } @@ -462,7 +485,7 @@ function insertMatch(db, redis, match, options, cb) duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, - leagueid: match.leagueid, + isProMatch: match.isProMatch, }, { lifo: options.lifo, diff --git a/svc/parser.js b/svc/parser.js index 5cff09a0c..1228af883 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -267,7 +267,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.leagueid ? processReduce(entries, match, meta) : undefined; + var logs = match.isProMatch ? processReduce(entries, match, meta) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); diff --git a/svc/scanner.js b/svc/scanner.js index 00d6e3215..fa15a8f35 100644 --- a/svc/scanner.js +++ b/svc/scanner.js @@ -112,12 +112,7 @@ function start() function processMatch(match, cb) { - if (config.ENABLE_PRO_PARSING && match.leagueid) - { - //parse tournament games - match.parse_status = 0; - } - else if (match.players.some(function(p) + if (match.players.some(function(p) { return (p.account_id in trackedPlayers); })) diff --git a/svc/worker.js b/svc/worker.js index 4c492994e..ee1cd62f4 100644 --- a/svc/worker.js +++ b/svc/worker.js @@ -124,6 +124,36 @@ invokeInterval(function notablePlayers(cb) }, cb); }); }, 10 * 60 * 1000); +invokeInterval(function leagues(cb) +{ + utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues) + { + if (err) + { + return cb(err); + } + utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/items_game.json', function(err, items) + { + if (err) + { + return cb(err); + } + var arr = []; + for (var key in leagues) + { + arr.push( + { + leagueid: key, + ticket: leagues[key].ticket, + banner: leagues[key].banner, + tier: leagues[key].tier, + name: items.items_game.items[key].name + }); + } + db('leagues').insert(arr).asCallback(cb); + }); + }); +}, 10 * 60 * 1000); function invokeInterval(func, delay) { From a628f581303fc556ad2bae23dbe22e86bcdd2e6b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 01:06:27 +0000 Subject: [PATCH 08/17] add leagues endpoint --- routes/api.js | 11 +++++++++++ sql/create_tables.sql | 4 ++-- store/queries.js | 8 +++++--- svc/worker.js | 33 ++++++++++++++++++++------------- 4 files changed, 38 insertions(+), 18 deletions(-) diff --git a/routes/api.js b/routes/api.js index 475d2eb85..d13e75341 100644 --- a/routes/api.js +++ b/routes/api.js @@ -145,6 +145,17 @@ module.exports = function(db, redis, cassandra) res.json(result.rows); }); }); + api.get('/leagues', function(req, res, cb) + { + db.raw(`SELECT * FROM leagues ORDER BY leagueid`).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); api.get('/distributions', function(req, res, cb) { queries.getDistributions(redis, function(err, result) diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 8d1564d77..7eaa7571a 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -227,8 +227,8 @@ CREATE TABLE picks_bans( is_pick boolean, hero_id int, team smallint, - order smallint, - PRIMARY KEY (match_id, order) + ord smallint, + PRIMARY KEY (match_id, ord) ); CREATE TABLE leagues( diff --git a/store/queries.js b/store/queries.js index f082aac51..50544476f 100644 --- a/store/queries.js +++ b/store/queries.js @@ -209,9 +209,11 @@ function insertMatch(db, redis, match, options, cb) function isProMatch(cb) { - //TODO check redis/postgres for professional/premium league - match.isProMatch = match.leagueid >= 0; - cb(); + redis.sismember('pro_leagueids', match.league_id, function(err, result) + { + match.isProMatch = Boolean(Number(result)); + cb(err); + }); } function upsertMatch(cb) diff --git a/svc/worker.js b/svc/worker.js index ee1cd62f4..4e3df510e 100644 --- a/svc/worker.js +++ b/svc/worker.js @@ -126,31 +126,38 @@ invokeInterval(function notablePlayers(cb) }, 10 * 60 * 1000); invokeInterval(function leagues(cb) { - utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues) + var container = utility.generateJob("api_leagues", + {}); + utility.getData(container.url, function(err, api_leagues) { if (err) { return cb(err); } - utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/items_game.json', function(err, items) + utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues) { if (err) { return cb(err); } - var arr = []; - for (var key in leagues) + async.each(api_leagues.result.leagues, function(l, cb) { - arr.push( + if (leagues[l.leagueid]) { - leagueid: key, - ticket: leagues[key].ticket, - banner: leagues[key].banner, - tier: leagues[key].tier, - name: items.items_game.items[key].name - }); - } - db('leagues').insert(arr).asCallback(cb); + l.tier = leagues[l.leagueid].tier; + l.ticket = leagues[l.leagueid].ticket; + l.banner = leagues[l.leagueid].banner; + } + l.name = l.description.substring("#DOTA_Item_Desc_".length).split('_').join(' '); + if (l.tier === "professional" || l.tier === "premium") + { + redis.sadd('pro_leagueids', l.leagueid); + } + queries.upsert(db, 'leagues', l, + { + leagueid: l.league_id + }, cb); + }, cb); }); }); }, 10 * 60 * 1000); From 8b77717f2b0c7caa2fbc2723d93bec4af56df9df Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 02:55:24 +0000 Subject: [PATCH 09/17] fix leagueid prop name --- config.js | 1 - store/queries.js | 15 +++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/config.js b/config.js index 0ee61c505..beccd9af9 100644 --- a/config.js +++ b/config.js @@ -54,7 +54,6 @@ var defaults = { "UI_HOST": "", //The host of the UI, redirect traffic from / and /return here "ENABLE_RECAPTCHA": "", //set to enable the recaptcha on the Request page "ENABLE_ADS": "", //set to turn on ads - "ENABLE_PRO_PARSING": "", // set to parse pro matches from sequential API "ENABLE_MATCH_CACHE": "", // set to enable caching matches (Redis) "ENABLE_PLAYER_CACHE": "", // set to enable caching players (Cassandra) "ENABLE_INSERT_ALL_MATCHES": "", //set to enable inserting all matches diff --git a/store/queries.js b/store/queries.js index 50544476f..189535a2b 100644 --- a/store/queries.js +++ b/store/queries.js @@ -209,11 +209,18 @@ function insertMatch(db, redis, match, options, cb) function isProMatch(cb) { - redis.sismember('pro_leagueids', match.league_id, function(err, result) + if (match.leagueid) { - match.isProMatch = Boolean(Number(result)); - cb(err); - }); + redis.sismember('pro_leagueids', match.leagueid, function(err, result) + { + match.isProMatch = Boolean(Number(result)); + cb(err); + }); + } + else + { + cb(); + } } function upsertMatch(cb) From b1ee5ea3897e3ab1c55aea49b27cdbaef1ed7298 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 03:03:37 +0000 Subject: [PATCH 10/17] transact match log insert --- store/queries.js | 75 +++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 46 deletions(-) diff --git a/store/queries.js b/store/queries.js index 189535a2b..0df215242 100644 --- a/store/queries.js +++ b/store/queries.js @@ -194,7 +194,6 @@ function insertMatch(db, redis, match, options, cb) "ipm": isProMatch, "u": upsertMatch, "uc": upsertMatchCassandra, - "uml": upsertMatchLogs, "upc": updatePlayerCaches, "uct": updateCounts, "cmc": clearMatchCache, @@ -233,12 +232,13 @@ function insertMatch(db, redis, match, options, cb) { async.series( { - "im": insertMatch, - "ipm": insertPlayerMatches, - "ipb": insertPicksBans + "m": upsertMatch, + "pm": upsertPlayerMatches, + "pb": upsertPicksBans, + "l": upsertMatchLogs, }, exit); - function insertMatch(cb) + function upsertMatch(cb) { upsert(trx, 'matches', match, { @@ -246,7 +246,7 @@ function insertMatch(db, redis, match, options, cb) }, cb); } - function insertPlayerMatches(cb) + function upsertPlayerMatches(cb) { async.each(players || [], function(pm, cb) { @@ -259,7 +259,7 @@ function insertMatch(db, redis, match, options, cb) }, cb); } - function insertPicksBans(cb) + function upsertPicksBans(cb) { async.each(match.picks_bans || [], function(p, cb) { @@ -274,6 +274,28 @@ function insertMatch(db, redis, match, options, cb) }, cb); } + function upsertMatchLogs(cb) + { + if (!match.logs) + { + return cb(); + } + else + { + trx.raw(`DELETE FROM match_logs WHERE match_id = ?`, [match.match_id]).asCallback(function(err) + { + if (err) + { + return exit(err); + } + async.eachLimit(match.logs, 100, function(e, cb) + { + trx('match_logs').insert(e).asCallback(cb); + }, cb); + }); + } + } + function exit(err) { if (err) @@ -351,45 +373,6 @@ function insertMatch(db, redis, match, options, cb) }); } - function upsertMatchLogs(cb) - { - if (!match.logs) - { - return cb(); - } - else - { - db.transaction(function(trx) - { - trx.raw(`DELETE FROM match_logs where match_id = ?`, [match.match_id]).asCallback(function(err) - { - if (err) - { - return exit(err); - } - async.eachLimit(match.logs, 10000, function(e, cb) - { - trx('match_logs').insert(e).asCallback(cb); - }, exit); - }); - - function exit(err) - { - if (err) - { - console.error(err); - trx.rollback(err); - } - else - { - trx.commit(); - } - cb(err); - } - }); - } - } - function updatePlayerCaches(cb) { var copy = createMatchCopy(match, players, options); From 48b06f12f9e3b0f8f3703d476f428cc3bcfb0587 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:03:50 +0000 Subject: [PATCH 11/17] remove parse_status --- store/queries.js | 15 ++++++++------- svc/parser.js | 3 +-- svc/requests.js | 2 +- svc/scanner.js | 12 +++++++----- tasks/getMatches.js | 2 +- views/match/match.jade | 1 - 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/store/queries.js b/store/queries.js index 0df215242..768a9af2f 100644 --- a/store/queries.js +++ b/store/queries.js @@ -191,7 +191,7 @@ function insertMatch(db, redis, match, options, cb) //we want to insert into matches, then insert into player_matches for each entry in players async.series( { - "ipm": isProMatch, + "dlp": decideLogParse, "u": upsertMatch, "uc": upsertMatchCassandra, "upc": updatePlayerCaches, @@ -205,14 +205,14 @@ function insertMatch(db, redis, match, options, cb) { return cb(err, results.dp); }); - - function isProMatch(cb) + + function decideLogParse(cb) { if (match.leagueid) { redis.sismember('pro_leagueids', match.leagueid, function(err, result) { - match.isProMatch = Boolean(Number(result)); + options.doLogParse = options.doLogParse || Boolean(Number(result)); cb(err); }); } @@ -224,7 +224,7 @@ function insertMatch(db, redis, match, options, cb) function upsertMatch(cb) { - if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !match.isProMatch) + if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !options.doLogParse) { return cb(); } @@ -458,9 +458,10 @@ function insertMatch(db, redis, match, options, cb) }, cb); } + function decideParse(cb) { - if (match.parse_status !== 0) + if (options.skipParse) { //not parsing this match //this isn't a error, although we want to report that we refused to parse back to user if it was a request @@ -477,7 +478,7 @@ function insertMatch(db, redis, match, options, cb) duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, - isProMatch: match.isProMatch, + doLogParse: options.doLogParse, }, { lifo: options.lifo, diff --git a/svc/parser.js b/svc/parser.js index 1228af883..586c8773e 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -91,7 +91,6 @@ pQueue.process(config.PARSER_PARALLELISM, function(job, cb) parsed_data.start_time = match.start_time; parsed_data.duration = match.duration; parsed_data.replay_blob_key = match.replay_blob_key; - parsed_data.parse_status = 2; if (match.replay_blob_key) { insertUploadedParse(parsed_data, cb); @@ -267,7 +266,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); - var logs = match.isProMatch ? processReduce(entries, match, meta) : undefined; + var logs = match.doLogParse ? processReduce(entries, match, meta) : undefined; var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); diff --git a/svc/requests.js b/svc/requests.js index c6007a733..08901defb 100644 --- a/svc/requests.js +++ b/svc/requests.js @@ -29,13 +29,13 @@ function processRequest(job, cb) } //match details response var match = body.result; - match.parse_status = 0; insertMatch(db, redis, match, { type: "api", attempts: 1, lifo: true, cassandra: cassandra, + skipParse: false, }, waitParse); }); } diff --git a/svc/scanner.js b/svc/scanner.js index fa15a8f35..ee8d86348 100644 --- a/svc/scanner.js +++ b/svc/scanner.js @@ -112,21 +112,22 @@ function start() function processMatch(match, cb) { + var insert = false; + var skipParse = true; if (match.players.some(function(p) { return (p.account_id in trackedPlayers); })) { - //queued - match.parse_status = 0; + insert = true; + skipParse = false; } else if (match.players.some(function(p) { return (config.ENABLE_INSERT_ALL_MATCHES || p.account_id in userPlayers); })) { - //skipped - match.parse_status = 3; + insert = true; } //check if match was previously processed redis.get('scanner_insert:' + match.match_id, function(err, result) @@ -137,7 +138,7 @@ function start() } //don't insert this match if we already processed it recently //deduplicate matches in this page set - if ((match.parse_status === 0 || match.parse_status === 3) && !result && !matchBuffer[match.match_id]) + if (insert && !result && !matchBuffer[match.match_id]) { matchBuffer[match.match_id] = 1; insertMatch(db, redis, match, @@ -146,6 +147,7 @@ function start() origin: "scanner", cassandra: cassandra, userPlayers: userPlayers, + skipParse: skipParse, }, function(err) { if (!err) diff --git a/tasks/getMatches.js b/tasks/getMatches.js index cf2d7fd92..0362a1b95 100644 --- a/tasks/getMatches.js +++ b/tasks/getMatches.js @@ -28,11 +28,11 @@ getData( if (body.result) { var match = body.result; - match.parse_status = 0; insertMatch(db, redis, match, { skipCounts: true, skipAbilityUpgrades: true, + skipParse: false, cassandra: cassandra, }, function(err) { diff --git a/views/match/match.jade b/views/match/match.jade index 53673b8d2..6d0a345d7 100644 --- a/views/match/match.jade +++ b/views/match/match.jade @@ -32,7 +32,6 @@ block content th Percentile tbody tr - //td= constants.parse_status[match.parse_status] td= constants.game_mode[match.game_mode] ? constants.game_mode[match.game_mode].name : match.game_mode td= constants.region[match.region] ? constants.region[match.region] : match.region td= moment().startOf('day').seconds(match.duration).format("H:mm:ss") From a95f2c3bf06b34035e32dd47a69eb4cebb032c1d Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:17:43 +0000 Subject: [PATCH 12/17] add endpoint to list pro matches --- routes/api.js | 20 +++++++++++++++++++- sql/create_tables.sql | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/routes/api.js b/routes/api.js index d13e75341..4a3311210 100644 --- a/routes/api.js +++ b/routes/api.js @@ -136,7 +136,25 @@ module.exports = function(db, redis, cassandra) */ api.get('/match_logs/:match_id', function(req, res, cb) { - db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time asc`, [req.params.match_id]).asCallback(function(err, result) + db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time ASC`, [req.params.match_id]).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); + api.get('/pro_matches', function(req, res, cb) + { + db.raw(` + SELECT match_id, start_time, duration, ma.leagueid, name + FROM matches ma + JOIN leagues le + ON ma.leagueid = le.leagueid + WHERE ma.leagueid > 0 + ORDER BY match_id DESC + `).asCallback(function(err, result) { if (err) { diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 7eaa7571a..936d49041 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -39,6 +39,7 @@ CREATE TABLE matches ( teamfights json[], version integer ); +CREATE INDEX on matches(leagueid) WHERE leagueid > 0; CREATE TABLE player_matches ( PRIMARY KEY(match_id, player_slot), From 77b4b90b89c029f6621fffce1af610cd66ed9d66 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:23:51 +0000 Subject: [PATCH 13/17] fix double parse --- svc/parser.js | 1 + 1 file changed, 1 insertion(+) diff --git a/svc/parser.js b/svc/parser.js index 586c8773e..ac5aa4864 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -157,6 +157,7 @@ function insertStandardParse(match, cb) { type: "parsed", cassandra: cassandra, + skipParse: true, }, cb); } From a46b6b8419988a2fd0a021605e33ae13f76d137e Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Sun, 19 Jun 2016 22:50:27 +0000 Subject: [PATCH 14/17] fix insert call in test --- svc/fullhistory.js | 1 + tasks/getMatchesSeq.js | 1 + test/test.js | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/svc/fullhistory.js b/svc/fullhistory.js index 312034a51..d5707ac4a 100644 --- a/svc/fullhistory.js +++ b/svc/fullhistory.js @@ -90,6 +90,7 @@ function processFullHistory(job, cb) type: "api", cassandra: cassandra, skipAbilityUpgrades: true, + skipParse: true, }, cb); }); }, function(err) diff --git a/tasks/getMatchesSeq.js b/tasks/getMatchesSeq.js index ebd445e36..331c5bb1c 100644 --- a/tasks/getMatchesSeq.js +++ b/tasks/getMatchesSeq.js @@ -86,6 +86,7 @@ function getPage(match_seq_num, bucket) { skipCounts: true, skipAbilityUpgrades: true, + skipParse: true, cassandra: cassandra, }, cb); }, function(err) diff --git a/test/test.js b/test/test.js index 73af32d1d..e6e4cebd7 100644 --- a/test/test.js +++ b/test/test.js @@ -128,7 +128,8 @@ before(function(done) { queries.insertMatch(db, redis, m, { - type: "api" + type: "api", + skipParse: true, }, cb); }, cb); }, From afa5e6167a65b8ac5a8596b2c52970d11b6f4a85 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 20 Jun 2016 00:54:20 +0000 Subject: [PATCH 15/17] remove userplayers --- store/buildSets.js | 14 -------------- store/queries.js | 9 +-------- svc/scanner.js | 8 +------- 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/store/buildSets.js b/store/buildSets.js index fed99d429..2c3794d77 100644 --- a/store/buildSets.js +++ b/store/buildSets.js @@ -18,20 +18,6 @@ module.exports = function buildSets(db, redis, cb) { cb(err, t); }); }, - //users in this set have their matches added - "userPlayers": function(cb) { - db.select(['account_id']).from('players').whereNotNull('last_login').asCallback(function(err, docs) { - if (err) { - return cb(err); - } - var t = {}; - docs.forEach(function(player) { - t[player.account_id] = true; - }); - //console.log(t); - cb(err, t); - }); - }, //users in this set are added to the trackedPlayers set "donators": function(cb) { db.select(['account_id']).from('players').where('cheese', '>', 0).asCallback(function(err, docs) { diff --git a/store/queries.js b/store/queries.js index 768a9af2f..49974199e 100644 --- a/store/queries.js +++ b/store/queries.js @@ -32,13 +32,6 @@ function getSets(redis, cb) cb(err, JSON.parse(tps || "{}")); }); }, - "userPlayers": function(cb) - { - redis.get("userPlayers", function(err, ups) - { - cb(err, JSON.parse(ups || "{}")); - }); - }, "donators": function(cb) { redis.get("donators", function(err, ds) @@ -426,7 +419,7 @@ function insertMatch(db, redis, match, options, cb) { async.each(match.players, function(p, cb) { - if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && (p.account_id in options.userPlayers || (config.ENABLE_RANDOM_MMR_UPDATE && match.match_id % 3 === 0))) + if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && config.ENABLE_RANDOM_MMR_UPDATE) { addToQueue(mQueue, { diff --git a/svc/scanner.js b/svc/scanner.js index ee8d86348..819db762d 100644 --- a/svc/scanner.js +++ b/svc/scanner.js @@ -13,7 +13,6 @@ var getData = utility.getData; var generateJob = utility.generateJob; var async = require('async'); var trackedPlayers; -var userPlayers; var parallelism = config.SCANNER_PARALLELISM; var PAGE_SIZE = 100; buildSets(db, redis, function(err) @@ -72,7 +71,6 @@ function start() } //set local vars trackedPlayers = result.trackedPlayers; - userPlayers = result.userPlayers; var arr = []; var matchBuffer = {}; var completePages = {}; @@ -122,10 +120,7 @@ function start() insert = true; skipParse = false; } - else if (match.players.some(function(p) - { - return (config.ENABLE_INSERT_ALL_MATCHES || p.account_id in userPlayers); - })) + else if (config.ENABLE_INSERT_ALL_MATCHES) { insert = true; } @@ -146,7 +141,6 @@ function start() type: "api", origin: "scanner", cassandra: cassandra, - userPlayers: userPlayers, skipParse: skipParse, }, function(err) { From 41b115c7a3eb3dcc07cd193641e866e6eaa7ff44 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Mon, 20 Jun 2016 01:19:10 +0000 Subject: [PATCH 16/17] update knex, remove modifier_remove events from log --- package.json | 2 +- processors/processReduce.js | 4 ++++ store/queries.js | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index b88d0ee1f..011261da3 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "http-proxy": "^1.13.1", "jade": "^1.11.0", "jquery": "^2.2.0", - "knex": "^0.10.0", + "knex": "^0.11.7", "moment": "^2.11.2", "multer": "^1.1.0", "numeral": "^1.5.3", diff --git a/processors/processReduce.js b/processors/processReduce.js index 2bf9dbe00..3fd6823b2 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -9,6 +9,10 @@ function processReduce(entries, match, meta) { return false; } + if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE") + { + return false; + } if (e.type === "interval" && e.time % 60 !== 0) { return false; diff --git a/store/queries.js b/store/queries.js index 49974199e..05ecb64cd 100644 --- a/store/queries.js +++ b/store/queries.js @@ -279,9 +279,9 @@ function insertMatch(db, redis, match, options, cb) { if (err) { - return exit(err); + return cb(err); } - async.eachLimit(match.logs, 100, function(e, cb) + async.eachLimit(match.logs, 10000, function(e, cb) { trx('match_logs').insert(e).asCallback(cb); }, cb); From 0165e1e4fa563a05e7bf41c12c4e3dadc31ad789 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Wed, 22 Jun 2016 08:10:10 +0000 Subject: [PATCH 17/17] remove non-hero logs --- processors/processReduce.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/processors/processReduce.js b/processors/processReduce.js index 3fd6823b2..434c032e9 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -13,6 +13,13 @@ function processReduce(entries, match, meta) { return false; } + if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL") + { + if (!e.targethero || e.targetillusion) + { + return false; + } + } if (e.type === "interval" && e.time % 60 !== 0) { return false;