diff --git a/config.js b/config.js index 0ee61c505..beccd9af9 100644 --- a/config.js +++ b/config.js @@ -54,7 +54,6 @@ var defaults = { "UI_HOST": "", //The host of the UI, redirect traffic from / and /return here "ENABLE_RECAPTCHA": "", //set to enable the recaptcha on the Request page "ENABLE_ADS": "", //set to turn on ads - "ENABLE_PRO_PARSING": "", // set to parse pro matches from sequential API "ENABLE_MATCH_CACHE": "", // set to enable caching matches (Redis) "ENABLE_PLAYER_CACHE": "", // set to enable caching players (Cassandra) "ENABLE_INSERT_ALL_MATCHES": "", //set to enable inserting all matches diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java index f507f938f..55dcdcca6 100644 --- a/java_parser/src/main/java/yasp/Main.java +++ b/java_parser/src/main/java/yasp/Main.java @@ -10,6 +10,7 @@ import skadistats.clarity.model.s1.GameRulesStateType; import skadistats.clarity.processor.entities.Entities; import skadistats.clarity.processor.entities.OnEntityEntered; +import skadistats.clarity.processor.entities.OnEntityLeft; import skadistats.clarity.processor.entities.UsesEntities; import skadistats.clarity.processor.gameevents.CombatLog; import skadistats.clarity.processor.gameevents.OnCombatLogEntry; @@ -70,14 +71,15 @@ private class Entry { public Integer gold_reason; public Integer xp_reason; public String valuename; - public Float stun_duration; - public Float slow_duration; + //public Float stun_duration; + //public Float slow_duration; //entity fields public Integer gold; public Integer lh; public Integer xp; public Integer x; public Integer y; + public Integer z; public Float stuns; public Integer hero_id; public Integer life_state; @@ -87,7 +89,9 @@ private class Entry { public Integer assists; public Integer denies; //public Boolean hasPredictedVictory; - + public Boolean entityleft; + public Integer ehandle; + public Entry() { } @@ -234,8 +238,8 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) { combatLogEntry.attackerillusion = cle.isAttackerIllusion(); combatLogEntry.targetillusion = cle.isTargetIllusion(); combatLogEntry.value = cle.getValue(); - combatLogEntry.stun_duration = cle.getStunDuration(); - combatLogEntry.slow_duration = cle.getSlowDuration(); + //combatLogEntry.stun_duration = cle.getStunDuration(); + //combatLogEntry.slow_duration = cle.getSlowDuration(); //value may be out of bounds in string table, we can only get valuename if a purchase (type 11) if (cle.getType() == DOTA_COMBATLOG_TYPES.DOTA_COMBATLOG_PURCHASE) { combatLogEntry.valuename = cle.getValueName(); @@ -255,28 +259,12 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) { @OnEntityEntered public void onEntityEntered(Context ctx, Entity e) { - //CDOTA_NPC_Observer_Ward - //CDOTA_NPC_Observer_Ward_TrueSight - //s1 "DT_DOTA_NPC_Observer_Ward" - //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight" - boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward"); - boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight"); - if (isObserver || isSentry) { - //System.err.println(e); - Entry entry = new Entry(time); - Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null); - Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null); - Integer[] pos = {x, y}; - entry.type = isObserver ? "obs" : "sen"; - entry.key = Arrays.toString(pos); - //System.err.println(entry.key); - Integer owner = getEntityProperty(e, "m_hOwnerEntity", null); - Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner); - entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null; - //2/3 radiant/dire - //entry.team = e.getProperty("m_iTeamNum"); - output(entry); - } + processWardEntity(ctx, e, false); + } + + @OnEntityLeft + public void onEntityLeft(Context ctx, Entity e) { + processWardEntity(ctx, e, true); } @UsesEntities @@ -448,6 +436,45 @@ public T getEntityProperty(Entity e, String property, Integer idx) { FieldPath fp = e.getDtClass().getFieldPathForName(property); return e.getPropertyForFieldPath(fp); } + + public void processWardEntity(Context ctx, Entity e, boolean entityLeft) + { + //CDOTA_NPC_Observer_Ward + //CDOTA_NPC_Observer_Ward_TrueSight + //s1 "DT_DOTA_NPC_Observer_Ward" + //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight" + boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward"); + boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight"); + if (isObserver || isSentry) { + //System.err.println(e); + Entry entry = new Entry(time); + Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null); + Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null); + Integer z = getEntityProperty(e, "CBodyComponent.m_cellZ", null); + Integer[] pos = {x, y}; + entry.x = x; + entry.y = y; + entry.z = z; + if (entityLeft) + { + entry.type = isObserver ? "obs_left" : "sen_left"; + } + else + { + entry.type = isObserver ? "obs" : "sen"; + } + entry.key = Arrays.toString(pos); + entry.entityleft = entityLeft; + entry.ehandle = e.getHandle(); + //System.err.println(entry.key); + Integer owner = getEntityProperty(e, "m_hOwnerEntity", null); + Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner); + entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null; + //2/3 radiant/dire + //entry.team = e.getProperty("m_iTeamNum"); + output(entry); + } + } public void run(String[] args) throws Exception { long tStart = System.currentTimeMillis(); diff --git a/json/navbar_pages.json b/json/navbar_pages.json index 6ddb4efd9..d57cb79fa 100644 --- a/json/navbar_pages.json +++ b/json/navbar_pages.json @@ -9,10 +9,6 @@ "benchmarks": { "name": "Benchmarks" }, - "top": { - "name": "Top", - "hide": true - }, "distributions": { "name": "Distributions" }, diff --git a/package.json b/package.json index b88d0ee1f..011261da3 100644 --- a/package.json +++ b/package.json @@ -55,7 +55,7 @@ "http-proxy": "^1.13.1", "jade": "^1.11.0", "jquery": "^2.2.0", - "knex": "^0.10.0", + "knex": "^0.11.7", "moment": "^2.11.2", "multer": "^1.1.0", "numeral": "^1.5.3", diff --git a/processors/populate.js b/processors/populate.js index 39d61bc47..35581c816 100644 --- a/processors/populate.js +++ b/processors/populate.js @@ -89,5 +89,4 @@ function populate(e, container) break; } } - module.exports = populate; \ No newline at end of file diff --git a/processors/processExpand.js b/processors/processExpand.js index 2e97f7bd1..ea92cd0b5 100644 --- a/processors/processExpand.js +++ b/processors/processExpand.js @@ -427,25 +427,25 @@ function processExpand(entries, meta) }, "obs": function(e) { - //key is a JSON array of position data - e.key = JSON.parse(e.key); - e.posData = true; - expand(e); var e2 = JSON.parse(JSON.stringify(e)); - e2.posData = false; e2.type = "obs_log"; expand(e2); + var e3 = JSON.parse(JSON.stringify(e)); + //key is a JSON array of position data + e3.key = JSON.parse(e3.key); + e3.posData = true; + expand(e3); }, "sen": function(e) { - e.key = JSON.parse(e.key); - e.posData = true; - expand(e); var e2 = JSON.parse(JSON.stringify(e)); - e2.posData = false; e2.type = "sen_log"; expand(e2); - } + var e3 = JSON.parse(JSON.stringify(e)); + e3.key = JSON.parse(e3.key); + e3.posData = true; + expand(e3); + }, }; //define the types we want to put into each array //null means all types diff --git a/processors/processReduce.js b/processors/processReduce.js index 41493a8fa..2d7f709a1 100644 --- a/processors/processReduce.js +++ b/processors/processReduce.js @@ -1,22 +1,85 @@ /** - * A processor to reduce the event stream by grouping similar events. - * NOT CURRENTLY IN PRODUCTION USE + * A processor to reduce the event stream to only logs we want to persist **/ -function processReduce(entries) +function processReduce(entries, match, meta) { - var reduceMap = {}; - //group by player_slot, type, targethero, targetillusion - for (var i = 0; i < entries.length; i++) + //for now, disable log parsing for regular matches + if (!match.doLogParse) { - //group big categories: actions, combat log damage - var e = entries[i]; - var identifier = [e.player_slot, e.type, e.key].join(":"); - e.value = e.value || 1; - //var identifier = e.type; - //e.value = 1; - reduceMap[identifier] = reduceMap[identifier] ? reduceMap[identifier] + e.value : e.value || 1; + return; } - //var fs = require('fs'); - //fs.writeFileSync('./output3.json', JSON.stringify(reduceMap, null , 2)); + var basicLogTypes = { + "obs": 1, + "sen": 1, + "obs_left": 1, + "sen_left": 1, + }; + var result = entries.filter(function(e) + { + if (match.doLogParse) + { + if (e.type === "actions") + { + return false; + } + if (e.type === "DOTA_COMBATLOG_MODIFIER_REMOVE") + { + return false; + } + if (e.type === "DOTA_COMBATLOG_DAMAGE" || e.type === "DOTA_COMBATLOG_MODIFIER_ADD" || e.type === "DOTA_COMBATLOG_HEAL") + { + if (!e.targethero || e.targetillusion) + { + return false; + } + } + if (e.type === "interval" && e.time % 60 !== 0) + { + return false; + } + if (!e.time) + { + return false; + } + return true; + } + else + { + return (e.type in basicLogTypes); + } + }).map(function(e) + { + var e2 = Object.assign( + {}, e, + { + match_id: match.match_id, + attackername_slot: meta.hero_to_slot[e.attackername], + targetname_slot: meta.hero_to_slot[e.targetname], + sourcename_slot: meta.hero_to_slot[e.sourcename], + targetsourcename_slot: meta.hero_to_slot[e.targetname], + player1_slot: meta.slot_to_playerslot[e.player1], + player_slot: e.player_slot || meta.slot_to_playerslot[e.slot], + inflictor: translate(e.inflictor), + }); + delete e2.attackername; + delete e2.targetname; + delete e2.sourcename; + delete e2.targetsourcename; + return e2; + }); + /* + var count = {}; + result.forEach(function(r) + { + count[r.type] = (count[r.type] || 0) + 1; + }); + console.log(count); + */ + return result; +} + +function translate(s) +{ + return s === "dota_unknown" ? null : s; } module.exports = processReduce; \ No newline at end of file diff --git a/routes/api.js b/routes/api.js index c2afa0c3f..e901914fd 100644 --- a/routes/api.js +++ b/routes/api.js @@ -134,6 +134,93 @@ module.exports = function(db, redis, cassandra) }); }); */ + /* + api.get('/match_logs/:match_id', function(req, res, cb) + { + db.raw(`SELECT * FROM match_logs WHERE match_id = ? ORDER BY time ASC`, [req.params.match_id]).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); + */ + api.get('/pro_matches', function(req, res, cb) + { + db.raw(` + SELECT match_id, start_time, duration, ma.leagueid, name + FROM matches ma + JOIN leagues le + ON ma.leagueid = le.leagueid + WHERE ma.leagueid > 0 + ORDER BY match_id DESC + `).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); + api.get('/pro_players', function(req, res, cb) + { + queries.getProPlayers(db, redis, function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result); + }); + }); + api.get('/drafts', function(req, res, cb) + { + db.raw(` + SELECT pb.hero_id, + sum(case when ((pm.player_slot < 128) = m.radiant_win) then 1 else 0 end) wins, + sum(case when is_pick is true then 1 else 0 end) picks, + sum(case when is_pick is false then 1 else 0 end) bans + FROM picks_bans pb + LEFT JOIN matches m + ON pb.match_id = m.match_id + LEFT JOIN player_matches pm + ON pb.hero_id = pm.hero_id + AND pm.match_id = m.match_id + GROUP BY pb.hero_id; + `).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); + api.get('/pick_order', function(req, res, cb) + { + db.raw(`SELECT hero_id, ord, count( * ) FROM picks_bans WHERE is_pick is true GROUP BY hero_id, ord;`).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); + api.get('/leagues', function(req, res, cb) + { + db.raw(`SELECT * FROM leagues ORDER BY leagueid DESC`).asCallback(function(err, result) + { + if (err) + { + return cb(err); + } + res.json(result.rows); + }); + }); api.get('/distributions', function(req, res, cb) { queries.getDistributions(redis, function(err, result) diff --git a/sql/cassandra.cql b/sql/cassandra.cql index 4c1572049..5b075cbf6 100644 --- a/sql/cassandra.cql +++ b/sql/cassandra.cql @@ -25,7 +25,7 @@ CREATE TABLE matches ( radiant_gold_adv text, radiant_xp_adv text, teamfights text, - version int, + version int ); CREATE TABLE player_matches ( diff --git a/sql/create_tables.sql b/sql/create_tables.sql index 134bda3f7..cd234ab73 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -39,6 +39,7 @@ CREATE TABLE matches ( teamfights json[], version integer ); +CREATE INDEX on matches(leagueid) WHERE leagueid > 0; CREATE TABLE player_matches ( PRIMARY KEY(match_id, player_slot), @@ -174,3 +175,70 @@ CREATE TABLE notable_players ( is_pro boolean, locked_until integer ); + +CREATE TABLE match_logs ( + match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, + time int, + type varchar(100), + team smallint, + unit varchar(100), + key varchar(1000), + value int, + slot smallint, + player_slot smallint, + player1 int, + player2 int, + attackerhero boolean, + targethero boolean, + attackerillusion boolean, + targetillusion boolean, + inflictor varchar(100), + gold_reason smallint, + xp_reason smallint, + valuename varchar(100), + gold int, + lh int, + xp int, + x smallint, + y smallint, + z smallint, + entityleft boolean, + ehandle int, + stuns real, + hero_id smallint, + life_state smallint, + level smallint, + kills smallint, + deaths smallint, + assists smallint, + denies smallint, + attackername_slot smallint, + targetname_slot smallint, + sourcename_slot smallint, + targetsourcename_slot smallint, + player1_slot smallint +); +CREATE INDEX ON match_logs(match_id); +CREATE INDEX ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL; +CREATE INDEX ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL; + +CREATE TABLE picks_bans( + match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, + is_pick boolean, + hero_id int, + team smallint, + ord smallint, + PRIMARY KEY (match_id, ord) +); + +CREATE TABLE leagues( + leagueid bigint PRIMARY KEY, + ticket varchar(255), + banner varchar(255), + tier varchar(255), + name varchar(255) +); diff --git a/store/buildSets.js b/store/buildSets.js index fed99d429..2c3794d77 100644 --- a/store/buildSets.js +++ b/store/buildSets.js @@ -18,20 +18,6 @@ module.exports = function buildSets(db, redis, cb) { cb(err, t); }); }, - //users in this set have their matches added - "userPlayers": function(cb) { - db.select(['account_id']).from('players').whereNotNull('last_login').asCallback(function(err, docs) { - if (err) { - return cb(err); - } - var t = {}; - docs.forEach(function(player) { - t[player.account_id] = true; - }); - //console.log(t); - cb(err, t); - }); - }, //users in this set are added to the trackedPlayers set "donators": function(cb) { db.select(['account_id']).from('players').where('cheese', '>', 0).asCallback(function(err, docs) { diff --git a/store/queries.js b/store/queries.js index c602eaacb..91f31c436 100644 --- a/store/queries.js +++ b/store/queries.js @@ -32,13 +32,6 @@ function getSets(redis, cb) cb(err, JSON.parse(tps || "{}")); }); }, - "userPlayers": function(cb) - { - redis.get("userPlayers", function(err, ups) - { - cb(err, JSON.parse(ups || "{}")); - }); - }, "donators": function(cb) { redis.get("donators", function(err, ds) @@ -131,7 +124,7 @@ function upsert(db, table, row, conflict, cb) return util.format("%s=%s", key, "EXCLUDED." + key); }); var query = util.format("INSERT INTO %s (%s) VALUES (%s) ON CONFLICT (%s) DO UPDATE SET %s", table, Object.keys(row).join(','), values, Object.keys(conflict).join(','), update.join(',')); - //require('fs').writeFileSync('output.json', query); + //console.log(query.toString()); db.raw(query, Object.keys(row).map(function(key) { return row[key]; @@ -168,7 +161,7 @@ function insertMatch(db, redis, match, options, cb) players.forEach(function(p, i) { match.pgroup[p.player_slot] = { - account_id: p.account_id, + account_id: p.account_id || null, hero_id: p.hero_id, player_slot: p.player_slot }; @@ -191,6 +184,7 @@ function insertMatch(db, redis, match, options, cb) //we want to insert into matches, then insert into player_matches for each entry in players async.series( { + "dlp": decideLogParse, "u": upsertMatch, "uc": upsertMatchCassandra, "upc": updatePlayerCaches, @@ -205,23 +199,48 @@ function insertMatch(db, redis, match, options, cb) return cb(err, results.dp); }); + function decideLogParse(cb) + { + if (match.leagueid) + { + redis.sismember('pro_leagueids', match.leagueid, function(err, result) + { + options.doLogParse = options.doLogParse || Boolean(Number(result)); + cb(err); + }); + } + else + { + cb(); + } + } + function upsertMatch(cb) { - if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE) + if (!config.ENABLE_POSTGRES_MATCH_STORE_WRITE && !options.doLogParse) { return cb(); } db.transaction(function(trx) { - upsert(trx, 'matches', match, + async.series( { - match_id: match.match_id - }, function(err) + "m": upsertMatch, + "pm": upsertPlayerMatches, + "pb": upsertPicksBans, + "l": upsertMatchLogs, + }, exit); + + function upsertMatch(cb) { - if (err) + upsert(trx, 'matches', match, { - return exit(err); - } + match_id: match.match_id + }, cb); + } + + function upsertPlayerMatches(cb) + { async.each(players || [], function(pm, cb) { pm.match_id = match.match_id; @@ -230,22 +249,59 @@ function insertMatch(db, redis, match, options, cb) match_id: pm.match_id, player_slot: pm.player_slot }, cb); - }, exit); + }, cb); + } - function exit(err) + function upsertPicksBans(cb) + { + async.each(match.picks_bans || [], function(p, cb) { - if (err) + //order is a reserved keyword + p.ord = p.order; + p.match_id = match.match_id; + upsert(trx, 'picks_bans', p, { - console.error(err); - trx.rollback(err); - } - else + match_id: p.match_id, + ord: p.ord + }, cb); + }, cb); + } + + function upsertMatchLogs(cb) + { + if (!match.logs) + { + return cb(); + } + else + { + trx.raw(`DELETE FROM match_logs WHERE match_id = ?`, [match.match_id]).asCallback(function(err) { - trx.commit(); - } - cb(err); + if (err) + { + return cb(err); + } + async.eachLimit(match.logs, 10000, function(e, cb) + { + trx('match_logs').insert(e).asCallback(cb); + }, cb); + }); } - }); + } + + function exit(err) + { + if (err) + { + console.error(err); + trx.rollback(err); + } + else + { + trx.commit(); + } + cb(err); + } }); } @@ -363,7 +419,7 @@ function insertMatch(db, redis, match, options, cb) { async.each(match.players, function(p, cb) { - if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && (p.account_id in options.userPlayers || (config.ENABLE_RANDOM_MMR_UPDATE && match.match_id % 3 === 0))) + if (options.origin === "scanner" && match.lobby_type === 7 && p.account_id && p.account_id !== constants.anonymous_account_id && config.ENABLE_RANDOM_MMR_UPDATE) { addToQueue(mQueue, { @@ -397,7 +453,7 @@ function insertMatch(db, redis, match, options, cb) function decideParse(cb) { - if (match.parse_status !== 0) + if (options.skipParse) { //not parsing this match //this isn't a error, although we want to report that we refused to parse back to user if it was a request @@ -414,6 +470,7 @@ function insertMatch(db, redis, match, options, cb) duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, + doLogParse: options.doLogParse, }, { lifo: options.lifo, @@ -667,7 +724,7 @@ function expectedWin(rates) return 1 - rates.reduce((prev, curr) => (100 - curr * 100) * prev, 1) / (Math.pow(50, rates.length - 1) * 100); } -function getTop(db, redis, cb) +function getProPlayers(db, redis, cb) { db.raw(` SELECT * from notable_players @@ -677,14 +734,7 @@ function getTop(db, redis, cb) { return cb(err); } - getLeaderboard(db, redis, 'solo_competitive_rank', 500, function(err, result2) - { - return cb(err, - { - notables: result.rows, - leaderboard: result2 - }); - }); + return cb(err, result.rows); }); } @@ -906,7 +956,7 @@ module.exports = { insertMatchSkill, getDistributions, getPicks, - getTop, + getProPlayers, getHeroRankings, getBenchmarks, benchmarkMatch, diff --git a/svc/fullhistory.js b/svc/fullhistory.js index 312034a51..d5707ac4a 100644 --- a/svc/fullhistory.js +++ b/svc/fullhistory.js @@ -90,6 +90,7 @@ function processFullHistory(job, cb) type: "api", cassandra: cassandra, skipAbilityUpgrades: true, + skipParse: true, }, cb); }); }, function(err) diff --git a/svc/parser.js b/svc/parser.js index 90ed32b2f..906fa5069 100644 --- a/svc/parser.js +++ b/svc/parser.js @@ -91,7 +91,6 @@ pQueue.process(config.PARSER_PARALLELISM, function(job, cb) parsed_data.start_time = match.start_time; parsed_data.duration = match.duration; parsed_data.replay_blob_key = match.replay_blob_key; - parsed_data.parse_status = 2; if (match.replay_blob_key) { insertUploadedParse(parsed_data, cb); @@ -158,6 +157,7 @@ function insertStandardParse(match, cb) { type: "parsed", cassandra: cassandra, + skipParse: true, }, cb); } @@ -245,7 +245,7 @@ function runParse(match, job, cb) } entries.push(e); }); - request.debug = true + request.debug = true; function exit(err) { @@ -267,6 +267,7 @@ function runParse(match, job, cb) var message = "time spent on post-processing match "; console.time(message); var meta = processMetadata(entries); + var logs = processReduce(entries, match, meta); var res = processExpand(entries, meta); var parsed_data = processParsedData(res.parsed_data); var teamfights = processTeamfights(res.tf_data, meta); @@ -276,8 +277,8 @@ function runParse(match, job, cb) parsed_data.radiant_gold_adv = ap.radiant_gold_adv; parsed_data.radiant_xp_adv = ap.radiant_xp_adv; parsed_data.upload = upload; + parsed_data.logs = logs; //processMultiKillStreaks(); - //processReduce(res.expanded); console.timeEnd(message); return cb(err, parsed_data); } diff --git a/svc/requests.js b/svc/requests.js index c6007a733..08901defb 100644 --- a/svc/requests.js +++ b/svc/requests.js @@ -29,13 +29,13 @@ function processRequest(job, cb) } //match details response var match = body.result; - match.parse_status = 0; insertMatch(db, redis, match, { type: "api", attempts: 1, lifo: true, cassandra: cassandra, + skipParse: false, }, waitParse); }); } diff --git a/svc/scanner.js b/svc/scanner.js index 00d6e3215..819db762d 100644 --- a/svc/scanner.js +++ b/svc/scanner.js @@ -13,7 +13,6 @@ var getData = utility.getData; var generateJob = utility.generateJob; var async = require('async'); var trackedPlayers; -var userPlayers; var parallelism = config.SCANNER_PARALLELISM; var PAGE_SIZE = 100; buildSets(db, redis, function(err) @@ -72,7 +71,6 @@ function start() } //set local vars trackedPlayers = result.trackedPlayers; - userPlayers = result.userPlayers; var arr = []; var matchBuffer = {}; var completePages = {}; @@ -112,26 +110,19 @@ function start() function processMatch(match, cb) { - if (config.ENABLE_PRO_PARSING && match.leagueid) - { - //parse tournament games - match.parse_status = 0; - } - else if (match.players.some(function(p) + var insert = false; + var skipParse = true; + if (match.players.some(function(p) { return (p.account_id in trackedPlayers); })) { - //queued - match.parse_status = 0; + insert = true; + skipParse = false; } - else if (match.players.some(function(p) - { - return (config.ENABLE_INSERT_ALL_MATCHES || p.account_id in userPlayers); - })) + else if (config.ENABLE_INSERT_ALL_MATCHES) { - //skipped - match.parse_status = 3; + insert = true; } //check if match was previously processed redis.get('scanner_insert:' + match.match_id, function(err, result) @@ -142,7 +133,7 @@ function start() } //don't insert this match if we already processed it recently //deduplicate matches in this page set - if ((match.parse_status === 0 || match.parse_status === 3) && !result && !matchBuffer[match.match_id]) + if (insert && !result && !matchBuffer[match.match_id]) { matchBuffer[match.match_id] = 1; insertMatch(db, redis, match, @@ -150,7 +141,7 @@ function start() type: "api", origin: "scanner", cassandra: cassandra, - userPlayers: userPlayers, + skipParse: skipParse, }, function(err) { if (!err) diff --git a/svc/web.js b/svc/web.js index bd493a5ad..50ee45663 100644 --- a/svc/web.js +++ b/svc/web.js @@ -284,17 +284,6 @@ app.get('/picks/:n?', function(req, res, cb) }); }); }); -app.get('/top', function(req, res, cb) -{ - queries.getTop(db, redis, function(err, result) - { - if (err) - { - return cb(err); - } - res.render('top', result); - }); -}); app.get('/rankings/:hero_id?', function(req, res, cb) { if (!req.params.hero_id) @@ -384,6 +373,7 @@ app.use(function(req, res, next) { if (config.UI_HOST) { + //route not found, redirect to SPA return res.redirect(config.UI_HOST + req.url); } var err = new Error("Not Found"); diff --git a/svc/worker.js b/svc/worker.js index a064db05e..68fec5d89 100644 --- a/svc/worker.js +++ b/svc/worker.js @@ -124,6 +124,43 @@ invokeInterval(function notablePlayers(cb) }, cb); }); }, 10 * 60 * 1000); +invokeInterval(function leagues(cb) +{ + var container = utility.generateJob("api_leagues", + {}); + utility.getData(container.url, function(err, api_leagues) + { + if (err) + { + return cb(err); + } + utility.getData('https://raw.githubusercontent.com/dotabuff/d2vpkr/master/dota/scripts/items/leagues.json', function(err, leagues) + { + if (err) + { + return cb(err); + } + async.each(api_leagues.result.leagues, function(l, cb) + { + if (leagues[l.leagueid]) + { + l.tier = leagues[l.leagueid].tier; + l.ticket = leagues[l.leagueid].ticket; + l.banner = leagues[l.leagueid].banner; + } + l.name = l.description.substring("#DOTA_Item_Desc_".length).split('_').join(' '); + if (l.tier === "professional" || l.tier === "premium") + { + redis.sadd('pro_leagueids', l.leagueid); + } + queries.upsert(db, 'leagues', l, + { + leagueid: l.league_id + }, cb); + }, cb); + }); + }); +}, 10 * 60 * 1000); function invokeInterval(func, delay) { diff --git a/tasks/getMatches.js b/tasks/getMatches.js new file mode 100644 index 000000000..0362a1b95 --- /dev/null +++ b/tasks/getMatches.js @@ -0,0 +1,50 @@ +var async = require('async'); +var utility = require('../util/utility'); +var generateJob = utility.generateJob; +var getData = utility.getData; +var db = require('../store/db'); +var redis = require('../store/redis'); +var cassandra = require('../store/cassandra'); +var queries = require('../store/queries'); +var insertMatch = queries.insertMatch; +var args = process.argv.slice(2); +var match_id = Number(args[0]); +var delay = 1000; +var job = generateJob("api_details", +{ + match_id: match_id +}); +var url = job.url; +getData( +{ + url: url, + delay: delay +}, function(err, body) +{ + if (err) + { + throw err; + } + if (body.result) + { + var match = body.result; + insertMatch(db, redis, match, + { + skipCounts: true, + skipAbilityUpgrades: true, + skipParse: false, + cassandra: cassandra, + }, function(err) + { + if (err) + { + throw err; + } + process.exit(0); + }); + } + else + { + throw body; + } +}); \ No newline at end of file diff --git a/dev/allMatches.js b/tasks/getMatchesSeq.js similarity index 98% rename from dev/allMatches.js rename to tasks/getMatchesSeq.js index ebd445e36..331c5bb1c 100644 --- a/dev/allMatches.js +++ b/tasks/getMatchesSeq.js @@ -86,6 +86,7 @@ function getPage(match_seq_num, bucket) { skipCounts: true, skipAbilityUpgrades: true, + skipParse: true, cassandra: cassandra, }, cb); }, function(err) diff --git a/test/test.js b/test/test.js index 73af32d1d..e6e4cebd7 100644 --- a/test/test.js +++ b/test/test.js @@ -128,7 +128,8 @@ before(function(done) { queries.insertMatch(db, redis, m, { - type: "api" + type: "api", + skipParse: true, }, cb); }, cb); }, diff --git a/util/analysis.js b/util/analysis.js index 8f9f2f163..6aca06a63 100644 --- a/util/analysis.js +++ b/util/analysis.js @@ -4,6 +4,7 @@ **/ var util = require('util'); var constants = require('../constants.js'); + function generatePlayerAnalysis(match, pm) { //define condition check for each advice point @@ -150,7 +151,8 @@ function generatePlayerAnalysis(match, pm) { var flying_available = 180; var time; - if (pm.purchase && pm.first_purchase_time && pm.first_purchase_time.flying_courier) { + if (pm.purchase && pm.first_purchase_time && pm.first_purchase_time.flying_courier) + { time = pm.first_purchase_time.flying_courier; } return { @@ -172,7 +174,7 @@ function generatePlayerAnalysis(match, pm) wards: function(m, pm) { var ward_cooldown = 60 * 7; - var wards = pm.obs_log ? pm.obs_log.length : 0; + var wards = getObsWardsPlaced(pm); //divide game length by ward cooldown //2 wards respawn every interval //split responsibility between 2 supports @@ -304,7 +306,19 @@ function generatePlayerAnalysis(match, pm) function isSupport(pm) { - return pm.obs_log && pm.obs_log.length >= 2 && pm.lh_t && pm.lh_t[10] < 20; + return getObsWardsPlaced(pm) >= 2 && pm.lh_t && pm.lh_t[10] < 20; + } + + function getObsWardsPlaced(pm) + { + if (!pm.obs_log) + { + return 0; + } + return pm.obs_log.filter(function(l) + { + return !l.entityleft; + }).length; } function isRoshHero(pm) diff --git a/views/match/match.jade b/views/match/match.jade index 53673b8d2..6d0a345d7 100644 --- a/views/match/match.jade +++ b/views/match/match.jade @@ -32,7 +32,6 @@ block content th Percentile tbody tr - //td= constants.parse_status[match.parse_status] td= constants.game_mode[match.game_mode] ? constants.game_mode[match.game_mode].name : match.game_mode td= constants.region[match.region] ? constants.region[match.region] : match.region td= moment().startOf('day').seconds(match.duration).format("H:mm:ss") diff --git a/views/match/match_performances.jade b/views/match/match_performances.jade index b78029092..2edb420df 100644 --- a/views/match/match_performances.jade +++ b/views/match/match_performances.jade @@ -9,8 +9,6 @@ block match_content tr th: abbr(title=tooltips.hero_id) Hero th: abbr(title=tooltips.lane) Lane - //th: abbr(title=tooltips.used_ward_observer) Observer - //th: abbr(title=tooltips.used_ward_sentry) Sentry th: abbr(title=tooltips.lane_efficiency) EFF@10 th: abbr(title=tooltips.lhten) LH@10 th: abbr(title=tooltips.stuns) Stuns @@ -19,8 +17,6 @@ block match_content tr.activate(data-index=i, data-type="lane_pos", class = player.isRadiant ? "radiant" : "dire") +hero_td(player) td #{constants.lane_ids[player.lane] || "-"} (#{constants.lane_role[player.lane_role] || "-"}) - //td.activate(data-index=i, data-type="obs") #{player.obs_log.length || "-"} - //td.activate(data-index=i, data-type="sen") #{player.sen_log.length || "-"} td.rankable #{(Number(player.lane_efficiency)*100).toFixed(2)} td.rankable #{player.lh_t && player.lh_t[10] ? player.lh_t[10] : "-"} td.rankable #{Number(Math.max(player.stuns, 0)).toFixed(2)}