fix conflicts

dev/postgresToCassandra.js

@@ -58,6 +58,7 @@ function run(start_id)
     {
         stream.pause();
         redis.set('postgresToCassandra:' + bucket, match.match_id);
+        delete match.parse_status;
         insertMatch(match, function(err)
         {
             if (err)

json/player_fields.json

@@ -42,7 +42,6 @@
     },
     "times":
     {
-        "duration": 1,
-        "first_blood_time": 1
+        "duration": 1
     }
 }

json/player_pages.json

@@ -14,6 +14,9 @@
   "activity": {
     "name": "Activity"
   },
+  "records": {
+    "name": "Records"
+  },
   "counts": {
     "name": "Counts"
   },
@@ -29,14 +32,11 @@
   "items": {
     "name": "Items"
   },
-  "skills": {
-    "name": "Skills"
-  },
   "wordcloud": {
     "name": "Wordcloud"
   },
   "rating": {
-    "name": "Rating"
+    "name": "MMR"
   },
   "rankings": {
     "name": "Rankings",

package.json

@@ -9,7 +9,7 @@
     "test": "test/run.sh",
     "coveralls": "cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js && rm -rf ./coverage",
     "ncu": "npm-check-updates -u",
-    "webpack": "webpack -p",
+    "webpack": "webpack -p --bail",
     "watch": "webpack --watch",
     "mocha": "mocha",
     "maven": "mvn -q -f java_parser/pom.xml clean install -U",

public/js/createHistogram.js

@@ -108,8 +108,7 @@ window.createHistogram = function createHistogram(counts, win_counts, label)
                         t = Number(t) / scalef;
                         //these labels are times and need to be formatted
                         var times = {
-                            "duration": 1,
-                            "first_blood_time": 1
+                            "duration": 1
                         };
                         if (times[label])
                         {

scripts/gce.sh

@@ -59,9 +59,4 @@ gcloud compute instance-templates delete -q task-1
 gcloud compute instance-templates create task-1 --preemptible --machine-type n1-highcpu-8 --image container-vm --boot-disk-size 50GB --boot-disk-type pd-ssd --metadata startup-script='#!/bin/bash
 sudo docker run -d --name task --restart=always --net=host yasp/yasp:latest sh -c "curl -H \"Metadata-Flavor: Google\" -L http://metadata.google.internal/computeMetadata/v1/project/attributes/env > /usr/src/yasp/.env && node dev/postgresToCassandra.js 0 2300000000"
 '
-gcloud compute instance-groups managed create "task-group-1" --base-instance-name "task-group-1" --template "task-1" --size "1"
-
-#Import a Postgres dump:
-sudo docker exec -i postgres pg_restore -d yasp --clean < ../dump.pgdump
-#Run a script in production:
-sudo docker run -d --name task --restart=always --net=host yasp/yasp:latest sh -c 'curl -H "Metadata-Flavor: Google" -L http://metadata.google.internal/computeMetadata/v1/project/attributes/env > /usr/src/yasp/.env && node dev/preloader.js'
+gcloud compute instance-groups managed create "task-group-1" --base-instance-name "task-group-1" --template "task-1" --size "1"

scripts/init.sh

@@ -48,4 +48,9 @@ sudo apt-get -y install redis-server postgresql-9.5
 #sudo apt-get -y install cassandra
 
 #clean up old versions
-sudo apt-get -y purge postgresql-9.3
+sudo apt-get -y purge postgresql-9.3
+
+#Import a Postgres dump:
+sudo docker exec -i postgres pg_restore -d yasp -U postgres --clean < ../dump.pgdump
+#Run a script in production:
+sudo docker run -d --name task --restart=always --net=host yasp/yasp:latest sh -c 'curl -H "Metadata-Flavor: Google" -L http://metadata.google.internal/computeMetadata/v1/project/attributes/env > /usr/src/yasp/.env && node dev/preloader.js'

sql/cassandra.cql

@@ -19,7 +19,6 @@ CREATE TABLE yasp.matches (
   game_mode int,
   engine int,
   picks_bans text,
-  parse_status int,
   chat text,
   objectives text,
   radiant_gold_adv text,
@@ -95,17 +94,12 @@ CREATE TABLE yasp.player_caches (
   match_id bigint,
   player_slot text,
   heroes text,
-  teammates text,
-  win text,
-  lose text,
   radiant_win text,
-  abandons text,
   start_time text,
   duration text,
   cluster text,
   region text,
   patch text,
-  first_blood_time text,
   lobby_type text,
   game_mode text,
   level text,
@@ -122,7 +116,6 @@ CREATE TABLE yasp.player_caches (
   xp_per_min text,
   hero_id text,
   leaver_status text,
-  isRadiant text,
   version text,
   courier_kills text,
   tower_kills text,
@@ -136,8 +129,6 @@ CREATE TABLE yasp.player_caches (
   item_usage text,
   item_win text,
   purchase text,
-  ability_uses text,
-  hero_hits text,
   multi_kills text,
   kill_streaks text,
   all_word_counts text,
@@ -155,7 +146,6 @@ CREATE TABLE yasp.player_caches (
   pings text,
   stuns text,
   lane_efficiency_pct text,
-  parse_status text,
   skill text,
 );
 

sql/create_tables.sql

@@ -32,7 +32,6 @@ CREATE TABLE matches (
   --dire_team_id integer,
   pgroup json,
   --parsed data below
-  parse_status integer,
   chat json[],
   objectives json[],
   radiant_gold_adv integer[],

store/buildMatch.js

@@ -99,7 +99,6 @@ function getMatch(db, redis, match_id, options, cb)
                     p.cluster = match.cluster;
                     p.lobby_type = match.lobby_type;
                     p.game_mode = match.game_mode;
-                    p.parse_status = match.parse_status;
                     computeMatchData(p);
                     if (ab_upgrades)
                     {
@@ -176,7 +175,7 @@ function getMatch(db, redis, match_id, options, cb)
         }
         else
         {
-            db.first(['matches.match_id', 'match_skill.skill', 'radiant_win', 'start_time', 'duration', 'tower_status_dire', 'tower_status_radiant', 'barracks_status_dire', 'barracks_status_radiant', 'cluster', 'lobby_type', 'leagueid', 'game_mode', 'picks_bans', 'parse_status', 'chat', 'teamfights', 'objectives', 'radiant_gold_adv', 'radiant_xp_adv', 'version']).from('matches').leftJoin('match_skill', 'matches.match_id', 'match_skill.match_id').where(
+            db.first(['matches.match_id', 'match_skill.skill', 'radiant_win', 'start_time', 'duration', 'tower_status_dire', 'tower_status_radiant', 'barracks_status_dire', 'barracks_status_radiant', 'cluster', 'lobby_type', 'leagueid', 'game_mode', 'picks_bans', 'chat', 'teamfights', 'objectives', 'radiant_gold_adv', 'radiant_xp_adv', 'version']).from('matches').leftJoin('match_skill', 'matches.match_id', 'match_skill.match_id').where(
             {
                 "matches.match_id": Number(match_id)
             }).asCallback(cb);

store/buildPlayer.js

@@ -22,28 +22,29 @@ var subkeys = player_fields.subkeys;
 var countCats = player_fields.countCats;
 //Fields to project from Cassandra player caches
 var cacheProj = ['account_id', 'match_id', 'player_slot', 'version', 'start_time', 'duration', 'game_mode', 'lobby_type', 'radiant_win', 'hero_id', 'game_mode', 'skill', 'duration', 'kills', 'deaths', 'assists', 'last_hits', 'gold_per_min'];
-var cacheFilters = ['lane_role', 'game_mode', 'lobby_type', 'region', 'patch', 'start_time', 'lane_role'];
+var cacheFilters = ['heroes', 'hero_id', 'lane_role', 'game_mode', 'lobby_type', 'region', 'patch', 'start_time'];
 //Fields to aggregate on
 //optimize by only aggregating certain columns based on tab
 //set query.js_agg based on this
-var basicAggs = ['match_id', 'version', 'abandons', 'win', 'lose'];
+var basicAggs = ['match_id', 'win', 'lose'];
 var aggs = {
     index: basicAggs.concat('heroes'),
     matches: basicAggs,
     heroes: basicAggs.concat('heroes'),
-    peers: basicAggs.concat('teammates'),
+    peers: basicAggs.concat(['heroes', 'teammates']),
     activity: basicAggs.concat('start_time'),
-    counts: basicAggs.concat(Object.keys(subkeys)).concat(Object.keys(countCats)).concat(['multi_kills', 'kill_streaks', 'lane_role']),
     //TODO only need one subkey at a time
+    records: basicAggs.concat(Object.keys(subkeys)),
+    counts: basicAggs.concat(Object.keys(countCats)).concat(['multi_kills', 'kill_streaks', 'lane_role']),
     histograms: basicAggs.concat(Object.keys(subkeys)),
     trends: basicAggs.concat(Object.keys(subkeys)),
     wardmap: basicAggs.concat(['obs', 'sen']),
     items: basicAggs.concat(['purchase_time', 'item_usage', 'item_uses', 'purchase', 'item_win']),
-    skills: basicAggs.concat(['hero_hits', 'ability_uses']),
     wordcloud: basicAggs.concat(['my_word_counts', 'all_word_counts']),
     rating: basicAggs,
     rankings: basicAggs,
 };
+
 function buildPlayer(options, cb)
 {
     var db = options.db;
@@ -121,7 +122,6 @@ function buildPlayer(options, cb)
                 {
                     return cb(err);
                 }
-                //don't need to wait for cache write
                 processResults(err, results);
             });
         }
@@ -166,7 +166,7 @@ function buildPlayer(options, cb)
                 {
                     if (info === "index" || info === "matches")
                     {
-                        var project = ["match_id", "player_slot", "hero_id", "game_mode", "kills", "deaths", "assists", "parse_status", "skill", "radiant_win", "start_time", "duration"].concat(queryObj.keywords.desc || []);
+                        var project = ["match_id", "player_slot", "hero_id", "game_mode", "kills", "deaths", "assists", "version", "skill", "radiant_win", "start_time", "duration"].concat(queryObj.keywords.desc || []);
                         var limit = Number(queryObj.keywords.limit) || 20;
                         //project
                         matches = matches.map(function(pm)
@@ -314,7 +314,7 @@ function buildPlayer(options, cb)
                 },
                 aggData: function(cb)
                 {
-                    if (info === "histograms" || info === "counts" || info === "trends" || info === "items" || info === "skills")
+                    if (info === "histograms" || info === "counts" || info === "trends" || info === "items" || info === "skills" || info === "records")
                     {
                         return cb(null, aggData);
                     }

util/aggregator.js

@@ -15,10 +15,6 @@ function aggregator(matches, fields, existing)
         {},
         heroes:
         {},
-        match_ids:
-        {},
-        parsed_match_ids:
-        {},
     };
     for (var key in fields)
     {
@@ -47,30 +43,18 @@ function aggregator(matches, fields, existing)
     });
     for (var i = 0; i < matches2.length; i++)
     {
-        var m = matches2[i];
-        var significant = isSignificant(m);
+        var pm = matches2[i];
+        pm.win = Number(isRadiant(pm) === pm.radiant_win);
+        pm.lose = Number(isRadiant(pm) === pm.radiant_win) ? 0 : 1;
+        var significant = isSignificant(pm);
         for (var key in fields)
         {
-            var isDup = false;
-            if (getAggs()[key] === "api")
-            {
-                isDup = isDup || (m.match_id in aggData.match_ids);
-            }
-            else if (getAggs()[key] === "parsed")
-            {
-                isDup = isDup || (m.match_id in aggData.parsed_match_ids);
-            }
-            if (significant && !isDup)
+            if (significant)
             {
                 //execute the aggregation function for each specified field
-                standardAgg(key, m[key], m);
+                standardAgg(key, pm[key], pm);
             }
         }
-        aggData.match_ids[m.match_id] = 1;
-        if (m.version)
-        {
-            aggData.parsed_match_ids[m.match_id] = 1;
-        }
     }
     return aggData;
 
@@ -120,7 +104,7 @@ function aggregator(matches, fields, existing)
             aggObj.avgs.push(~~(aggObj.sum / aggObj.n * 100) / 100);
         }
     }
-};
+}
 
 function aggHeroes(aggData, m)
 {
@@ -188,7 +172,7 @@ function aggTeammates(aggData, m)
 {
     var teammates = aggData.teammates;
     var player_win = isRadiant(m) === m.radiant_win;
-    var group = m.teammates ||
+    var group = m.heroes ||
     {};
     for (var key in group)
     {

util/compute.js

@@ -8,16 +8,16 @@ var isRadiant = utility.isRadiant;
 var generatePositionData = utility.generatePositionData;
 var sentiment = require('sentiment');
 var ancients = {
-    npc_dota_neutral_black_drake: 1,
-    npc_dota_neutral_black_dragon: 1,
-    npc_dota_neutral_blue_dragonspawn_sorcerer: 1,
-    npc_dota_neutral_blue_dragonspawn_overseer: 1,
-    npc_dota_neutral_granite_golem: 1,
-    npc_dota_neutral_elder_jungle_stalker: 1,
-    npc_dota_neutral_rock_golem: 1,
-    npc_dota_neutral_small_thunder_lizard: 1,
-    npc_dota_neutral_jungle_stalker: 1,
-    npc_dota_neutral_big_thunder_lizard: 1,
+    "npc_dota_neutral_black_drake": 1,
+    "npc_dota_neutral_black_dragon": 1,
+    "npc_dota_neutral_blue_dragonspawn_sorcerer": 1,
+    "npc_dota_neutral_blue_dragonspawn_overseer": 1,
+    "npc_dota_neutral_granite_golem": 1,
+    "npc_dota_neutral_elder_jungle_stalker": 1,
+    "npc_dota_neutral_rock_golem": 1,
+    "npc_dota_neutral_small_thunder_lizard": 1,
+    "npc_dota_neutral_jungle_stalker": 1,
+    "npc_dota_neutral_big_thunder_lizard": 1,
 };
 var specific = {
     "npc_dota_beastmaster_boar_#": "Boar",
@@ -68,11 +68,10 @@ for (var key in specific)
     }
 }
 /**
- * Computes additional stats from stored data for a player_match
+ * Computes additional properties from a match/player_match
  **/
 function computeMatchData(pm)
 {
-    //WARNING: Don't store fields that mutate based on self_hero in Cassandra.  It isn't set post-parse and the cache update will write incorrect data.
     var self_hero = constants.heroes[pm.hero_id];
     // Compute patch based on start_time
     if (pm.start_time)
@@ -117,7 +116,6 @@ function computeMatchData(pm)
     if (pm.pgroup)
     {
         pm.heroes = pm.pgroup;
-        pm.teammates = pm.pgroup;
     }
     else
     {

util/filter.js

@@ -41,15 +41,15 @@ function filter(matches, filters)
         },
         isRadiant: function(m, key)
         {
-            return Number(m.isRadiant) === key;
+            return Number(utility.isRadiant(m)) === key;
         },
         included_account_id: function(m, key, arr)
         {
             return arr.every(function(k)
             {
-                for (var key in m.teammates)
+                for (var key in m.heroes)
                 {
-                    if (m.teammates[key].account_id === k)
+                    if (m.heroes[key].account_id === k)
                     {
                         return true;
                     }
@@ -61,9 +61,9 @@ function filter(matches, filters)
         {
             return arr.every(function(k)
             {
-                for (var key in m.teammates)
+                for (var key in m.heroes)
                 {
-                    if (m.teammates[key].account_id === k)
+                    if (m.heroes[key].account_id === k)
                     {
                         return false;
                     }