Merge remote-tracking branch 'upstream/master'

kennsippell · Jun 26, 2016 · d2eeea8 · d2eeea8
2 parents 1d2f3dc + 5ed309c
commit d2eeea8
Show file tree

Hide file tree

Showing 37 changed files with 650 additions and 310 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -4,18 +4,19 @@ FROM mhart/alpine-node:5.10.1
 RUN apk update && apk add git curl wget bash
 
 # Java
-# RUN apk update && apk add openjdk8=8.77.03-r0
+# as of 2016-06-12, running on alpine openjdk causes a unsatisfiedlinkerror crash in Java
+#RUN apk update && apk add openjdk8=8.92.14-r1
 # Here we install GNU libc (aka glibc) and set C.UTF-8 locale as default.
 
-RUN ALPINE_GLIBC_BASE_URL="https://github.com/andyshinn/alpine-pkg-glibc/releases/download" && \
-    ALPINE_GLIBC_PACKAGE_VERSION="2.23-r1" && \
+RUN ALPINE_GLIBC_BASE_URL="https://github.com/sgerrand/alpine-pkg-glibc/releases/download" && \
+    ALPINE_GLIBC_PACKAGE_VERSION="2.23-r2" && \
     ALPINE_GLIBC_BASE_PACKAGE_FILENAME="glibc-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
     ALPINE_GLIBC_BIN_PACKAGE_FILENAME="glibc-bin-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
     ALPINE_GLIBC_I18N_PACKAGE_FILENAME="glibc-i18n-$ALPINE_GLIBC_PACKAGE_VERSION.apk" && \
     apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
     wget \
-        "https://raw.githubusercontent.com/andyshinn/alpine-pkg-glibc/master/andyshinn.rsa.pub" \
-        -O "/etc/apk/keys/andyshinn.rsa.pub" && \
+        "https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$ALPINE_GLIBC_PACKAGE_VERSION/sgerrand.rsa.pub" \
+        -O "/etc/apk/keys/sgerrand.rsa.pub" && \
     wget \
         "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BASE_PACKAGE_FILENAME" \
         "$ALPINE_GLIBC_BASE_URL/$ALPINE_GLIBC_PACKAGE_VERSION/$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
@@ -25,7 +26,7 @@ RUN ALPINE_GLIBC_BASE_URL="https://github.com/andyshinn/alpine-pkg-glibc/release
         "$ALPINE_GLIBC_BIN_PACKAGE_FILENAME" \
         "$ALPINE_GLIBC_I18N_PACKAGE_FILENAME" && \
     \
-    rm "/etc/apk/keys/andyshinn.rsa.pub" && \
+    rm "/etc/apk/keys/sgerrand.rsa.pub" && \
     /usr/glibc-compat/bin/localedef --force --inputfile POSIX --charmap UTF-8 C.UTF-8 || true && \
     echo "export LANG=C.UTF-8" > /etc/profile.d/locale.sh && \
     \
@@ -91,7 +92,8 @@ RUN apk add --no-cache --virtual=build-dependencies wget ca-certificates && \
     rm "/tmp/"*
 
 # Maven
-# RUN apk update && apk add maven=3.3.9-r0
+# as of 2016-06-12 maven apk is still in edge branch and not accessible by default
+#RUN apk update && apk add maven=3.3.9-r0
 ENV MAVEN_HOME="/usr/share/maven"
 ENV MAVEN_VERSION="3.3.9"
 RUN cd / && \
@@ -115,4 +117,4 @@ RUN npm run webpack
 
 ENV PATH /usr/src/yasp/node_modules/pm2/bin:$PATH
 
-CMD [ "node", "deploy.js" ]
+CMD [ "npm", "start" ]
diff --git a/README.md b/README.md
@@ -20,15 +20,12 @@ Tech Stack
 * Storage: PostgreSQL/Redis/Cassandra
 * Parser: Java (powered by [clarity](https://github.com/skadistats/clarity))
 
-Quickstart
+Quickstart (Docker)
 ----
-* Using Docker will let you run the code in a container with all dependencies properly installed.
 * Install Docker: `curl -sSL https://get.docker.com/ | sh`
 * Clone the repo: `git clone https://github.com/yasp-dota/yasp`
 * Go into the directory: `cd yasp`
-* Create .env file with required config values in KEY=VALUE format (see config.js for a full listing of options) `cp .env_example .env`
-  * `STEAM_API_KEY` You need this in order to access the Steam Web API.  
-  * `STEAM_USER, STEAM_PASS` The retriever requires a Steam account in order to fetch replay salts.  We recommend creating a new account for this purpose (you won't be able to log into the account while the retriever is using it).  If you don't care about getting replay salts/downloading replays then you can skip this step.
+* Build the Docker container: `sudo docker build -t yasp/yasp .`
 * Start a new container running the image, and map your local directory into the container: `sudo docker run -v $(pwd):/usr/src/yasp -di --name yasp --net=host yasp/yasp:latest`
 * Start the external dependencies in separate containers.
   * `sudo docker run -d --name postgres --net=host postgres:9.5`
@@ -37,20 +34,17 @@ Quickstart
 * Initialize Postgres: `sudo docker exec -i postgres psql -U postgres < sql/init.sql`
 * Create tables: `sudo docker exec -i postgres psql -U postgres yasp < sql/create_tables.sql`
 * Set up Cassandra (optional): `sudo docker exec -i cassandra cqlsh < sql/cassandra.cql`
+* Create .env file with required config values in KEY=VALUE format (see config.js for a full listing of options) `cp .env_example .env`
+  * `STEAM_API_KEY` You need this in order to access the Steam Web API.  
+  * `STEAM_USER, STEAM_PASS` The retriever requires a Steam account in order to fetch replay salts.  We recommend creating a new account for this purpose (you won't be able to log into the account while the retriever is using it).  If you don't care about getting replay salts/downloading replays then you can skip this step.
 * Get a terminal into the running container: `sudo docker exec -it yasp bash`
-* Build inside the container: `npm run build`
 * Start the services you want to run:
-  * `pm2 start profiles/basic.json` This starts all the basic services to be able to open the site in a browser and request parses by ID (which is a useful end-to-end test).  Use `profiles/everything.json` to start everything.
-  * `pm2 start svc/web.js --watch` This starts a specific service and watches it for changes.
-  * `pm2 logs web` You can use this command to inspect the output of a service.
-  * `pm2 delete all` Stop and remove all the services.
-* Alternatively, if you have Docker Compose [installed](https://docs.docker.com/compose/install/) you can just run `docker-compose up`.
- * 3 containers will be built and launched - one with postgres database, one with redis and one with web service.
- * Database is inited and tables are created automatically.
- * By default, minimal configuration necessairy to open the site in a browser and request parses by ID is started. This can be overridden via `docker-compose.override.yml`.
- * `sudo docker exec -it yasp_web_1 bash` will give you a terminal into the running web container.
+  * `pm2 start profiles/basic.json` This starts all the basic services to be able to read the API and request parses (which is a useful end-to-end test).  Use `profiles/everything.json` to start everything.
+  * Useful PM2 commands:
+    * `pm2 start svc/web.js --watch` This starts a specific service and watches it for changes.
+    * `pm2 logs web` You can use this command to inspect the output of a service.
+    * `pm2 delete all` Stop and remove all the services.
 * Useful commands
-  * `npm run watch`: If you want to make changes to client side JS, you will want to run the watch script in order to automatically rebuild after making changes.
   * `npm test` runs the full test suite.  Use `mocha` for more fine-grained control over the tests you want to run.
   * `node tasks/updateconstants` pulls latest constants data and saves to `json` directory.
 * Get some starter data
@@ -61,6 +55,14 @@ Quickstart
 * Submit a pull request.  Wait for it to be reviewed and merged.
 * Congratulations!  You're a contributor.
 
+Docker Compose
+----
+* Alternatively, if you have Docker Compose [installed](https://docs.docker.com/compose/install/) you can just run `docker-compose up`.
+ * 3 containers will be built and launched - one with postgres database, one with redis and one with web service.
+ * Database is inited and tables are created automatically.
+ * By default, minimal configuration necessairy to open the site in a browser and request parses by ID is started. This can be overridden via `docker-compose.override.yml`.
+ * `sudo docker exec -it yasp_web_1 bash` will give you a terminal into the running web container.
+
 Getting Help
 ----
 * Feel free to open a new issue to ask questions/get help!

diff --git a/config.js b/config.js
@@ -39,10 +39,13 @@ var defaults = {
     "PROXY_URLS": "", //comma separated list of proxy urls to use
     "STEAM_API_HOST": "api.steampowered.com", //the list of hosts to fetch Steam API data from
     "ROLE": "", //for specifying the file that should be run when deploy.js is invoked
+    "GROUP": "", //for specifying the group of apps that should be run when deploy.js is invoked
     "MMSTATS_DATA_INTERVAL": 3, //minutes between requests for MMStats data
     "DEFAULT_DELAY": 1000, // delay between API requests (default: 1000)
     "SCANNER_DELAY": 300, //delay for scanner API requests (more time-sensitive)
     "SCANNER_PARALLELISM": 1, //Number of simultaneous API requests to make in scanner
+    "MMR_PARALLELISM": 15,
+    "PARSER_PARALLELISM": 1,
     "PLAYER_MATCH_LIMIT": 50000, //max results to return from player matches
     "BENCHMARK_RETENTION_HOURS": 1, //hours in block to retain benchmark data for percentile
     "MATCH_RATING_RETENTION_HOURS": 12, //hours in block to retain match rating data for percentile
@@ -51,7 +54,6 @@ var defaults = {
     "UI_HOST": "", //The host of the UI, redirect traffic from / and /return here
     "ENABLE_RECAPTCHA": "", //set to enable the recaptcha on the Request page
     "ENABLE_ADS": "", //set to turn on ads
-    "ENABLE_PRO_PARSING": "", // set to parse pro matches from sequential API
     "ENABLE_MATCH_CACHE": "", // set to enable caching matches (Redis)
     "ENABLE_PLAYER_CACHE": "", // set to enable caching players (Cassandra)
     "ENABLE_INSERT_ALL_MATCHES": "", //set to enable inserting all matches

diff --git a/deploy.js b/deploy.js
@@ -2,6 +2,7 @@
  * Deployment entry point for the application.
  **/
 var args = process.argv.slice(2);
+var group = args[0] || process.env.GROUP;
 var cp = require('child_process');
 if (process.env.PROVIDER === "gce")
 {
@@ -12,17 +13,16 @@ if (process.env.ROLE)
     //if role variable is set just run that script
     require('./svc/' + process.env.ROLE + ".js");
 }
-else if (args[0])
+else if (group)
 {
     var pm2 = require('pm2');
     var async = require('async');
     var manifest = require('./profiles/everything.json').apps;
-    //if argument supplied use pm2 to run processes in that group
     pm2.connect(function()
     {
         async.each(manifest, function start(app, cb)
         {
-            if (args[0] === app.group)
+            if (group === app.group)
             {
                 console.log(app.script, app.instances);
                 pm2.start(app.script,

diff --git a/java_parser/src/main/java/yasp/Main.java b/java_parser/src/main/java/yasp/Main.java
@@ -10,6 +10,7 @@
 import skadistats.clarity.model.s1.GameRulesStateType;
 import skadistats.clarity.processor.entities.Entities;
 import skadistats.clarity.processor.entities.OnEntityEntered;
+import skadistats.clarity.processor.entities.OnEntityLeft;
 import skadistats.clarity.processor.entities.UsesEntities;
 import skadistats.clarity.processor.gameevents.CombatLog;
 import skadistats.clarity.processor.gameevents.OnCombatLogEntry;
@@ -70,14 +71,15 @@ private class Entry {
         public Integer gold_reason;
         public Integer xp_reason;
         public String valuename;
-        public Float stun_duration;
-        public Float slow_duration;
+        //public Float stun_duration;
+        //public Float slow_duration;
         //entity fields
         public Integer gold;
         public Integer lh;
         public Integer xp;
         public Integer x;
         public Integer y;
+        public Integer z;
         public Float stuns;
         public Integer hero_id;
         public Integer life_state;
@@ -87,7 +89,9 @@ private class Entry {
         public Integer assists;
         public Integer denies;
         //public Boolean hasPredictedVictory;
-
+        public Boolean entityleft;
+        public Integer ehandle;
+
         public Entry() {
         }
 
@@ -234,8 +238,8 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) {
             combatLogEntry.attackerillusion = cle.isAttackerIllusion();
             combatLogEntry.targetillusion = cle.isTargetIllusion();
             combatLogEntry.value = cle.getValue();
-            combatLogEntry.stun_duration = cle.getStunDuration();
-            combatLogEntry.slow_duration = cle.getSlowDuration();
+            //combatLogEntry.stun_duration = cle.getStunDuration();
+            //combatLogEntry.slow_duration = cle.getSlowDuration();
             //value may be out of bounds in string table, we can only get valuename if a purchase (type 11)
             if (cle.getType() == DOTA_COMBATLOG_TYPES.DOTA_COMBATLOG_PURCHASE) {
                 combatLogEntry.valuename = cle.getValueName();
@@ -255,28 +259,12 @@ public void onCombatLogEntry(Context ctx, CombatLogEntry cle) {
 
     @OnEntityEntered
     public void onEntityEntered(Context ctx, Entity e) {
-        //CDOTA_NPC_Observer_Ward
-        //CDOTA_NPC_Observer_Ward_TrueSight
-        //s1 "DT_DOTA_NPC_Observer_Ward"
-        //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight"
-        boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward");
-        boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight");
-        if (isObserver || isSentry) {
-            //System.err.println(e);
-            Entry entry = new Entry(time);
-            Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null);
-            Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null);
-            Integer[] pos = {x, y};
-            entry.type = isObserver ? "obs" : "sen";
-            entry.key = Arrays.toString(pos);
-            //System.err.println(entry.key);
-            Integer owner = getEntityProperty(e, "m_hOwnerEntity", null);
-            Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner);
-            entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null;
-            //2/3 radiant/dire
-            //entry.team = e.getProperty("m_iTeamNum");
-            output(entry);
-        }
+        processWardEntity(ctx, e, false);
+    }
+
+    @OnEntityLeft
+    public void onEntityLeft(Context ctx, Entity e) {
+        processWardEntity(ctx, e, true);
     }
 
     @UsesEntities
@@ -448,6 +436,45 @@ public <T> T getEntityProperty(Entity e, String property, Integer idx) {
         FieldPath fp = e.getDtClass().getFieldPathForName(property);
         return e.getPropertyForFieldPath(fp);
     }
+
+    public void processWardEntity(Context ctx, Entity e, boolean entityLeft)
+    {
+        //CDOTA_NPC_Observer_Ward
+        //CDOTA_NPC_Observer_Ward_TrueSight
+        //s1 "DT_DOTA_NPC_Observer_Ward"
+        //s1 "DT_DOTA_NPC_Observer_Ward_TrueSight"
+        boolean isObserver = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward");
+        boolean isSentry = e.getDtClass().getDtName().equals("CDOTA_NPC_Observer_Ward_TrueSight");
+        if (isObserver || isSentry) {
+            //System.err.println(e);
+            Entry entry = new Entry(time);
+            Integer x = getEntityProperty(e, "CBodyComponent.m_cellX", null);
+            Integer y = getEntityProperty(e, "CBodyComponent.m_cellY", null);
+            Integer z = getEntityProperty(e, "CBodyComponent.m_cellZ", null);
+            Integer[] pos = {x, y};
+            entry.x = x;
+            entry.y = y;
+            entry.z = z;
+            if (entityLeft)
+            {
+                entry.type = isObserver ? "obs_left" : "sen_left";
+            }
+            else
+            {
+                entry.type = isObserver ? "obs" : "sen";
+            }
+            entry.key = Arrays.toString(pos);
+            entry.entityleft = entityLeft;
+            entry.ehandle = e.getHandle();
+            //System.err.println(entry.key);
+            Integer owner = getEntityProperty(e, "m_hOwnerEntity", null);
+            Entity ownerEntity = ctx.getProcessor(Entities.class).getByHandle(owner);
+            entry.slot = ownerEntity != null ? (Integer) getEntityProperty(ownerEntity, "m_iPlayerID", null) : null;
+            //2/3 radiant/dire
+            //entry.team = e.getProperty("m_iTeamNum");
+            output(entry);
+        }
+    }
 
     public void run(String[] args) throws Exception {
         long tStart = System.currentTimeMillis();

diff --git a/json/navbar_pages.json b/json/navbar_pages.json
@@ -9,10 +9,6 @@
   "benchmarks": {
     "name": "Benchmarks"
   },
-  "top": {
-    "name": "Top",
-    "hide": true
-  },
   "distributions": {
     "name": "Distributions"
   },

diff --git a/json/patch.json b/json/patch.json
@@ -70,5 +70,9 @@
 {
   "name": "6.87",
   "date": "2016-04-26T01:00:00Z"
+},
+{
+  "name": "6.88",
+  "date": "2016-06-12T08:00:00Z"
 }
 ]
diff --git a/package.json b/package.json
@@ -17,7 +17,8 @@
     "kube": "kubectl get rc -o name --selector tier=backend | cut -d '/' -f2 | xargs -n1 kubectl rolling-update --image=yasp/yasp:latest",
     "rediskeys": "redis-cli keys '*' | cut -d':' -f1 | sort | uniq -c",
     "resetpicks": "redis-cli keys 'picks_*' | xargs redis-cli del",
-    "updateconstants": "node tasks/updateconstants"
+    "updateconstants": "node tasks/updateconstants",
+    "dockerbuild": "sudo docker build -t yasp/yasp ."
   },
   "repository": {
     "type": "git",
@@ -52,13 +53,11 @@
     "font-awesome": "^4.5.0",
     "heatmap.js": "2.0.0",
     "http-proxy": "^1.13.1",
-    "isomorphic-fetch": "^2.2.1",
     "jade": "^1.11.0",
     "jquery": "^2.2.0",
-    "knex": "^0.10.0",
+    "knex": "^0.11.7",
     "moment": "^2.11.2",
     "multer": "^1.1.0",
-    "ndjson": "^1.4.3",
     "numeral": "^1.5.3",
     "passport": "0.3.2",
     "passport-steam": "1.0.1",
@@ -92,9 +91,5 @@
     "supertest": "^1.1.0",
     "url-loader": "^0.5.7",
     "webpack": "^1.12.12"
-  },
-  "engines": {
-    "node": "6.0.0",
-    "npm": "3.8.8"
   }
 }
diff --git a/processors/populate.js b/processors/populate.js
@@ -89,5 +89,4 @@ function populate(e, container)
             break;
     }
 }
-
 module.exports = populate;
-Original file line number
+Diff line change
@@ Expand Up / @@ -89,5 +89,4 @@ function populate(e, container) @@
                 break;
         }
     }
     module.exports = populate;