Merge pull request #115 from neilcook/gendata

Utility scripts to generate data for wforce

gendata/README.md

@@ -0,0 +1,11 @@
+The files in this directory are used to generate data to feed into wforce,
+primarily to create data to populate named report sinks (typically
+logstash/elasticsearch) with report data.
+
+The following files are used:
+ * gen_success_reports.lua - This is a file to be used by wrk2 in order to construct "random" data about successful login attempts to send to wforce as reports
+ * gen_fail_reports.lua - This is a file to be used by wrk2 in order to construct "random" data about failed login attempts to send to wforce as reports
+ * wforce_elastic.conf - This is a very simple (no policy) wforce config file that just sends reports to a logstash instance on localhost:14501. This is not the standard logstash port because I run logstash and elasticsearch in a Docker container.
+ * send_reports.py - This is a python script that start wforce, and send reports to it using wrk2, which must be installed and in the path.
+
+Wrk2 can be found at https://github.com/giltene/wrk2. It is also a very good tool for loadtesting wforce, and the gen_xxxx_reports.lua scripts are a good place to start if you wish to do so.

gendata/gen_fail_reports.lua

@@ -0,0 +1,54 @@
+# This lua script generates random reports for wforce logins
+
+counter = 0
+one_counter = 0
+wrk.headers["Content-Type"] = "application/json"
+wrk.headers["Authorization"] = "Basic d2ZvcmNlOnN1cGVy"
+wrk.method = 'POST'
+wrk.path = "/?command=report"
+
+dp_access = {
+   {protocol="http", device="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A"},
+   {protocol="https", device="Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25"},
+   {protocol="https", device="Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"},
+   {protocol="http", device="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36"},
+   {protocol="https", device="Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko"},
+   {protocol="http", device="Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)"},
+   {protocol="https", device="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1"},
+   {protocol="imap", device='\\"name\\" \\"Bad Guy Mailer\\" \\"version\\" \\"10.0 (3226)\\" \\"os\\" \\"Mac OS X\\" \\"os-version\\" \\"10.12 (16A323)\\" \\"vendor\\" \\"Bad Guy Inc.\\"'}
+}
+
+ip_access = {
+   "2.2.2.2",
+   "3.3.3.3",
+   "4.4.4.4",
+   "5.5.5.5",
+   "6.6.6.6",
+   "7.7.7.7"
+}
+
+pwd_access = {
+   "ijfds",
+   "sokee",
+   "sdefw",
+   "93ked",
+   "0kwepw",
+   "ow3002",
+   "2303wp",
+   "2020ew",
+   "309oka"
+}
+
+request = function()
+   ip_mod = one_counter % table.getn(ip_access) + 1
+   dp_mod = one_counter % table.getn(dp_access) + 1
+   pwd_mod = counter % table.getn(pwd_access) + 1
+   mybody = '{"login":"user' .. counter .. '", "remote":"' .. ip_access[ip_mod] .. '", "pwhash":"' .. pwd_access[pwd_mod] .. '", "success": false, "policy_reject":false, "protocol":"' .. dp_access[dp_mod].protocol .. '", "device_id":"' .. dp_access[dp_mod].device .. '" }'
+   counter = counter + 10
+   one_counter = one_counter + 1
+   if (counter>1000)
+        then
+                counter = 0
+        end
+   return wrk.format(nil, nil, nil, mybody)
+end

gendata/gen_success_reports.lua

@@ -0,0 +1,69 @@
+# This lua script generates random reports for wforce logins
+
+counter = 0
+wrk.headers["Content-Type"] = "application/json"
+wrk.headers["Authorization"] = "Basic d2ZvcmNlOnN1cGVy"
+wrk.method = 'POST'
+wrk.path = "/?command=report"
+
+dp_access = {
+   {protocol="http", device="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A"},
+   {protocol="https", device="Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25"},
+   {protocol="https", device="Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"},
+   {protocol="http", device="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36"},
+   {protocol="https", device="Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko"},
+   {protocol="http", device="Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)"},
+   {protocol="https", device="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1"},
+   {protocol="imap", device='\\"name\\" \\"iPhone Mail\\" \\"version\\" \\"14D27\\" \\"os\\" \\"iOS\\" \\"os-version\\" \\"10.2.1 (14D27)\\"'},
+   {protocol="imap", device='\\"name\\" \\"iPad Mail\\" \\"version\\" \\"14D27\\" \\"os\\" \\"iOS\\" \\"os-version\\" \\"10.2.1 (14D27)\\"'},
+   {protocol="imap", device='\\"name\\" \\"Mac OS X Mail\\" \\"version\\" \\"10.2 (3259)\\" \\"os\\" \\"Mac OS X\\" \\"os-version\\" \\"10.12.3 (16D32)\\" \\"vendor\\" \\"Apple Inc.\\"'},
+   {protocol="imap", device='\\"name\\" \\"Mac OS X Notes\\" \\"version\\" \\"4.3.1 (698.50)\\" \\"os\\" \\"Mac OS X\\" \\"os-version\\" \\"10.12.3 (16D32)\\" \\"vendor\\" \\"Apple Inc.\\"'},
+   {protocol="imap", device='\\"name\\" \\"Mac OS X accountsd\\" \\"version\\" \\"113 (113)\\" \\"os\\" \\"Mac OS X\\" \\"os-version\\" \\"10.12.3 (16D32)\\" \\"vendor\\" \\"Apple Inc.\\"'},
+   {protocol="imap", device='\\"name\\" \\"Microsoft Outlook\\" \\"version\\" \\"14.0.7172.5000\\"'},
+   {protocol="imap", device='\\"name\\" \\"Thunderbird\\" \\"version\\" \\"45.7.1\\"'},
+   {protocol="imap", device='\\"name\\" \\"Icedove\\" \\"version\\" \\"38.5.0\\"'},
+   {protocol="imap", device='vendor\\" \\"Microsoft\\" \\"os\\" \\"Windows Mobile\\" \\"os-version\\" \\"10.0\\" \\"guid\\" \\"31414643323441464239373433384242374438463641383146423942364639383931324435323232\\"'},
+   {protocol="imap", device='\\"name\\" \\"com.google.android.gm\\" \\"os\\" \\"android\\" \\"os-version\\" \\"7.1.1; N4F26O\\" \\"vendor\\" \\"LGE\\" \\"x-android-device-model\\" \\"Nexus 5X\\" \\"x-android-mobile-net-operator\\" \\"Sonera AGUID\\" \\"XwzK66ekvB2DkNmD8EyAuwr8M4s\\"'},
+   {protocol="imap", device='\\"name\\" \\"com.android.email\\" \\"os\\" \\"android\\" \\"os-version\\" \\"6.0; 32.1.F.1.67\\" \\"vendor\\" \\"Sony\\" \\"x-android-device-model\\" \\"SO-01H\\" \\"x-android-mobile-net-operator\\" \\"NTT DOCOMO\\" \\"AGUID\\" \\"QHaFdAshRmaLYCPM0KVo7lwajhE\\"'},
+   {protocol="imap", device='\\"name\\" \\"com.samsung.android.email.provider\\" \\"os\\" \\"android\\" \\"os-version\\" \\"6.0.1; MMB29K\\" \\"vendor\\" \\"samsung\\" \\"x-android-device-model\\" \\"SM-G925F\\" \\"x-android-mobile-net-operator\\" \\"Sonera\\" \\"AGUID\\" \\"X9I/elUUlMNosGZjexmq6mp1zuA\\"'},
+   {protocol="imap", device='\\"name\\" \\"com.sonymobile.email\\" \\"os\\" \\"android\\" \\"os-version\\" \\"6.0.1; 32.2.A.5.11\\" \\"vendor\\" \\"Sony\\" \\"x-android-device-model\\" \\"SGP771\\" \\"AGUID\\" \\"cJwASIogV2kpgCa3kPG+77TgCZk\\"'},
+   {protocol="imap", device='\\"name\\" \\"eM Client for OX App Suite\\" \\"version\\" \\"6.0.28376.0\\" \\"GUID\\" \\"1\\"'}
+}
+
+ip_access = {
+   "23.96.52.53",
+   "86.128.243.146",
+   "213.20.82.16",
+   "212.223.44.11",
+   "8.8.8.8",
+   "2a03:b0c0:2:d0::4ab:8001",
+   "2a00:1450:4009:80a::200e",
+   "2a00:1450:4009:806::2005",
+   "180.22.47.32",
+   "52.48.64.3"
+}
+
+pwd_access = {
+   "ijfds",
+   "sokee",
+   "sdefw",
+   "93ked",
+   "0kwepw",
+   "ow3002",
+   "2303wp",
+   "2020ew",
+   "309oka"
+}
+
+request = function()
+   ip_mod = counter % table.getn(ip_access) + 1
+   dp_mod = counter % table.getn(dp_access) + 1
+   pwd_mod = counter % table.getn(pwd_access) + 1
+   mybody = '{"login":"user' .. counter .. '", "remote":"' .. ip_access[ip_mod] .. '", "pwhash":"' .. pwd_access[pwd_mod] .. '", "success": true, "policy_reject":false, "protocol":"' .. dp_access[dp_mod].protocol .. '", "device_id":"' .. dp_access[dp_mod].device .. '" }'
+   counter = counter + 1
+   if (counter==1000)
+        then
+                counter = 0
+        end
+   return wrk.format(nil, nil, nil, mybody)
+end

gendata/send_reports.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+#
+# Shell-script style.
+
+import os
+import requests
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+WEBPORT = '8084'
+APIKEY = 'super'
+
+cmd1 = ("../wforce -C ./wforce_elastic.conf -R ../regexes.yaml").split()
+wrksuccesscmd = ("wrk -c 10 -d 60 -t 1 -s ./gen_success_reports.lua -R 50 http://127.0.0.1:8084").split()
+wrkfailcmd = ("wrk -c 10 -d 30 -t 1 -s ./gen_fail_reports.lua -R 50 http://127.0.0.1:8084").split()
+
+# Now run wforce and the tests.
+print "Launching wforce..."
+print ' '.join(cmd1)
+proc1 = subprocess.Popen(cmd1, close_fds=True)
+wforcepid = proc1.pid
+
+print "Waiting for webserver port to become available..."
+available = False
+for try_number in range(0, 10):
+    try:
+        res = requests.get('http://127.0.0.1:%s/' % WEBPORT)
+        available = True
+        break
+    except:
+        time.sleep(0.5)
+
+if not available:
+    print "Webserver port not reachable after 10 tries, giving up."
+    proc1.terminate()
+    proc1.wait()
+    sys.exit(2)
+
+print "Sending Reports..."
+wrkproc = subprocess.Popen(wrksuccesscmd, close_fds=True)
+wrkproc.wait()
+wrkproc = subprocess.Popen(wrkfailcmd, close_fds=True)
+wrkproc.wait()
+print "Done sending reports..."
+proc1.terminate()
+proc1.wait()
+print "Exiting"
+sys.exit(0)

gendata/wforce_elastic.conf

@@ -0,0 +1,39 @@
+webserver("0.0.0.0:8084", "super")
+setKey("Ay9KXgU3g4ygK+qWT0Ut4gH8PPz02gbtPeXWPdjD0HE=")
+controlSocket("0.0.0.0:4004")
+
+addACL("127.0.0.0/8")
+addACL("192.168.0.0/16")
+
+addNamedReportSink("logstash", "127.0.0.1:14501")
+
+function report(lt)
+end
+
+function allow(lt)
+	-- return must have these 4 arguments
+	return 0, "allowed", "allowed", {}
+end
+
+-- Use this function to reset stats if needed for particular IPs, logins or both
+function reset(type, login, ip)
+end
+
+setReport(report)
+setAllow(allow)
+setReset(reset)
+
+function custom(args)
+   for k,v in pairs(args.attrs) do
+      infoLog("custom func argument attrs", { key=k, value=v });
+   end
+
+   runCustomWebHook("mycustomhook", "{ \"foo\":\"bar\" }")
+
+   -- return consists of a boolean, followed by { key-value pairs }
+   return true, { key=value }
+end
+
+-- Register a custom endpoint
+-- Parameters: name, send arguments to report sink?, function)
+setCustomEndpoint("custom", false, custom)