diff --git a/example-config.yaml b/example-config.yaml index 6e9ee57..f6f0534 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -149,16 +149,23 @@ action_url: # # Syntax: # proxy_pass: -# - /url_to_be_passed example.com +# - /url_to_be_passed example.com [option1,option2=value2] # # The first parameter 'url' is a regex (case-insensitive). Must start with '/'. -# The begin/end regex operands are implicit and will constitute following regex with URL: -# '^' + url + '$' +# The regex begin/end operators are implied and will constitute following regex to be +# matched against inbound request's URL: +# '^/' + url_to_be_passed + '$' +# +# Following options are supported: +# - nodrop - Process this rule at first, before evaluating any DROP-logic. +# Ensures requests with matching URL to be proxy-passed no matter what. # # Default: No proxy pass rules. # proxy_pass: - /foobar\d* bing.com + - /alwayspass google.com nodrop + # # If set, removes all HTTP headers sent by Client that are not expected by Teamserver according diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ipLookupHelper.py b/lib/ipLookupHelper.py similarity index 79% rename from ipLookupHelper.py rename to lib/ipLookupHelper.py index 4c47def..9823e74 100644 --- a/ipLookupHelper.py +++ b/lib/ipLookupHelper.py @@ -35,17 +35,19 @@ import threading import requests import urllib3 + from urllib.parse import urlparse, parse_qsl from subprocess import Popen, PIPE -from proxylogger import ProxyLogger -from pluginsloader import PluginsLoader -from sslintercept import SSLInterception from http.server import BaseHTTPRequestHandler, HTTPServer from socketserver import ThreadingMixIn -import plugins.IProxyPlugin from io import StringIO, BytesIO from html.parser import HTMLParser +import lib.plugins.IProxyPlugin +from lib.proxylogger import ProxyLogger +from lib.pluginsloader import PluginsLoader +from lib.sslintercept import SSLInterception + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) ssl._create_default_https_context = ssl._create_unverified_context @@ -94,27 +96,22 @@ class IPLookupHelper: cached_lookups_file = 'ip-lookups-cache.json' - def __init__(self, apiKeys): + def __init__(self, logger, apiKeys): + self.logger = logger self.apiKeys = { 'ip_api_com': 'this-provider-not-requires-api-key-for-free-plan', 'ipapi_co': 'this-provider-not-requires-api-key-for-free-plan', } - self.httpHeaders = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit (KHTML, like Gecko) Chrome/87', - 'Accept': 'text/json, */*', - 'Host': '', - } - if len(apiKeys) > 0: for prov in IPLookupHelper.supported_providers: if prov in apiKeys.keys(): - if len(apiKeys[prov].strip()) < 2: continue + if apiKeys[prov] == None or len(apiKeys[prov].strip()) < 2: continue self.apiKeys[prov] = apiKeys[prov].strip() self.cachedLookups = {} - Logger.dbg('Following IP Lookup providers will be used: ' + str(list(self.apiKeys.keys()))) + self.logger.dbg('Following IP Lookup providers will be used: ' + str(list(self.apiKeys.keys()))) try: with open(IPLookupHelper.cached_lookups_file) as f: @@ -122,10 +119,10 @@ def __init__(self, apiKeys): if len(data) > 0: cached = json.loads(data) self.cachedLookups = cached - Logger.dbg(f'Read {len(cached)} cached entries from file.') + self.logger.dbg(f'Read {len(cached)} cached entries from file.') except json.decoder.JSONDecodeError as e: - Logger.err(f'Corrupted JSON data in cache file: {IPLookupHelper.cached_lookups_file}! Error: {e}') + self.logger.err(f'Corrupted JSON data in cache file: {IPLookupHelper.cached_lookups_file}! Error: {e}') raise except FileNotFoundError as e: @@ -133,15 +130,15 @@ def __init__(self, apiKeys): json.dump({}, f) except Exception as e: - Logger.err(f'Exception raised while loading cached lookups from file ({IPLookupHelper.cached_lookups_file}: {e}') + self.logger.err(f'Exception raised while loading cached lookups from file ({IPLookupHelper.cached_lookups_file}: {e}') raise def lookup(self, ipAddress): if len(self.apiKeys) == 0: - return + return {} if ipAddress in self.cachedLookups.keys(): - Logger.dbg(f'Returning cached entry for IP address: {ipAddress}') + self.logger.dbg(f'Returning cached entry for IP address: {ipAddress}') return self.cachedLookups[ipAddress] leftProvs = list(self.apiKeys.keys()) @@ -152,7 +149,7 @@ def lookup(self, ipAddress): if hasattr(self, prov) != None: method = getattr(self, prov) - Logger.dbg(f'Calling IP Lookup provider: {prov}') + self.logger.dbg(f'Calling IP Lookup provider: {prov}') result = method(ipAddress) if len(result) > 0: @@ -167,7 +164,7 @@ def lookup(self, ipAddress): with open(IPLookupHelper.cached_lookups_file, 'w') as f: json.dump(self.cachedLookups, f) - Logger.dbg(f'New IP lookup entry cached: {ipAddress}') + self.logger.dbg(f'New IP lookup entry cached: {ipAddress}') return result @@ -282,7 +279,7 @@ def update(out, data, keydst, keysrc): return output def ip_api_com(self, ipAddress): - # $ curl -s ip-api.com/json/89.167.131.40 + # $ curl -s ip-api.com/json/89.167.131.40 [21:05] # { # "status": "success", # "country": "Germany", @@ -301,9 +298,7 @@ def ip_api_com(self, ipAddress): # } try: - self.httpHeaders['Host'] = 'ip-api.com' - r = requests.get(f'http://ip-api.com/json/{ipAddress}', - headers = self.httpHeaders) + r = requests.get(f'http://ip-api.com/json/{ipAddress}') if r.status_code != 200: raise Exception(f'ip-api.com returned unexpected status code: {r.status_code}.\nOutput text:\n' + r.json()) @@ -311,7 +306,7 @@ def ip_api_com(self, ipAddress): return r.json() except Exception as e: - Logger.err(f'Exception catched while querying ip-api.com with {ipAddress}:\nName: {e}') + self.logger.err(f'Exception catched while querying ip-api.com with {ipAddress}:\nName: {e}', color='cyan') return {} @@ -346,9 +341,7 @@ def ipapi_co(self, ipAddress): # } try: - self.httpHeaders['Host'] = 'ipapi.co' - r = requests.get(f'https://ipapi.co/{ipAddress}/json/', - headers = self.httpHeaders) + r = requests.get(f'https://ipapi.co/{ipAddress}/json/') if r.status_code != 200: raise Exception(f'ipapi.co returned unexpected status code: {r.status_code}.\nOutput text:\n' + r.json()) @@ -356,7 +349,7 @@ def ipapi_co(self, ipAddress): return r.json() except Exception as e: - Logger.err(f'Exception catched while querying ipapi.co with {ipAddress}:\nName: {e}') + self.logger.err(f'Exception catched while querying ipapi.co with {ipAddress}:\nName: {e}', color='cyan') return {} @@ -400,9 +393,7 @@ def ipgeolocation_io(self, ipAddress): # } # } try: - self.httpHeaders['Host'] = 'api.ipgeolocation.io' - r = requests.get(f'https://api.ipgeolocation.io/ipgeo?apiKey={self.apiKeys["ipgeolocation_io"]}&ip={ipAddress}', - headers = self.httpHeaders) + r = requests.get(f'https://api.ipgeolocation.io/ipgeo?apiKey={self.apiKeys["ipgeolocation_io"]}&ip={ipAddress}') if r.status_code != 200: raise Exception(f'ipapi.co returned unexpected status code: {r.status_code}.\nOutput text:\n' + r.json()) @@ -410,7 +401,7 @@ def ipgeolocation_io(self, ipAddress): return r.json() except Exception as e: - Logger.err(f'Exception catched while querying ipapi.co with {ipAddress}:\nName: {e}') + self.logger.err(f'Exception catched while querying ipapi.co with {ipAddress}:\nName: {e}', color='cyan') return {} @@ -425,7 +416,8 @@ class IPGeolocationDeterminant: 'timezone' ) - def __init__(self, determinants): + def __init__(self, logger, determinants): + self.logger = logger if type(determinants) != dict: raise Exception('Specified ip_geolocation_requirements must be a valid dictonary!') @@ -466,13 +458,13 @@ def determine(self, ipLookupResult): for exp in expected: if georesult in exp.lower(): - Logger.dbg(f'IP Geo result {determinant} value "{georesult}" met expected value "{exp}"') + self.logger.dbg(f'IP Geo result {determinant} value "{georesult}" met expected value "{exp}"') matched = True break m = re.search(exp, georesult, re.I) if m: - Logger.dbg(f'IP Geo result {determinant} value "{georesult}" met expected regular expression: ({exp})') + self.logger.dbg(f'IP Geo result {determinant} value "{georesult}" met expected regular expression: ({exp})') matched = True break @@ -480,11 +472,62 @@ def determine(self, ipLookupResult): break if not matched: - Logger.dbg(f'IP Geo result {determinant} values {ipLookupResult[determinant]} DID NOT met expected set {expected}') + self.logger.dbg(f'IP Geo result {determinant} values {ipLookupResult[determinant]} DID NOT met expected set {expected}') result = False return result + @staticmethod + def getValues(v, n = 0): + values = [] + + if type(v) == str: + if ' ' in v: + values.extend(v.split(' ')) + values.append(v) + elif type(v) == int or type(v) == float: + values.extend([str(v)]) + elif type(v) == tuple or type(v) == list: + for w in v: + values.extend(IPGeolocationDeterminant.getValues(w, n+1)) + elif type(v) == dict and n < 10: + values.extend(IPGeolocationDeterminant.getValuesDict(v, n+1)) + + return values + + @staticmethod + def getValuesDict(data, n = 0): + values = [] + + for k, v in data.items(): + if type(v) == dict and n < 10: + values.extend(IPGeolocationDeterminant.getValuesDict(v, n+1)) + elif n < 10: + values.extend(IPGeolocationDeterminant.getValues(v, n+1)) + + return values + + def validateIpGeoMetadata(self, ipLookupDetails): + if len(ipLookupDetails) == 0: return (True, '') + + words = set(list(filter(None, IPGeolocationDeterminant.getValuesDict(ipLookupDetails)))) + if len(words) == 0: return (True, '') + + self.logger.dbg(f"Extracted keywords from Peer's IP Geolocation metadata: ({words})") + + for w in words: + for x in BANNED_AGENTS: + if ((' ' in x) and (x.lower() in w.lower())): + self.logger.dbg(f"Peer's IP Geolocation metadata contained banned phrase: ({w})") + return (False, w) + + elif (w.lower() == x.lower()): + self.logger.dbg(f"Peer's IP Geolocation metadata contained banned keyword: ({w})") + return (False, w) + + self.logger.dbg(f"Peer's IP Geolocation metadata didn't raise any suspicion.") + return (True, '') + def main(argv): if len(argv) < 2: print (''' diff --git a/optionsparser.py b/lib/optionsparser.py old mode 100755 new mode 100644 similarity index 95% rename from optionsparser.py rename to lib/optionsparser.py index ad5a1f6..f9cdf69 --- a/optionsparser.py +++ b/lib/optionsparser.py @@ -1,236 +1,236 @@ -#!/usr/bin/python - -import yaml -import os, sys -from pluginsloader import PluginsLoader -from proxylogger import ProxyLogger -from argparse import ArgumentParser - -ProxyOptionsDefaultValues = { -} - - -def parse_options(opts, version): - global ProxyOptionsDefaultValues - ProxyOptionsDefaultValues.update(opts) - - usage = "Usage: %%prog [options]" - parser = ArgumentParser(usage=usage, prog="%prog " + version) - - parser.add_argument("-c", "--config", dest='config', - help="External configuration file. Defines values for below options, however specifying them on command line will supersed ones from file.") - - # General options - parser.add_argument("-v", "--verbose", dest='verbose', - help="Displays verbose output.", action="store_true") - parser.add_argument("-V", "--trace", dest='trace', - help="Displays HTTP requests and responses.", action="store_true") - parser.add_argument("-d", "--debug", dest='debug', - help="Displays debugging informations (implies verbose output).", action="store_true") - parser.add_argument("-s", "--silent", dest='silent', - help="Surpresses all of the output logging.", action="store_true") - parser.add_argument("-z", "--allow-invalid", dest='allow_invalid', - help="Process invalid HTTP requests. By default if a stream not resembling HTTP protocol reaches proxy2 listener - it will be dropped.", action="store_true") - parser.add_argument("-N", "--no-proxy", dest='no_proxy', - help="Disable standard HTTP/HTTPS proxy capability (will not serve CONNECT requests). Useful when we only need plugin to run.", action="store_true") - parser.add_argument("-W", "--tee", dest='tee', - help="While logging to output file, print to stdout also.", action="store_true") - parser.add_argument("-w", "--output", dest='log', - help="Specifies output log file.", metavar="PATH", type=str) - parser.add_argument("-B", "--bind", dest='bind', metavar='NAME', - help="Specifies proxy's binding address along with protocol to serve (http/https). If scheme is specified here, don't add another scheme specification to the listening port number (123/https). Default: "+ opts['bind'] +".", - type=str, default=opts['bind']) - parser.add_argument("-P", "--port", dest='port', metavar='NUM', - help="Specifies proxy's binding port number(s). A value can be followed with either '/http' or '/https' to specify which type of server to bound on this port. Supports multiple binding ports by repeating this option: '--port 80 --port 443/https'. The port specification may also override globally used --bind address by preceding it with address and colon (--port 127.0.0.1:80/http). Default: "+ str(opts['port'][0]) +".", - type=str, action="append", default = []) - parser.add_argument("-t", "--timeout", dest='timeout', metavar='SECS', - help="Specifies timeout for proxy's response in seconds. Default: "+ str(opts['timeout']) +".", - type=int, default=opts['timeout']) - parser.add_argument("-u", "--proxy-url", dest='proxy_self_url', metavar='URL', - help="Specifies proxy's self url. Default: "+ opts['proxy_self_url'] +".", - type=str, default=opts['proxy_self_url']) - - - # SSL Interception - sslgroup = parser.add_argument_group("SSL Interception setup") - sslgroup.add_argument("-S", "--no-ssl-mitm", dest='no_ssl', - help="Turns off SSL interception/MITM and falls back on straight forwarding.", action="store_true") - sslgroup.add_argument('--ssl-certdir', dest='certdir', metavar='DIR', - help='Sets the destination for all of the SSL-related files, including keys, certificates (self and of'\ - ' the visited websites). If not specified, a default value will be used to create a directory and remove it upon script termination. Default: "'+ opts['certdir'] +'"', default=opts['certdir']) - sslgroup.add_argument('--ssl-cakey', dest='cakey', metavar='NAME', - help='Specifies this proxy server\'s (CA) certificate private key. Default: "'+ opts['cakey'] +'"', default=opts['cakey']) - sslgroup.add_argument('--ssl-cacert', dest='cacert', metavar='NAME', - help='Specifies this proxy server\'s (CA) certificate. Default: "'+ opts['cacert'] +'"', default=opts['cacert']) - sslgroup.add_argument('--ssl-certkey', dest='certkey', metavar='NAME', - help='Specifies CA certificate\'s public key. Default: "'+ opts['certkey'] +'"', default=opts['certkey']) - sslgroup.add_argument('--ssl-cacn', dest='cacn', metavar='CN', - help='Sets the common name of the proxy\'s CA authority. If this option is not set, will use --hostname instead. It is required only when no --ssl-cakey/cert were specified and proxy2 will need to generate ones automatically. Default: "'+ opts['cacn'] +'"', default=opts['cacn']) - - # Plugins handling - plugins = parser.add_argument_group("Plugins handling") - plugins.add_argument('-L', '--list-plugins', action='store_true', help='List available plugins.') - plugins.add_argument('-p', '--plugin', dest='plugin', action='append', metavar='PATH', type=str, - help="Specifies plugin's path to be loaded.") - - feed_with_plugin_options(opts, parser) - - params = parser.parse_args() - - if hasattr(params, 'config') and params.config != '': - try: - params = parseParametersFromConfigFile(params) - except Exception as e: - parser.error(str(e)) - - opts.update(params) - else: - opts.update(vars(params)) - - if opts['list_plugins']: - files = sorted([f for f in os.scandir(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'plugins/'))], key = lambda f: f.name) - for _, entry in enumerate(files): - if entry.name.endswith(".py") and entry.is_file() and entry.name.lower() not in ['iproxyplugin.py', '__init__.py']: - print('[+] Plugin: {}'.format(entry.name)) - - sys.exit(0) - - if opts['plugin'] != None and len(opts['plugin']) > 0: - for i, opt in enumerate(opts['plugin']): - decomposed = PluginsLoader.decompose_path(opt) - if not os.path.isfile(decomposed['path']): - opt = opt.replace('.py', '') - opt2 = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'plugins/{}.py'.format(opt))) - if not os.path.isfile(opt2): - raise Exception('Specified plugin: "%s" does not exist.' % decomposed['path']) - else: - opt = opt2 - - opts['plugins'].add(opt) - - #if opts['debug']: - # opts['trace'] = True - - if opts['silent'] and opts['log']: - parser.error("Options -s and -w are mutually exclusive.") - - if opts['silent']: - opts['log'] = 'none' - elif opts['log'] and len(opts['log']) > 0: - try: - if not os.path.isfile(opts['log']): - with open(opts['log'], 'w') as f: - pass - opts['log'] = opts['log'] - - except Exception as e: - raise Exception('[ERROR] Failed to open log file for writing. Error: "%s"' % e) - else: - opts['log'] = sys.stdout - - if opts['log'] and opts['log'] != sys.stdout: opts['log'] = os.path.normpath(opts['log']) - if opts['cakey']: opts['cakey'] = os.path.normpath(opts['cakey']) - if opts['certdir']: opts['certdir'] = os.path.normpath(opts['certdir']) - if opts['certkey']: opts['certkey'] = os.path.normpath(opts['certkey']) - -def parseParametersFromConfigFile(_params): - parametersRequiringDirectPath = ( - 'log', - 'output', - 'certdir', - 'certkey', - 'cakey', - 'cacert', - 'ssl_certdir', - 'ssl_certkey', - 'ssl_cakey', - 'ssl_cacert', - ) - - translateParamNames = { - 'output' : 'log', - 'proxy_url' : 'proxy_self_url', - 'no_ssl_mitm' : 'no_ssl', - 'ssl_certdir' : 'certdir', - 'ssl_certkey' : 'certkey', - 'ssl_cakey' : 'cakey', - 'ssl_cacert' : 'cacert', - 'ssl_cacn' : 'cacn', - 'drop_invalid_http_requests': 'allow_invalid', - } - - valuesThatNeedsToBeList = ( - 'port', - 'plugin', - ) - - outparams = vars(_params) - config = {} - configBasePath = '' - - if not 'config' in outparams.keys() or not os.path.isfile(outparams['config']): - raise Exception(f'proxy2 config file not found: ({outparams["config"]}) or --config not specified!') - - try: - with open(outparams['config']) as f: - #config = yaml.load(f, Loader=yaml.FullLoader) - config = yaml.load(f) - - outparams.update(config) - - for val in valuesThatNeedsToBeList: - if val in outparams.keys() and val in config.keys(): - if type(config[val]) == str: - outparams[val] = [config[val], ] - else: - outparams[val] = config[val] - - for k, v in ProxyOptionsDefaultValues.items(): - if k not in outparams.keys(): - outparams[k] = v - - for k, v in translateParamNames.items(): - if k in outparams.keys(): - outparams[v] = outparams[k] - if v in outparams.keys(): - outparams[k] = outparams[v] - - configBasePath = os.path.dirname(os.path.abspath(outparams['config'])) - - for paramName in parametersRequiringDirectPath: - if paramName in outparams.keys() and \ - outparams[paramName] != '' and outparams[paramName] != None: - outparams[paramName] = os.path.join(configBasePath, outparams[paramName]) - - return outparams - - except FileNotFoundError as e: - raise Exception(f'proxy2 config file not found: ({outparams["config"]})!') - - except Exception as e: - raise Exception(f'Unhandled exception occured while parsing proxy2 config file: {e}') - - return outparams - - -def feed_with_plugin_options(opts, parser): - logger = ProxyLogger() - plugins = [] - files = sorted([f for f in os.scandir(os.path.join(os.path.dirname(os.path.realpath(__file__)), 'plugins/'))], key = lambda f: f.name) - for _, entry in enumerate(files): - if entry.name.endswith(".py") and entry.is_file() and entry.name.lower() not in ['iproxyplugin.py', '__init__.py']: - plugins.append(entry.path) - - options = opts.copy() - options['plugins'] = plugins - options['verbose'] = True - options['debug'] = False - - plugin_own_options = {} - - pl = PluginsLoader(logger, options) - for name, plugin in pl.get_plugins().items(): - logger.dbg("Fetching plugin {} options.".format(name)) - if hasattr(plugin, 'help'): - plugin_options = parser.add_argument_group("Plugin '{}' options".format(plugin.get_name())) - plugin.help(plugin_options) +#!/usr/bin/python + +import yaml +import os, sys +from lib.pluginsloader import PluginsLoader +from lib.proxylogger import ProxyLogger +from argparse import ArgumentParser + +ProxyOptionsDefaultValues = { +} + + +def parse_options(opts, version): + global ProxyOptionsDefaultValues + ProxyOptionsDefaultValues.update(opts) + + usage = "Usage: %%prog [options]" + parser = ArgumentParser(usage=usage, prog="%prog " + version) + + parser.add_argument("-c", "--config", dest='config', + help="External configuration file. Defines values for below options, however specifying them on command line will supersed ones from file.") + + # General options + parser.add_argument("-v", "--verbose", dest='verbose', + help="Displays verbose output.", action="store_true") + parser.add_argument("-V", "--trace", dest='trace', + help="Displays HTTP requests and responses.", action="store_true") + parser.add_argument("-d", "--debug", dest='debug', + help="Displays debugging informations (implies verbose output).", action="store_true") + parser.add_argument("-s", "--silent", dest='silent', + help="Surpresses all of the output logging.", action="store_true") + parser.add_argument("-z", "--allow-invalid", dest='allow_invalid', + help="Process invalid HTTP requests. By default if a stream not resembling HTTP protocol reaches proxy2 listener - it will be dropped.", action="store_true") + parser.add_argument("-N", "--no-proxy", dest='no_proxy', + help="Disable standard HTTP/HTTPS proxy capability (will not serve CONNECT requests). Useful when we only need plugin to run.", action="store_true") + parser.add_argument("-W", "--tee", dest='tee', + help="While logging to output file, print to stdout also.", action="store_true") + parser.add_argument("-w", "--output", dest='log', + help="Specifies output log file.", metavar="PATH", type=str) + parser.add_argument("-B", "--bind", dest='bind', metavar='NAME', + help="Specifies proxy's binding address along with protocol to serve (http/https). If scheme is specified here, don't add another scheme specification to the listening port number (123/https). Default: "+ opts['bind'] +".", + type=str, default=opts['bind']) + parser.add_argument("-P", "--port", dest='port', metavar='NUM', + help="Specifies proxy's binding port number(s). A value can be followed with either '/http' or '/https' to specify which type of server to bound on this port. Supports multiple binding ports by repeating this option: '--port 80 --port 443/https'. The port specification may also override globally used --bind address by preceding it with address and colon (--port 127.0.0.1:80/http). Default: "+ str(opts['port'][0]) +".", + type=str, action="append", default = []) + parser.add_argument("-t", "--timeout", dest='timeout', metavar='SECS', + help="Specifies timeout for proxy's response in seconds. Default: "+ str(opts['timeout']) +".", + type=int, default=opts['timeout']) + parser.add_argument("-u", "--proxy-url", dest='proxy_self_url', metavar='URL', + help="Specifies proxy's self url. Default: "+ opts['proxy_self_url'] +".", + type=str, default=opts['proxy_self_url']) + + + # SSL Interception + sslgroup = parser.add_argument_group("SSL Interception setup") + sslgroup.add_argument("-S", "--no-ssl-mitm", dest='no_ssl', + help="Turns off SSL interception/MITM and falls back on straight forwarding.", action="store_true") + sslgroup.add_argument('--ssl-certdir', dest='certdir', metavar='DIR', + help='Sets the destination for all of the SSL-related files, including keys, certificates (self and of'\ + ' the visited websites). If not specified, a default value will be used to create a directory and remove it upon script termination. Default: "'+ opts['certdir'] +'"', default=opts['certdir']) + sslgroup.add_argument('--ssl-cakey', dest='cakey', metavar='NAME', + help='Specifies this proxy server\'s (CA) certificate private key. Default: "'+ opts['cakey'] +'"', default=opts['cakey']) + sslgroup.add_argument('--ssl-cacert', dest='cacert', metavar='NAME', + help='Specifies this proxy server\'s (CA) certificate. Default: "'+ opts['cacert'] +'"', default=opts['cacert']) + sslgroup.add_argument('--ssl-certkey', dest='certkey', metavar='NAME', + help='Specifies CA certificate\'s public key. Default: "'+ opts['certkey'] +'"', default=opts['certkey']) + sslgroup.add_argument('--ssl-cacn', dest='cacn', metavar='CN', + help='Sets the common name of the proxy\'s CA authority. If this option is not set, will use --hostname instead. It is required only when no --ssl-cakey/cert were specified and proxy2 will need to generate ones automatically. Default: "'+ opts['cacn'] +'"', default=opts['cacn']) + + # Plugins handling + plugins = parser.add_argument_group("Plugins handling") + plugins.add_argument('-L', '--list-plugins', action='store_true', help='List available plugins.') + plugins.add_argument('-p', '--plugin', dest='plugin', action='append', metavar='PATH', type=str, + help="Specifies plugin's path to be loaded.") + + feed_with_plugin_options(opts, parser) + + params = parser.parse_args() + + if hasattr(params, 'config') and params.config != '': + try: + params = parseParametersFromConfigFile(params) + except Exception as e: + parser.error(str(e)) + + opts.update(params) + else: + opts.update(vars(params)) + + if opts['list_plugins']: + files = sorted([f for f in os.scandir(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plugins/'))], key = lambda f: f.name) + for _, entry in enumerate(files): + if entry.name.endswith(".py") and entry.is_file() and entry.name.lower() not in ['iproxyplugin.py', '__init__.py']: + print('[+] Plugin: {}'.format(entry.name)) + + sys.exit(0) + + if opts['plugin'] != None and len(opts['plugin']) > 0: + for i, opt in enumerate(opts['plugin']): + decomposed = PluginsLoader.decompose_path(opt) + if not os.path.isfile(decomposed['path']): + opt = opt.replace('.py', '') + opt2 = os.path.normpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plugins/{}.py'.format(opt))) + if not os.path.isfile(opt2): + raise Exception('Specified plugin: "%s" does not exist.' % decomposed['path']) + else: + opt = opt2 + + opts['plugins'].add(opt) + + #if opts['debug']: + # opts['trace'] = True + + if opts['silent'] and opts['log']: + parser.error("Options -s and -w are mutually exclusive.") + + if opts['silent']: + opts['log'] = 'none' + elif opts['log'] and len(opts['log']) > 0: + try: + if not os.path.isfile(opts['log']): + with open(opts['log'], 'w') as f: + pass + opts['log'] = opts['log'] + + except Exception as e: + raise Exception('[ERROR] Failed to open log file for writing. Error: "%s"' % e) + else: + opts['log'] = sys.stdout + + if opts['log'] and opts['log'] != sys.stdout: opts['log'] = os.path.normpath(opts['log']) + if opts['cakey']: opts['cakey'] = os.path.normpath(opts['cakey']) + if opts['certdir']: opts['certdir'] = os.path.normpath(opts['certdir']) + if opts['certkey']: opts['certkey'] = os.path.normpath(opts['certkey']) + +def parseParametersFromConfigFile(_params): + parametersRequiringDirectPath = ( + 'log', + 'output', + 'certdir', + 'certkey', + 'cakey', + 'cacert', + 'ssl_certdir', + 'ssl_certkey', + 'ssl_cakey', + 'ssl_cacert', + ) + + translateParamNames = { + 'output' : 'log', + 'proxy_url' : 'proxy_self_url', + 'no_ssl_mitm' : 'no_ssl', + 'ssl_certdir' : 'certdir', + 'ssl_certkey' : 'certkey', + 'ssl_cakey' : 'cakey', + 'ssl_cacert' : 'cacert', + 'ssl_cacn' : 'cacn', + 'drop_invalid_http_requests': 'allow_invalid', + } + + valuesThatNeedsToBeList = ( + 'port', + 'plugin', + ) + + outparams = vars(_params) + config = {} + configBasePath = '' + + if not 'config' in outparams.keys() or not os.path.isfile(outparams['config']): + raise Exception(f'proxy2 config file not found: ({outparams["config"]}) or --config not specified!') + + try: + with open(outparams['config']) as f: + #config = yaml.load(f, Loader=yaml.FullLoader) + config = yaml.load(f) + + outparams.update(config) + + for val in valuesThatNeedsToBeList: + if val in outparams.keys() and val in config.keys(): + if type(config[val]) == str: + outparams[val] = [config[val], ] + else: + outparams[val] = config[val] + + for k, v in ProxyOptionsDefaultValues.items(): + if k not in outparams.keys(): + outparams[k] = v + + for k, v in translateParamNames.items(): + if k in outparams.keys(): + outparams[v] = outparams[k] + if v in outparams.keys(): + outparams[k] = outparams[v] + + configBasePath = os.path.dirname(os.path.abspath(outparams['config'])) + + for paramName in parametersRequiringDirectPath: + if paramName in outparams.keys() and \ + outparams[paramName] != '' and outparams[paramName] != None: + outparams[paramName] = os.path.join(configBasePath, outparams[paramName]) + + return outparams + + except FileNotFoundError as e: + raise Exception(f'proxy2 config file not found: ({outparams["config"]})!') + + except Exception as e: + raise Exception(f'Unhandled exception occured while parsing proxy2 config file: {e}') + + return outparams + + +def feed_with_plugin_options(opts, parser): + logger = ProxyLogger() + plugins = [] + files = sorted([f for f in os.scandir(os.path.join(os.path.dirname(os.path.realpath(__file__)), '../plugins/'))], key = lambda f: f.name) + for _, entry in enumerate(files): + if entry.name.endswith(".py") and entry.is_file() and entry.name.lower() not in ['iproxyplugin.py', '__init__.py']: + plugins.append(entry.path) + + options = opts.copy() + options['plugins'] = plugins + options['verbose'] = True + options['debug'] = False + + plugin_own_options = {} + + pl = PluginsLoader(logger, options) + for name, plugin in pl.get_plugins().items(): + logger.dbg("Fetching plugin {} options.".format(name)) + if hasattr(plugin, 'help'): + plugin_options = parser.add_argument_group("Plugin '{}' options".format(plugin.get_name())) + plugin.help(plugin_options) diff --git a/pluginsloader.py b/lib/pluginsloader.py old mode 100755 new mode 100644 similarity index 95% rename from pluginsloader.py rename to lib/pluginsloader.py index fae20a3..8397971 --- a/pluginsloader.py +++ b/lib/pluginsloader.py @@ -1,160 +1,162 @@ -#!/usr/bin/python3 -import os -import sys -import inspect -from io import StringIO -from proxylogger import ProxyLogger -import csv - -# -# Plugin that attempts to load all of the supplied plugins from -# program launch options. -class PluginsLoader: - class InjectedLogger(ProxyLogger): - def __init__(self, name, options = None): - self.name = name - super().__init__(options) - - def _text(self, txt): - return '[{}] {}'.format(self.name, txt) - - # Info shall be used as an ordinary logging facility, for every desired output. - def info(self, txt, forced = False, **kwargs): - super().info(self._text(txt), forced, **kwargs) - - # Trace by default does not uses [TRACE] prefix. Shall be used - # for dumping packets, headers, metadata and longer technical output. - def trace(self, txt, **kwargs): - super().trace(self._text(txt), **kwargs) - - def dbg(self, txt, **kwargs): - super().dbg(self._text(txt), **kwargs) - - def err(self, txt, **kwargs): - super().err(self._text(txt), **kwargs) - - def fatal(self, txt, **kwargs): - super().fatal(self._text(txt), **kwargs) - - def __init__(self, logger, options, instantiate = True): - self.options = options - self.plugins = {} - self.called = False - self.logger = logger - self.instantiate = instantiate - plugins_count = len(self.options['plugins']) - - if plugins_count > 0: - self.logger.info('Loading %d plugin%s...' % (plugins_count, '' if plugins_count == 1 else 's')) - - for plugin in self.options['plugins']: - self.load(plugin) - - self.called = True - - # Output format: - # plugins = {'plugin1': instance, 'plugin2': instance, ...} - def get_plugins(self): - return self.plugins - - # - # Following function parses input plugin path with parameters and decomposes - # them to extract plugin's arguments along with it's path. - # For instance, having such string: - # -p "plugins/my_plugin.py",argument1="test",argument2,argument3=test2 - # - # It will return: - # {'path':'plugins/my_plugin.py', 'argument1':'t,e,s,t', 'argument2':'', 'argument3':'test2'} - # - @staticmethod - def decompose_path(p): - decomposed = {} - f = StringIO(p) - rows = list(csv.reader(f, quoting=csv.QUOTE_ALL, skipinitialspace=True)) - - for i in range(len(rows[0])): - row = rows[0][i] - if i == 0: - decomposed['path'] = row - continue - - if '=' in row: - s = row.split('=') - decomposed[s[0]] = s[1].replace('"', '') - else: - decomposed[row] = '' - - return decomposed - - - def load(self, path): - instance = None - - self.logger.dbg('Plugin string: "%s"' % path) - decomposed = PluginsLoader.decompose_path(path) - self.logger.dbg('Decomposed as: %s' % str(decomposed)) - - plugin = decomposed['path'].strip() - - if not os.path.isfile(plugin): - _plugin = os.path.normpath(os.path.join(os.path.dirname(__file__), 'plugins/{}'.format(plugin))) - if os.path.isfile(_plugin): - plugin = _plugin - elif os.path.isfile(_plugin+'.py'): - plugin = _plugin + '.py' - - name = os.path.basename(plugin).lower().replace('.py', '') - - if name in self.plugins or name in ['iproxyplugin', '__init__']: - # Plugin already loaded. - return - - self.logger.dbg('Attempting to load plugin: %s ("%s")...' % (name, plugin)) - - try: - sys.path.append(os.path.dirname(plugin)) - __import__(name) - module = sys.modules[name] - self.logger.dbg('Module imported.') - - try: - handler = getattr(module, self.options['plugin_class_name']) - - found = False - for base in inspect.getmro(handler): - if base.__name__ == 'IProxyPlugin': - found = True - break - - if not found: - raise TypeError('Plugin does not inherit from IProxyPlugin.') - - # Call plugin's __init__ with the `logger' instance passed to it. - if self.instantiate: - instance = handler(PluginsLoader.InjectedLogger(name), self.options) - else: - instance = handler - - self.logger.dbg('Found class "%s".' % self.options['plugin_class_name']) - - except AttributeError as e: - self.logger.err('Plugin "%s" loading has failed: "%s".' % - (name, self.options['plugin_class_name'])) - self.logger.err('\tError: %s' % e) - if self.options['debug']: - raise - - except TypeError as e: - self.logger.err('Plugin "{}" instantiation failed due to interface incompatibility.'.format(name)) - raise - - if not instance: - self.logger.err('Didn\'t find supported class in module "%s"' % name) - else: - self.plugins[name] = instance - self.logger.info('Plugin "%s" has been installed.' % name) - - except ImportError as e: - self.logger.err('Couldn\'t load specified plugin: "%s". Error: %s' % (plugin, e)) - if self.options['debug']: - raise +#!/usr/bin/python3 +import os +import sys +import inspect +from io import StringIO +import csv + +from lib.proxylogger import ProxyLogger + + +# +# Plugin that attempts to load all of the supplied plugins from +# program launch options. +class PluginsLoader: + class InjectedLogger(ProxyLogger): + def __init__(self, name, options = None): + self.name = name + super().__init__(options) + + def _text(self, txt): + return '[{}] {}'.format(self.name, txt) + + # Info shall be used as an ordinary logging facility, for every desired output. + def info(self, txt, forced = False, **kwargs): + super().info(self._text(txt), forced, **kwargs) + + # Trace by default does not uses [TRACE] prefix. Shall be used + # for dumping packets, headers, metadata and longer technical output. + def trace(self, txt, **kwargs): + super().trace(self._text(txt), **kwargs) + + def dbg(self, txt, **kwargs): + super().dbg(self._text(txt), **kwargs) + + def err(self, txt, **kwargs): + super().err(self._text(txt), **kwargs) + + def fatal(self, txt, **kwargs): + super().fatal(self._text(txt), **kwargs) + + def __init__(self, logger, options, instantiate = True): + self.options = options + self.plugins = {} + self.called = False + self.logger = logger + self.instantiate = instantiate + plugins_count = len(self.options['plugins']) + + if plugins_count > 0: + self.logger.info('Loading %d plugin%s...' % (plugins_count, '' if plugins_count == 1 else 's')) + + for plugin in self.options['plugins']: + self.load(plugin) + + self.called = True + + # Output format: + # plugins = {'plugin1': instance, 'plugin2': instance, ...} + def get_plugins(self): + return self.plugins + + # + # Following function parses input plugin path with parameters and decomposes + # them to extract plugin's arguments along with it's path. + # For instance, having such string: + # -p "plugins/my_plugin.py",argument1="test",argument2,argument3=test2 + # + # It will return: + # {'path':'plugins/my_plugin.py', 'argument1':'t,e,s,t', 'argument2':'', 'argument3':'test2'} + # + @staticmethod + def decompose_path(p): + decomposed = {} + f = StringIO(p) + rows = list(csv.reader(f, quoting=csv.QUOTE_ALL, skipinitialspace=True)) + + for i in range(len(rows[0])): + row = rows[0][i] + if i == 0: + decomposed['path'] = row + continue + + if '=' in row: + s = row.split('=') + decomposed[s[0]] = s[1].replace('"', '') + else: + decomposed[row] = '' + + return decomposed + + + def load(self, path): + instance = None + + self.logger.dbg('Plugin string: "%s"' % path) + decomposed = PluginsLoader.decompose_path(path) + self.logger.dbg('Decomposed as: %s' % str(decomposed)) + + plugin = decomposed['path'].strip() + + if not os.path.isfile(plugin): + _plugin = os.path.normpath(os.path.join(os.path.dirname(__file__), '../plugins/{}'.format(plugin))) + if os.path.isfile(_plugin): + plugin = _plugin + elif os.path.isfile(_plugin+'.py'): + plugin = _plugin + '.py' + + name = os.path.basename(plugin).lower().replace('.py', '') + + if name in self.plugins or name in ['iproxyplugin', '__init__']: + # Plugin already loaded. + return + + self.logger.dbg('Attempting to load plugin: %s ("%s")...' % (name, plugin)) + + try: + sys.path.append(os.path.dirname(plugin)) + __import__(name) + module = sys.modules[name] + self.logger.dbg('Module imported.') + + try: + handler = getattr(module, self.options['plugin_class_name']) + + found = False + for base in inspect.getmro(handler): + if base.__name__ == 'IProxyPlugin': + found = True + break + + if not found: + raise TypeError('Plugin does not inherit from IProxyPlugin.') + + # Call plugin's __init__ with the `logger' instance passed to it. + if self.instantiate: + instance = handler(PluginsLoader.InjectedLogger(name), self.options) + else: + instance = handler + + self.logger.dbg('Found class "%s".' % self.options['plugin_class_name']) + + except AttributeError as e: + self.logger.err('Plugin "%s" loading has failed: "%s".' % + (name, self.options['plugin_class_name'])) + self.logger.err('\tError: %s' % e) + if self.options['debug']: + raise + + except TypeError as e: + self.logger.err('Plugin "{}" instantiation failed due to interface incompatibility.'.format(name)) + raise + + if not instance: + self.logger.err('Didn\'t find supported class in module "%s"' % name) + else: + self.plugins[name] = instance + self.logger.info('Plugin "%s" has been installed.' % name) + + except ImportError as e: + self.logger.err('Couldn\'t load specified plugin: "%s". Error: %s' % (plugin, e)) + if self.options['debug']: + raise diff --git a/proxylogger.py b/lib/proxylogger.py old mode 100755 new mode 100644 similarity index 92% rename from proxylogger.py rename to lib/proxylogger.py index 80a38f2..1373a5d --- a/proxylogger.py +++ b/lib/proxylogger.py @@ -1,145 +1,146 @@ -#!/usr/bin/python3 - -# To be used as a default proxy logging facility. - -import time -import sys, os -import threading - -globalLock = threading.Lock() - -class ProxyLogger: - options = { - 'debug': False, - 'verbose': False, - 'trace': False, - 'tee': False, - 'log': sys.stdout, - } - - colors_map = { - 'red': 31, - 'green': 32, - 'yellow': 33, - 'blue': 34, - 'magenta': 35, - 'cyan': 36, - 'white': 30, - 'grey': 37, - } - - colors_dict = { - 'error': colors_map['red'], - 'trace': colors_map['magenta'], - 'info ': colors_map['green'], - 'debug': colors_map['grey'], - 'other': colors_map['grey'], - } - - def __init__(self, options = None): - if options != None: - self.options.update(options) - - @staticmethod - def with_color(c, s): - return "\x1b[1;{}m{}\x1b[0m".format(c, s) - - - # Invocation: - # def out(txt, mode='info ', fd=None, color=None, noprefix=False, newline=True): - @staticmethod - def out(txt, fd, mode='info ', **kwargs): - if txt == None or fd == 'none': - return - elif fd == None: - raise Exception('[ERROR] Logging descriptor has not been specified!') - - args = { - 'color': None, - 'noprefix': False, - 'newline': True, - } - args.update(kwargs) - - if args['color']: - col = args['color'] - if type(col) == str and col in ProxyLogger.colors_map.keys(): - col = ProxyLogger.colors_map[col] - else: - col = ProxyLogger.colors_dict.setdefault(mode, ProxyLogger.colors_map['grey']) - - #tm = str(time.strftime("%H:%M:%S", time.gmtime())) - tm = str(time.strftime("%Y-%m-%d/%H:%M:%S", time.gmtime())) - - prefix = '' - if mode: - mode = '[%s] ' % mode - - if not args['noprefix']: - prefix = ProxyLogger.with_color(ProxyLogger.colors_dict['other'], '%s%s: ' - % (mode.upper(), tm)) - - nl = '' - if 'newline' in args: - if args['newline']: - nl = '\n' - - if type(fd) == str: - prefix2 = '%s%s: ' % (mode.upper(), tm) - line = prefix2 + txt + nl - ProxyLogger.writeToLogfile(fd, line) - - #with open(fd, 'a+') as f: - # f.write(line) - # f.flush() - - if 'tee' in args.keys() and args['tee']: - sys.stdout.write(prefix + ProxyLogger.with_color(col, txt) + nl) - sys.stdout.flush() - - else: - fd.write(prefix + ProxyLogger.with_color(col, txt) + nl) - - @staticmethod - def writeToLogfile(fd, line): - with globalLock: - with open(fd, 'a') as f: - f.write(line) - f.flush() - - - # Info shall be used as an ordinary logging facility, for every desired output. - def info(self, txt, forced = False, **kwargs): - if self.options['tee']: - kwargs['tee'] = True - if forced or (self.options['verbose'] or \ - self.options['debug'] or self.options['trace']) \ - or (type(self.options['log']) == str and self.options['log'] != 'none'): - ProxyLogger.out(txt, self.options['log'], 'info', **kwargs) - - # Trace by default does not uses [TRACE] prefix. Shall be used - # for dumping packets, headers, metadata and longer technical output. - def trace(self, txt, **kwargs): - if self.options['tee']: - kwargs['tee'] = True - - if self.options['trace']: - kwargs['noprefix'] = True - ProxyLogger.out(txt, self.options['log'], 'trace', **kwargs) - - def dbg(self, txt, **kwargs): - if self.options['tee']: - kwargs['tee'] = True - if self.options['debug']: - ProxyLogger.out(txt, self.options['log'], 'debug', **kwargs) - - def err(self, txt, **kwargs): - if self.options['tee']: - kwargs['tee'] = True - ProxyLogger.out(txt, self.options['log'], 'error', **kwargs) - - def fatal(self, txt, **kwargs): - if self.options['tee']: - kwargs['tee'] = True - ProxyLogger.out(txt, self.options['log'], 'error', **kwargs) - os._exit(1) +#!/usr/bin/python3 + +# To be used as a default proxy logging facility. + +import time +import sys, os +import threading + +globalLock = threading.Lock() + +class ProxyLogger: + options = { + 'debug': False, + 'verbose': False, + 'trace': False, + 'tee': False, + 'log': sys.stdout, + } + + colors_map = { + 'red': 31, + 'green': 32, + 'yellow': 33, + 'blue': 34, + 'magenta': 35, + 'cyan': 36, + 'white': 30, + 'grey': 37, + } + + colors_dict = { + 'error': colors_map['red'], + 'trace': colors_map['magenta'], + 'info ': colors_map['green'], + 'debug': colors_map['grey'], + 'other': colors_map['grey'], + } + + def __init__(self, options = None): + if options != None: + self.options.update(options) + + @staticmethod + def with_color(c, s): + return "\x1b[1;{}m{}\x1b[0m".format(c, s) + + + # Invocation: + # def out(txt, mode='info ', fd=None, color=None, noprefix=False, newline=True): + @staticmethod + def out(txt, fd, mode='info ', **kwargs): + if txt == None or fd == 'none': + return + elif fd == None: + raise Exception('[ERROR] Logging descriptor has not been specified!') + + args = { + 'color': None, + 'noprefix': False, + 'newline': True, + } + args.update(kwargs) + + if args['color']: + col = args['color'] + if type(col) == str and col in ProxyLogger.colors_map.keys(): + col = ProxyLogger.colors_map[col] + else: + col = ProxyLogger.colors_dict.setdefault(mode, ProxyLogger.colors_map['grey']) + + #tm = str(time.strftime("%H:%M:%S", time.gmtime())) + tm = str(time.strftime("%Y-%m-%d/%H:%M:%S", time.gmtime())) + + prefix = '' + if mode: + mode = '[%s] ' % mode + + if not args['noprefix']: + prefix = ProxyLogger.with_color(ProxyLogger.colors_dict['other'], '%s%s: ' + % (mode.upper(), tm)) + + nl = '' + if 'newline' in args: + if args['newline']: + nl = '\n' + + if type(fd) == str: + prefix2 = '%s%s: ' % (mode.upper(), tm) + line = prefix2 + txt + nl + ProxyLogger.writeToLogfile(fd, line) + + #with open(fd, 'a+') as f: + # f.write(line) + # f.flush() + + if 'tee' in args.keys() and args['tee']: + with globalLock: + sys.stdout.write(prefix + ProxyLogger.with_color(col, txt) + nl) + #sys.stdout.flush() + + else: + with globalLock: + fd.write(prefix + ProxyLogger.with_color(col, txt) + nl) + + @staticmethod + def writeToLogfile(fd, line): + with globalLock: + with open(fd, 'a') as f: + f.write(line) + f.flush() + + # Info shall be used as an ordinary logging facility, for every desired output. + def info(self, txt, forced = False, **kwargs): + if self.options['tee']: + kwargs['tee'] = True + if forced or (self.options['verbose'] or \ + self.options['debug'] or self.options['trace']) \ + or (type(self.options['log']) == str and self.options['log'] != 'none'): + ProxyLogger.out(txt, self.options['log'], 'info', **kwargs) + + # Trace by default does not uses [TRACE] prefix. Shall be used + # for dumping packets, headers, metadata and longer technical output. + def trace(self, txt, **kwargs): + if self.options['tee']: + kwargs['tee'] = True + + if self.options['trace']: + kwargs['noprefix'] = True + ProxyLogger.out(txt, self.options['log'], 'trace', **kwargs) + + def dbg(self, txt, **kwargs): + if self.options['tee']: + kwargs['tee'] = True + if self.options['debug']: + ProxyLogger.out(txt, self.options['log'], 'debug', **kwargs) + + def err(self, txt, **kwargs): + if self.options['tee']: + kwargs['tee'] = True + ProxyLogger.out(txt, self.options['log'], 'error', **kwargs) + + def fatal(self, txt, **kwargs): + if self.options['tee']: + kwargs['tee'] = True + ProxyLogger.out(txt, self.options['log'], 'error', **kwargs) + os._exit(1) diff --git a/sslintercept.py b/lib/sslintercept.py old mode 100755 new mode 100644 similarity index 97% rename from sslintercept.py rename to lib/sslintercept.py index 3c1136b..f9fbfcd --- a/sslintercept.py +++ b/lib/sslintercept.py @@ -1,123 +1,123 @@ -#!/usr/bin/python3 - -#import shutil -import glob -import os -from subprocess import Popen, PIPE - -class SSLInterception: - - def __init__(self, logger, options): - self.logger = logger - self.options = options - self.status = False - - self.dontRemove = [] - - if not options['no_ssl']: - self.setup() - - def setup(self): - def _setup(self): - self.logger.dbg('Setting up SSL interception certificates') - - if not os.path.isabs(self.options['certdir']): - self.logger.dbg('Certificate directory path was not absolute. Assuming relative to current programs\'s directory') - path = os.path.join(os.path.dirname(os.path.realpath(__file__)), self.options['certdir']) - self.options['certdir'] = path - self.logger.dbg('Using path: "%s"' % self.options['certdir']) - - # Step 1: Create directory for certificates and asynchronous encryption keys - if not os.path.isdir(self.options['certdir']): - try: - self.logger.dbg("Creating directory for certificate: '%s'" % self.options['certdir']) - os.mkdir(self.options['certdir']) - except Exception as e: - self.logger.fatal("Couldn't make directory for certificates: '%s'" % e) - return False - - # Step 2: Create CA key - if not self.options['cakey']: - self.options['cakey'] = os.path.join(self.options['certdir'], 'ca.key') - - if not os.path.isdir(self.options['cakey']): - self.logger.dbg("Creating CA key file: '%s'" % self.options['cakey']) - p = Popen(["openssl", "genrsa", "-out", self.options['cakey'], "2048"], stdout=PIPE, stderr=PIPE) - (out, error) = p.communicate() - self.logger.dbg(out + error) - - if not self.options['cakey']: - self.logger.fatal('Creating of CA key process has failed.') - return False - else: - self.dontRemove.append(self.options['cakey']) - self.logger.info('Using provided CA key file: {}'.format(self.options['cakey'])) - - # Step 3: Create CA certificate - if not self.options['cacert']: - self.options['cacert'] = os.path.join(self.options['certdir'], 'ca.crt') - - if not os.path.isdir(self.options['cacert']): - self.logger.dbg("Creating CA certificate file: '%s'" % self.options['cacert']) - p = Popen(["openssl", "req", "-new", "-x509", "-days", "3650", "-key", self.options['cakey'], "-out", self.options['cacert'], "-subj", "/CN="+self.options['cacn']], stdout=PIPE, stderr=PIPE) - (out, error) = p.communicate() - self.logger.dbg(out + error) - - if not self.options['cacert']: - self.logger.fatal('Creating of CA certificate process has failed.') - return False - else: - self.dontRemove.append(self.options['cacert']) - self.logger.info('Using provided CA certificate file: {}'.format(self.options['cacert'])) - - # Step 4: Create certificate key file - if not self.options['certkey']: - self.options['certkey'] = os.path.join(self.options['certdir'], 'cert.key') - - if not os.path.isdir(self.options['certkey']): - self.logger.dbg("Creating Certificate key file: '%s'" % self.options['certkey']) - self.logger.dbg("Creating CA key file: '%s'" % self.options['cakey']) - p = Popen(["openssl", "genrsa", "-out", self.options['certkey'], "2048"], stdout=PIPE, stderr=PIPE) - (out, error) = p.communicate() - self.logger.dbg(out + error) - - if not self.options['certkey'] or not os.path.isfile(self.options['certkey']): - self.logger.fatal('Creating of Certificate key process has failed.') - return False - else: - self.dontRemove.append(self.options['certkey']) - self.logger.info('Using provided Certificate key: {}'.format(self.options['certkey'])) - - self.logger.dbg('SSL interception has been setup.') - return True - - self.logger.info('Preparing SSL certificates and keys for https traffic interception...') - self.status = _setup(self) - return self.status - - def cleanup(self): - if not self.status: - return - - try: - #shutil.rmtree(self.options['certdir']) - - for i in range(len(self.dontRemove)): - self.dontRemove[i] = os.path.abspath(self.dontRemove[i]) - - for file in glob.glob(os.path.join(self.options['certdir'], '*.*')): - absfile = os.path.abspath(file) - if absfile in self.dontRemove: - self.logger.dbg('SSL file not cleaned: "{}"'.format(absfile)) - continue - - self.logger.dbg('Removing old certificate: {}'.format(absfile)) - os.remove(absfile) - - self.logger.dbg('SSL interception files cleaned up.') - - except Exception as e: - self.logger.err("Couldn't perform SSL interception files cleaning: '%s'" % e) - - def __str__(self): - return 'SSL %sbeing intercepted.' % ('NOT ' if not self.status else '') +#!/usr/bin/python3 + +#import shutil +import glob +import os +from subprocess import Popen, PIPE + +class SSLInterception: + + def __init__(self, logger, options): + self.logger = logger + self.options = options + self.status = False + + self.dontRemove = [] + + if not options['no_ssl']: + self.setup() + + def setup(self): + def _setup(self): + self.logger.dbg('Setting up SSL interception certificates') + + if not os.path.isabs(self.options['certdir']): + self.logger.dbg('Certificate directory path was not absolute. Assuming relative to current programs\'s directory') + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), self.options['certdir']) + self.options['certdir'] = path + self.logger.dbg('Using path: "%s"' % self.options['certdir']) + + # Step 1: Create directory for certificates and asynchronous encryption keys + if not os.path.isdir(self.options['certdir']): + try: + self.logger.dbg("Creating directory for certificate: '%s'" % self.options['certdir']) + os.mkdir(self.options['certdir']) + except Exception as e: + self.logger.fatal("Couldn't make directory for certificates: '%s'" % e) + return False + + # Step 2: Create CA key + if not self.options['cakey']: + self.options['cakey'] = os.path.join(self.options['certdir'], 'ca.key') + + if not os.path.isdir(self.options['cakey']): + self.logger.dbg("Creating CA key file: '%s'" % self.options['cakey']) + p = Popen(["openssl", "genrsa", "-out", self.options['cakey'], "2048"], stdout=PIPE, stderr=PIPE) + (out, error) = p.communicate() + self.logger.dbg(out + error) + + if not self.options['cakey']: + self.logger.fatal('Creating of CA key process has failed.') + return False + else: + self.dontRemove.append(self.options['cakey']) + self.logger.info('Using provided CA key file: {}'.format(self.options['cakey'])) + + # Step 3: Create CA certificate + if not self.options['cacert']: + self.options['cacert'] = os.path.join(self.options['certdir'], 'ca.crt') + + if not os.path.isdir(self.options['cacert']): + self.logger.dbg("Creating CA certificate file: '%s'" % self.options['cacert']) + p = Popen(["openssl", "req", "-new", "-x509", "-days", "3650", "-key", self.options['cakey'], "-out", self.options['cacert'], "-subj", "/CN="+self.options['cacn']], stdout=PIPE, stderr=PIPE) + (out, error) = p.communicate() + self.logger.dbg(out + error) + + if not self.options['cacert']: + self.logger.fatal('Creating of CA certificate process has failed.') + return False + else: + self.dontRemove.append(self.options['cacert']) + self.logger.info('Using provided CA certificate file: {}'.format(self.options['cacert'])) + + # Step 4: Create certificate key file + if not self.options['certkey']: + self.options['certkey'] = os.path.join(self.options['certdir'], 'cert.key') + + if not os.path.isdir(self.options['certkey']): + self.logger.dbg("Creating Certificate key file: '%s'" % self.options['certkey']) + self.logger.dbg("Creating CA key file: '%s'" % self.options['cakey']) + p = Popen(["openssl", "genrsa", "-out", self.options['certkey'], "2048"], stdout=PIPE, stderr=PIPE) + (out, error) = p.communicate() + self.logger.dbg(out + error) + + if not self.options['certkey'] or not os.path.isfile(self.options['certkey']): + self.logger.fatal('Creating of Certificate key process has failed.') + return False + else: + self.dontRemove.append(self.options['certkey']) + self.logger.info('Using provided Certificate key: {}'.format(self.options['certkey'])) + + self.logger.dbg('SSL interception has been setup.') + return True + + self.logger.info('Preparing SSL certificates and keys for https traffic interception...') + self.status = _setup(self) + return self.status + + def cleanup(self): + if not self.status: + return + + try: + #shutil.rmtree(self.options['certdir']) + + for i in range(len(self.dontRemove)): + self.dontRemove[i] = os.path.abspath(self.dontRemove[i]) + + for file in glob.glob(os.path.join(self.options['certdir'], '*.*')): + absfile = os.path.abspath(file) + if absfile in self.dontRemove: + self.logger.dbg('SSL file not cleaned: "{}"'.format(absfile)) + continue + + self.logger.dbg('Removing old certificate: {}'.format(absfile)) + os.remove(absfile) + + self.logger.dbg('SSL interception files cleaned up.') + + except Exception as e: + self.logger.err("Couldn't perform SSL interception files cleaning: '%s'" % e) + + def __str__(self): + return 'SSL %sbeing intercepted.' % ('NOT ' if not self.status else '') diff --git a/plugins/malleable_redirector.py b/plugins/malleable_redirector.py index ac7d0ac..611b2a7 100644 --- a/plugins/malleable_redirector.py +++ b/plugins/malleable_redirector.py @@ -846,7 +846,7 @@ class AlterHostHeader(Exception): #'teamserver_url' : [], 'drop_action': 'redirect', 'action_url': ['https://google.com', ], - 'proxy_pass': [], + 'proxy_pass': {}, 'log_dropped': False, 'report_only': False, 'ban_blacklisted_ip_addresses': True, @@ -958,14 +958,16 @@ def help(self, parser): except Exception as e: self.logger.fatal(f'Unhandled exception occured while parsing Malleable-redirector config file: {e}') + profileSkipped = False if ('profile' not in self.proxyOptions.keys()) or (not self.proxyOptions['profile']): self.logger.err(''' -============================================================================================== - MALLEABLE C2 PROFILE PATH NOT SPECIFIED! LOGIC BASED ON PARSING HTTP REQUESTS WON\'T BE USED! -============================================================================================== +================================================================================================= + MALLEABLE C2 PROFILE PATH NOT SPECIFIED! LOGIC BASED ON PARSING HTTP REQUESTS WILL BE DISABLED! +================================================================================================= ''') self.malleable = None + profileSkipped = True else: self.malleable = MalleableParser(self.logger) @@ -974,8 +976,9 @@ def help(self, parser): if not self.malleable.parse(self.proxyOptions['profile']): self.logger.fatal('Could not parse specified Malleable C2 profile!') - if not self.proxyOptions['action_url'] or len(self.proxyOptions['action_url']) == 0: - self.logger.fatal('Action/Drop URL must be specified!') + if not profileSkipped and (not self.proxyOptions['action_url'] or len(self.proxyOptions['action_url']) == 0): + if self.proxyOptions['drop_action'] != 'reset': + self.logger.fatal('Action/Drop URL must be specified!') elif type(self.proxyOptions['action_url']) == str: url = self.proxyOptions['action_url'] @@ -984,26 +987,53 @@ def help(self, parser): else: self.proxyOptions['action_url'] = [x.strip() for x in url.split(',')] - if (type(self.proxyOptions['proxy_pass']) != list) and \ + elif type(self.proxyOptions['action_url']) == None and profileSkipped: + self.proxyOptions['action_url'] = [] + + if self.proxyOptions['proxy_pass'] == None: + self.proxyOptions['proxy_pass'] = {} + + elif (type(self.proxyOptions['proxy_pass']) != list) and \ (type(self.proxyOptions['proxy_pass']) != tuple): - self.logger.fatal('Proxy Pass must be a list of entries if specified!') + self.logger.fatal('Proxy Pass must be a list of entries if used!') + else: - passes = [] - for entry in self.proxyOptions['proxy_pass']: - entry = entry.strip() + passes = {} + num = 0 + for entry in self.proxyOptions['proxy_pass']: if len(entry) < 6: self.logger.fatal('Invalid Proxy Pass entry: ({}): too short!',format(entry)) + splits = list(filter(None, entry.strip().split(' '))) + url = '' host = '' - if len(entry.split(' ')) > 2: - self.logger.fatal('Invalid Proxy Pass entry: ({}): entry contains more than one space character breaking syntax! Neither URL nor host can contain space.'.format(entry)) - else: - (url, host) = entry.split(' ') - url = url.strip() - host = host.strip().replace('https://', '').replace('http://', '').replace('/', '') + if len(splits) < 2: + self.logger.fatal('Invalid Proxy Pass entry: ({}): invalid syntax: required!'.format(entry)) + + url = splits[0].strip() + host = splits[1].strip() + host = host.strip().replace('https://', '').replace('http://', '').replace('/', '') + + passes[num] = {} + passes[num]['url'] = url + passes[num]['redir'] = host + passes[num]['options'] = {} + + if len(splits) > 2: + opts = ' '.join(splits[2:]) + for opt in opts.split(','): + opt2 = opt.split('=') + k = opt2[0] + v = '' + if len(opt2) == 2: + v = opt2[1] + else: + v = '='.join(opt2[1:]) + + passes[num]['options'][k.strip()] = v.strip() if len(url) == 0 or len(host) < 4: self.logger.fatal('Invalid Proxy Pass entry: (url="{}" host="{}"): either URL or host part were missing or too short (schema is ignored).',format(url, host)) @@ -1011,12 +1041,33 @@ def help(self, parser): if not url.startswith('/'): self.logger.fatal('Invalid Proxy Pass entry: (url="{}" host="{}"): URL must start with slash character (/).',format(url, host)) - passes.append((url, host)) - self.logger.info('Will proxy-pass requests targeted to: "^{}$" onto host: "{}"'.format(url, host)) + num += 1 if len(passes) > 0: - self.proxyOptions['proxy_pass'] = passes[:] - self.logger.info('Collected {} proxy-pass statements.'.format(len(passes))) + self.proxyOptions['proxy_pass'] = passes.copy() + + lines = [] + for num, e in passes.items(): + line = "\tRule {}. Proxy requests with URL: \"^{}$\" to host {}".format( + num, e['url'], e['redir'] + ) + + if len(e['options']) > 0: + line += " (options: " + opts = [] + for k,v in e['options'].items(): + if len(v) > 0: + opts.append("{}: {}".format(k, v)) + else: + opts.append("{}".format(k)) + + line += ', '.join(opts) + ")" + + lines.append(line) + + self.logger.info('Collected {} proxy-pass statements: \n{}'.format( + len(passes), '\n'.join(lines) + )) if not self.proxyOptions['teamserver_url']: self.logger.fatal('Teamserver URL must be specified!') @@ -1088,13 +1139,16 @@ def help(self, parser): if 'add_peers_to_whitelist_if_they_sent_valid_requests' in self.proxyOptions.keys() and self.proxyOptions['add_peers_to_whitelist_if_they_sent_valid_requests'] != None \ and len(self.proxyOptions['add_peers_to_whitelist_if_they_sent_valid_requests']) > 0: + log = 'Dynamic peers whitelisting enabled with thresholds:\n' + for k, v in self.proxyOptions['add_peers_to_whitelist_if_they_sent_valid_requests'].items(): if k not in ProxyPlugin.DefaultRedirectorConfig['add_peers_to_whitelist_if_they_sent_valid_requests'].keys(): self.logger.err("Dynamic whitelisting threshold named ({}) not supported! Skipped..".format(k)) log += '\t{}: {}\n'.format(k, str(v)) self.logger.dbg(log) + else: self.logger.info("Dynamic peers whitelisting disabled.") self.proxyOptions['add_peers_to_whitelist_if_they_sent_valid_requests'] = {} @@ -1762,7 +1816,8 @@ def validatePeerAndHttpHeaders(self, peerIP, ts, req, req_body, res, res_body, p msg = '[ALLOW, {}, reason:99, {}] Peer IP and HTTP headers did not contain anything suspicious.'.format( ts, peerIP) - if len(ipLookupDetails) == 0: + if not ipLookupDetails or \ + (type(ipLookupDetails) == dict and len(ipLookupDetails) == 0): respJson['ipgeo'] = self.printPeerInfos(peerIP, True) else: respJson['ipgeo'] = ipLookupDetails @@ -1774,28 +1829,29 @@ def validatePeerAndHttpHeaders(self, peerIP, ts, req, req_body, res, res_body, p else: return (False, '') - def drop_check(self, req, req_body, malleable_meta): - peerIP = self.get_peer_ip(req) - ts = datetime.now().strftime('%Y-%m-%d/%H:%M:%S') - userAgentValue = '' - if self.malleable != None: - userAgentValue = req.headers.get('User-Agent') - - (outstatus, outresult) = self.validatePeerAndHttpHeaders(peerIP, ts, req, req_body, '', '', None) - if outstatus: - return outresult - + def processProxyPass(self, ts, peerIP, req, processNodrops): if self.proxyOptions['proxy_pass'] != None and len(self.proxyOptions['proxy_pass']) > 0 \ and self.proxyOptions['policy']['allow_proxy_pass']: - for entry in self.proxyOptions['proxy_pass']: - (url, host) = entry + + for num, entry in self.proxyOptions['proxy_pass'].items(): + url = entry['url'] + host = entry['redir'] + opts = '' + + if processNodrops: + if ('options' not in entry.keys()) or ('nodrop' not in entry['options'].keys()): + continue + + if 'nodrop' in entry['options'].keys(): + opts += ', nodrop' if re.match('^' + url + '$', req.path, re.I) != None: self.logger.info( - '[ALLOW, {}, reason:0, {}] Request conforms ProxyPass entry (url="{}" host="{}"). Passing request to specified host.'.format( - ts, peerIP, url, host + '[ALLOW, {}, reason:0, {}] Request conforms ProxyPass entry {} (url="{}" host="{}"{}). Passing request to specified host.'.format( + ts, peerIP, num, url, host, opts ), color='green') self.printPeerInfos(peerIP) + self.report(False, ts, peerIP, req.path, req.headers.get('User-Agent')) del req.headers['Host'] req.headers['Host'] = host @@ -1803,7 +1859,21 @@ def drop_check(self, req, req_body, malleable_meta): raise ProxyPlugin.AlterHostHeader(host) else: - self.logger.dbg('(ProxyPass) Processed request with URL ("{}"...) didnt match ProxyPass entry URL regex: "^{}$".'.format(req.path[:32], url)) + self.logger.dbg('(ProxyPass) Processed request with URL ("{}"...) didnt match ProxyPass entry {} URL regex: "^{}$".'.format(req.path[:32], num, url)) + + + def drop_check(self, req, req_body, malleable_meta): + peerIP = self.get_peer_ip(req) + ts = datetime.now().strftime('%Y-%m-%d/%H:%M:%S') + userAgentValue = req.headers.get('User-Agent') + + self.processProxyPass(ts, peerIP, req, True) + + (outstatus, outresult) = self.validatePeerAndHttpHeaders(peerIP, ts, req, req_body, '', '', None) + if outstatus: + return outresult + + self.processProxyPass(ts, peerIP, req, False) # User-agent conformancy if self.malleable != None: @@ -1815,7 +1885,7 @@ def drop_check(self, req, req_body, malleable_meta): userAgentValue, self.malleable.config['useragent'])) return self.report(True, ts, peerIP, req.path, userAgentValue) else: - self.logger.dbg("(No malleable profile) User-agent test skipped, as there was no profile provided.", color='red') + self.logger.dbg("(No malleable profile) User-agent test skipped, as there was no profile provided.", color='magenta') if self.proxyOptions['mitigate_replay_attack']: with SqliteDict(ProxyPlugin.RequestsHashesDatabaseFile) as mydict: @@ -1881,7 +1951,7 @@ def drop_check(self, req, req_body, malleable_meta): self.drop_reason('[DROP, {}, reason:11a, {}] Requested URI does not align any of Malleable defined variants: "{}"'.format(ts, peerIP, req.path)) return self.report(True, ts, peerIP, req.path, userAgentValue) else: - self.logger.dbg("(No malleable profile) Request contents validation skipped, as there was no profile provided.", color='red') + self.logger.dbg("(No malleable profile) Request contents validation skipped, as there was no profile provided.", color='magenta') return self.report(False, ts, peerIP, req.path, userAgentValue) @@ -1904,7 +1974,7 @@ def _client_request_inspect(self, section, variant, req, req_body, malleable_met rehdrskeys = [x.lower() for x in req.headers.keys()] if self.malleable == None: - self.logger.dbg("(No malleable profile) Request contents validation skipped, as there was no profile provided.", color='red') + self.logger.dbg("(No malleable profile) Request contents validation skipped, as there was no profile provided.", color='magenta') return False self.logger.dbg("Deep request inspection of URI ({}) parsed as section:{}, variant:{}".format( diff --git a/proxy2.py b/proxy2.py index a8e134e..f29cbb7 100755 --- a/proxy2.py +++ b/proxy2.py @@ -42,7 +42,6 @@ import threading import gzip, zlib import json, re -import optionsparser import traceback import threading import requests @@ -50,16 +49,18 @@ from urllib.parse import urlparse, parse_qsl from subprocess import Popen, PIPE -from proxylogger import ProxyLogger -from pluginsloader import PluginsLoader -from sslintercept import SSLInterception from http.server import BaseHTTPRequestHandler, HTTPServer -from socketserver import ThreadingMixIn, ForkingMixIn - -import plugins.IProxyPlugin +from socketserver import ThreadingMixIn from io import StringIO, BytesIO from html.parser import HTMLParser +import lib.optionsparser +import plugins.IProxyPlugin +from lib.proxylogger import ProxyLogger +from lib.pluginsloader import PluginsLoader +from lib.sslintercept import SSLInterception + + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) ssl._create_default_https_context = ssl._create_unverified_context @@ -100,8 +101,7 @@ # Asynchronously serving HTTP server class. -#class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): -class ThreadingHTTPServer(ForkingMixIn, HTTPServer): +class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): address_family = socket.AF_INET # ThreadMixIn, Should the server wait for thread termination? @@ -1043,7 +1043,7 @@ def init(): global logger global sslintercept - optionsparser.parse_options(options, VERSION) + lib.optionsparser.parse_options(options, VERSION) logger = ProxyLogger(options) pluginsloaded = PluginsLoader(logger, options) sslintercept = SSLInterception(logger, options) diff --git a/requirements.txt b/requirements.txt index 2a44969..c2e93f7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ brotli requests PyYaml -sqlitedict \ No newline at end of file +sqlitedict +tornado \ No newline at end of file