Lots of improvements, bug fixes and refactorings. Added also pre-gene…

…rated CA certificate & key

ca-cert/ca.crt

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC+zCCAeOgAwIBAgIJANTTZNmoSV/qMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCXByb3h5MiBDQTAeFw0xOTAxMTMxMzI0MjNaFw0yOTAxMTAxMzI0MjNaMBQx
+EjAQBgNVBAMMCXByb3h5MiBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAKwJPhLJsfao0pfKFxwNNQTa9Xblm30jJUuACl+JARBJAy44Q6qCZPKoE1AY
+WaNsgH64U4UcgVj+d+vzHLjhK0jeBkSYtnj1nKODIfHagOrrknSVVxSdYf3WcVoR
+8aGiDJOzBek/KUenGz+DxU08U7Hyuw2K/fZae7x6btEn1gbsxFn9b9lGvib7NtO2
+cfI+AVB+kT5tlMTHcw/pnuVWHQM78PZGTe44FaASMeH4VhnFj87RstHbmVnqvxb4
+IEuAlLrMrbUVYalejsM5QPqwksmZgS2G8UElQOWY4eKg9NxyBjjrkPwCE/DhXSIF
+WzHHbq1EBeEqUR85GtLGUpbWrnECAwEAAaNQME4wHQYDVR0OBBYEFHJxb0wgqh5m
+OCYpVd+VrCXbVkioMB8GA1UdIwQYMBaAFHJxb0wgqh5mOCYpVd+VrCXbVkioMAwG
+A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAxhjxq7Ij9pc8EPicgkLA32
+w8ybuDcp49x1k5OqC8jKV04LdOHUGFD46VIWpTb19TxKDy6kRX0tZ0QWH9r/7Tgf
+xUWy0jUd+05z/3qN7cDGVGm+BCfv4UsfD5zO6lkfOk0mpsAYo/IxfThH/BbjCEp/
+7hk6aVqhk96xQy7qlcEt3lGwDwqfiS84UixcPgDsckyt50ln/zzBL6Whzjnnahx2
+PNCy5USV5udjyzugaOiPLeGnEbLq0LaZEsQTC6+wvPjURg9fgd8e4foV2SCKhhI5
+rZAkmSX6Q0ymtbMlmSLkWvLquTBZsvO+o64MWDifZQ52RtTNCuBGvuW1B8rmWZE=
+-----END CERTIFICATE-----

ca-cert/ca.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEArAk+Esmx9qjSl8oXHA01BNr1duWbfSMlS4AKX4kBEEkDLjhD
+qoJk8qgTUBhZo2yAfrhThRyBWP536/McuOErSN4GRJi2ePWco4Mh8dqA6uuSdJVX
+FJ1h/dZxWhHxoaIMk7MF6T8pR6cbP4PFTTxTsfK7DYr99lp7vHpu0SfWBuzEWf1v
+2Ua+Jvs207Zx8j4BUH6RPm2UxMdzD+me5VYdAzvw9kZN7jgVoBIx4fhWGcWPztGy
+0duZWeq/FvggS4CUusyttRVhqV6OwzlA+rCSyZmBLYbxQSVA5Zjh4qD03HIGOOuQ
+/AIT8OFdIgVbMcdurUQF4SpRHzka0sZSltaucQIDAQABAoIBAFNCAdWL4WHTYF/v
+gPGlfqRD54nMI00Thkgcxmhn4Kjl/PEQb8cEZiB9sSMRNch+iU1KnbkNC5hrRtRd
+Cuh6qL0SHoxyL9UoYM9Ndk8bBUssCOv9HnCuni7/6knB52PnDhkpCdJRLAQuXmSF
+vCXd7U9wfpBWVQQ11C5qPllg4xbkEfggDEesVranAQVprgex6KlJKBQcRfjwZmv3
+/MRyDwUFOpYjx7riQm+B/TNSQgy6wiqXh14ejp0FEHWlVd9I4K0MWUMXD2zkjKcE
+h2qiIrIBuTmIOEL2IeelVhCLgBc7yd9Dh3cXGpGTP8sqGZl1IHPBH5wOK/PwVXic
+FwGU+gECgYEA2gAOnM4hbuTbNXnM1E2D4k9Z3KH/XFRoGmUkzoxy/ZQK9ZvTfwsQ
+BVsazJ3jm1r4jm5o8t5usWDwIeyEpULs40Si9IzW1vOvydqueBR8K3al+u8BIs4H
+IH4aofsijhIxK+p0Z8JnzfZOGKBmkUbcT/1HmOKrjZHO8XXa38rroNECgYEAygYa
+eJ0xQItTn9h3Vm04e9kWKn/fBHD/vvoXTjV2m32nOxLqu9mC42LmU7M6b5vis7wz
+CwzDGDHFlMH+MPD4XBt5Bad6nOtLArTTy4NXizM+Vg5XTQHp0zo+OMEFX8UlJqMb
+V9tAbO0CPPCqzQVIIpxwybVpG7koxaIo2bvIm6ECgYEAk3fIasByE18S/pi3O8J3
+/aZqBns7kAy1I23aOTL/MpRr2Xug1Wb5XnYjqdkAt/4Q9+Cuc+SOAsWti3VAwb6F
+GrQ6e62uQ1gzSRv6O9a3rHslipsVLKMsZQmJIJwO4wZhZvDB79Ktf8EnUTdoSswh
+iqauQTjMjgbdc6+i8RKG1JECgYAKIeo7+G5a9WH5V2sM26eElqvE7+rolx5MntCC
+bK4JOHElxlodl9g3vWMd+ZRJusDREPRibn5ufTiSsHQmUj+ypvIX9YFx019MwHMK
+9whyA9zxhgCc7SakIHy0bgHt/r5RRMb/ThDaJb0B/3Qhmk025y/E/iNKb6v61ZpE
+6WUzYQKBgClLRIUjVRXxRoWaGgZi2hHF1s3VzmO4kOLDs7z939Ygx5tJzNaNBw4L
+hF5+mRi3C1nvcS4OP4xv4IstKB+RR03Vp1Q5bvouYwLiPV+rBDzjRyQQXfJPJbnt
+Ok+nI4uku5efUQbcCRjVdS9UgSonfwEz+q7eqD3BJidFFIHXzIec
+-----END RSA PRIVATE KEY-----

ca-cert/cert.key

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEA4R1UwUsGO4SSXavpoNu1gqQsfLW1flR/PMFkNVRxwwHFnekV
+62Ig4EreYXLQE49cZs3HzFR5sM+TaWTUWdtxVwqr9e1eTyF15vG9o3zAcdCRwefF
+AxrOoBcICw6Y66JYczpg+Fhc+gzfisG5s86dPzgFKIDXvBED31Q5FYhWWMv0uaTh
+1UlhobH/yUBEJrMRW2U7Q5vkk1vsG/LGoN9Obssyu96qjFH0xg5s+pgLWIhaggP/
+geM7+FI4XiCkDh1uIdZf5TJh5565bZYWI20r0KZRQwdoi4+ynkY4O+/iGoW30eSx
+KqGFGwrtR4G6OEg8ZNqHEoAqzFTgnj0HGSD8nQIDAQABAoIBAH6yeGI1rHNRAOOx
+ftMm9PhrGBK0XkqUmewC2Dhfp4tecu+WIN0SpHg4CwMDkHKBzDqb9KhenwLRQSEf
+O5i7NgYMHo5SIzMcHPR2+AmMi+9CuNZOcIZ3zvUxITi/5XcxLuDjaXI8oU+mcSXy
+NGcrkTrkd5q9MS5K0UgfaeVhj193lKfJe0QY7qdt9xlv2S2v5PE1sRcoOuTKkyVu
+bg5wO7IA5z5c2sbMSAeHlTs+RB8pL2BVzPhzCxcnwPCc5WNWaBfkYfOjWMD4sE/8
+cYnCux+Kdm6za+dUiwtystMOK/99ZMIzETSWhnPHot7t2llCj592FlEafP8aRTen
+QoRzXUECgYEA9GkHrn684GJR4XIFb/PB818gKOjTqFCseRxEBTPEpqxhz/NMUF0C
+AUYORY6cUkFgJ0YtVq4Hk3QwX34iPdpbEhoj9jmdwGTSLh6YcdRpsIUo7vkAKYIN
+flSYD62y68DNJhhujmKGZqMAylRu9wQNMX+eHjK4/vH3qgDGoaiXEC0CgYEA68oQ
+lDIPshzwe+cXuDmgA1xpervULfvXK4oXxMzYn1+ogEA+8ot9Eo9c0ziSVghmQFc5
+dlHj669RKOrHyNT14fj3RT1NElxxWeFxm1qfiEyqZKlkQ/PmVsvGTzdCisZ0deHN
+AdZpgvcQV/+qTmNJrIaDoKkU0+fqwy5uoO989DECgYA9znWn1drzr0lfhpMDbZQF
+dG/QiJhFvyjuc4xr+Fxpfbw6dx88T1jbc5jWVCsJzgh/xgpfGiFGU6KL83y7QYW4
+PS4M7SMMbTKNgSUx2/JiNjpUvFkjJgU9hizyAg31+kqmsJT8osO0HtJrWBC7nKWt
+d8VHg7Iunofv0MRqSxTwfQKBgCAWvNDeS0KLK7NBDPpWZU9vyS8Z1tN3PZ5ASeHP
+mv99jjn+BFMP5rKa7iAUx92LgRbqh/hxRppxnpL5+Lx9NwVM06IJqK6CBC8ePk7N
+M37iKCJQ50NUMxnG27M2Kwkl3v2YAEVqv6tCImhHdA789i7Tk6BOwnXgTxPHAulG
+DnRRAoGBAPMXCtGbPGjXOwa+lOKhKNMVeneGGXsq2YfJUU4C3LiPxCW/PgPXHfDx
+B3eNm4tLSKwzklU0OqHo/QEWLaQ7wQOoLDjJT3s2QZLW2F7DmtJCn8qigA66ig/D
+j9RNWkS8PXTiCG726+AjJSok4djSoKK04EXaChNtvogZR0qb0PL7
+-----END RSA PRIVATE KEY-----

optionsparser.py

@@ -10,8 +10,10 @@ def parse_options(options, version):
     parser = OptionParser(usage=usage, version="%prog " + version)
 
     # General options
-    parser.add_option(  "-v", "--verbose", dest='trace',
-        help="Displays verbose output along with packets' contents dumping/tracing.", action="store_true")
+    parser.add_option(  "-v", "--verbose", dest='verbose',
+        help="Displays verbose output.", action="store_true")
+    parser.add_option(  "-V", "--trace", dest='trace',
+        help="Displays HTTP requests and responses.", action="store_true")
     parser.add_option(  "-d", "--debug", dest='debug',
         help="Displays debugging informations (implies verbose output).", action="store_true")
     parser.add_option(  "-s", "--silent", dest='silent',
@@ -72,8 +74,8 @@ def parse_options(options, version):
             else:
                 options['plugins'].add(opt)
 
-    if params.debug:
-        options['trace'] = True
+    #if params.debug:
+    #   options['trace'] = True
 
     if params.silent and params.log:
         parser.error("Options -s and -w are mutually exclusive.")
@@ -87,3 +89,4 @@ def parse_options(options, version):
             raise Exception('[ERROR] Failed to open log file for writing. Error: "%s"' % e)
     else:
         options['log'] = sys.stdout
+

plugins/dummy.py

@@ -17,8 +17,9 @@ class ProxyHandler:
     # params =  {'path':'plugins/my_plugin.py', 
     #           'argument1':'test', 'argument2':'', 'argument3':'test2'}
     #
-    def __init__(self, logger, params):
+    def __init__(self, logger, params, proxyOptions):
         self.logger = logger
+        self.proxyOptions = proxyOptions
         logger.info('hello world from __init__ in ProxyHandler.')
         if len(params) > 1:
             logger.info('\tI have received such params: %s' % str(params))

plugins/sslstrip.py

@@ -5,8 +5,9 @@
 class ProxyHandler:
     replaced_urls = deque(maxlen=1024)
 
-    def __init__(self, logger, params):
+    def __init__(self, logger, params, proxyOptions):
         self.logger = logger
+        self.proxyOptions = proxyOptions
         logger.info('hello world from __init__ in SSLStrip ProxyHandler')
 
     def request_handler(self, req, req_body):

pluginsloader.py

@@ -84,15 +84,16 @@ def load(self, path):
                 handler = getattr(module, self.options['plugin_class_name'])
 
                 # Call plugin's __init__ with the `logger' instance passed to it.
-                instance = handler(self.logger, decomposed)
+                instance = handler(self.logger, decomposed, self.options)
 
                 self.logger.dbg('Found class "%s".' % self.options['plugin_class_name'])
 
             except AttributeError as e:
                 self.logger.err('Plugin "%s" loading has failed: "%s".' % 
                     (name, self.options['plugin_class_name']))
                 self.logger.err('\tError: %s' % e)
-                return
+                if self.options['debug']:
+                    raise
 
             if not instance:
                 self.logger.err('Didn\'t find supported class in module "%s"' % name)
@@ -101,4 +102,6 @@ def load(self, path):
                 self.logger.info('Plugin "%s" has been installed.' % name)
 
         except ImportError as e:
-            self.logger.err('Couldn\' load specified plugin: "%s". Error: %s' % (plugin, e))
+            self.logger.err('Couldn\'t load specified plugin: "%s". Error: %s' % (plugin, e))
+            if self.options['debug']:
+                raise

proxy2.py

@@ -28,6 +28,7 @@
 
 import time
 import sys, os
+import brotli
 import socket, ssl, select
 import httplib, urlparse
 import threading
@@ -51,15 +52,16 @@
 options = {
     'hostname': 'localhost',
     'port': 8080,
-    'debug': True,                  # Print's out debuging informations
-    'trace': True,                  # Displays packets contents
+    'debug': False,                  # Print's out debuging informations
+    'verbose': True,
+    'trace': False,                  # Displays packets contents
     'log': None,
     'proxy_self_url': 'http://proxy2.test/',
     'timeout': 5,
     'no_ssl': False,
-    'cakey': 'ca.key',
-    'cacert': 'ca.crt',
-    'certkey': 'cert.key',
+    'cakey': 'ca-cert/ca.key',
+    'cacert': 'ca-cert/ca.crt',
+    'certkey': 'ca-cert/cert.key',
     'certdir': 'certs/',
     'cacn': 'proxy2 CA',
     'plugins': set(),
@@ -105,14 +107,22 @@ def __init__(self, *args, **kwargs):
         self.options = options
         self.plugins = plugins.get_plugins()
 
-        BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+        try:
+            BaseHTTPRequestHandler.__init__(self, *args, **kwargs)
+        except Exception as e:
+            logger.dbg('Failure along __init__ of BaseHTTPRequestHandler: {}'.format(str(e)))
+            if options['debug']:
+                raise
 
     def log_error(self, format, *args):
 
         # Surpress "Request timed out: timeout('timed out',)" if not in debug mode.
         if isinstance(args[0], socket.timeout) and not self.options['debug']:
             return
 
+        if not options['trace'] or not options['debug']:
+            return
+
         self.log_message(format, *args)
 
     def do_CONNECT(self):
@@ -158,6 +168,7 @@ def connect_relay(self):
         try:
             s = socket.create_connection(address, timeout=self.options['timeout'])
         except Exception as e:
+            logger.err("Could not relay connection: ({})".format(str(e)))
             self.send_error(502)
             return
 
@@ -196,7 +207,8 @@ def do_GET(self):
                 req.path = "http://%s%s" % (req.headers['Host'], req.path)
 
 
-        (logger.dbg if self.options['trace'] else logger.info)('Request: "%s"' % req.path)
+        if options['trace'] or options['debug']:
+            (logger.dbg if self.options['trace'] else logger.info)('Request: "%s"' % req.path)
 
         req_body_modified = self.request_handler(req, req_body)
         if req_body_modified is not None:
@@ -205,25 +217,30 @@ def do_GET(self):
 
         u = urlparse.urlsplit(req.path)
         scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path)
-        assert scheme in ('http', 'https')
-        if netloc:
-            req.headers['Host'] = netloc
-        req_headers = self.filter_headers(req.headers)
 
         try:
+            assert scheme in ('http', 'https')
+            if netloc:
+                req.headers['Host'] = netloc
+            req_headers = self.filter_headers(req.headers)
+
             origin = (scheme, netloc)
             if not origin in self.tls.conns:
                 if scheme == 'https':
                     self.tls.conns[origin] = httplib.HTTPSConnection(netloc, timeout=self.options['timeout'])
                 else:
                     self.tls.conns[origin] = httplib.HTTPConnection(netloc, timeout=self.options['timeout'])
             conn = self.tls.conns[origin]
+
+            logger.dbg('Final request headers: ({})'.format(dict(req_headers)))
             conn.request(self.command, path, req_body, dict(req_headers))
             res = conn.getresponse()
             res_body = res.read()
         except Exception as e:
             if origin in self.tls.conns:
                 del self.tls.conns[origin]
+            logger.err("Could not proxy request: ({})".format(str(e)))
+            raise
             self.send_error(502)
             return
 
@@ -249,12 +266,17 @@ def do_GET(self):
         self.wfile.write(res_body)
         self.wfile.flush()
 
-        with self.lock:
-            self.save_handler(req, req_body, res, res_body_plain)
+        if options['trace'] and options['debug']:
+            with self.lock:
+                self.save_handler(req, req_body, res, res_body_plain)
 
     do_HEAD = do_GET
     do_POST = do_GET
     do_OPTIONS = do_GET
+    do_DELETE = do_GET
+    do_PUT = do_GET
+    do_TRACE = do_GET
+    do_PATCH = do_GET
 
     def filter_headers(self, headers):
         # http://tools.ietf.org/html/rfc2616#section-13.5.1
@@ -289,6 +311,12 @@ def decode_content_body(self, data, encoding):
                 text = zlib.decompress(data)
             except zlib.error:
                 text = zlib.decompress(data, -zlib.MAX_WBITS)
+        elif encoding == 'br':
+            # Brotli algorithm
+            try:
+                text = brotli.decompress(data)
+            except Exception as e:
+                raise Exception('Could not decompress Brotli stream: "{}"'.format(str(e)))
         else:
             raise Exception("Unknown Content-Encoding: %s" % encoding)
         return text
@@ -308,6 +336,9 @@ def print_info(self, req, req_body, res, res_body):
         def parse_qsl(s):
             return '\n'.join("%-20s %s" % (k, v) for k, v in urlparse.parse_qsl(s, keep_blank_values=True))
 
+        if not options['trace'] or not options['debug']:
+            return
+
         req_header_text = "%s %s %s\n%s" % (req.command, req.path, req.request_version, req.headers)
         res_header_text = "%s %d %s\n%s" % (res.response_version, res.status, res.reason, res.headers)
 
@@ -390,10 +421,18 @@ def request_handler(self, req, req_body):
             try:
                 handler = getattr(instance, 'request_handler')
                 logger.dbg("Calling `request_handler' from plugin %s" % plugin_name)
-                handler(req, req_body)
+                req_body = handler(req, req_body)
             except AttributeError as e:
-                logger.dbg('Plugin "%s" does not implement `request_handler\'')
-                pass
+                if 'object has no attribute' in str(e):
+                    logger.dbg('Plugin "{}" does not implement `request_handler\''.format(plugin_name))
+                    if options['debug']:
+                        raise
+                else:
+                    logger.err("Plugin {} has thrown an exception: '{}'".format(plugin_name, str(e)))
+                    if options['debug']:
+                        raise
+
+        return req_body
 
 
     def response_handler(self, req, req_body, res, res_body):
@@ -410,8 +449,12 @@ def response_handler(self, req, req_body, res, res_body):
                 if altered:
                     logger.dbg('Plugin has altered the response.')
             except AttributeError as e:
-                logger.dbg('Plugin "%s" does not implement `response_handler\'')
-                pass
+                if 'object has no attribute' in str(e):
+                    logger.dbg('Plugin "{}" does not implement `response_handler\''.format(plugin_name))
+                else:
+                    logger.err("Plugin {} has thrown an exception: '{}'".format(plugin_name, str(e)))
+                    if options['debug']:
+                        raise
 
         if not altered:
             return None

proxylogger.py

@@ -23,8 +23,8 @@ class ProxyLogger:
     colors_dict = {
         'error': colors_map['red'],
         'trace': colors_map['magenta'],
-        'info ': colors_map['white'],
-        'debug': colors_map['yellow'],
+        'info ': colors_map['green'],
+        'debug': colors_map['grey'],
         'other': colors_map['grey'],
     }
 
@@ -72,8 +72,9 @@ def out(txt, fd, mode='info ', **kwargs):
         fd.write(prefix + ProxyLogger.with_color(col, txt) + nl)
 
     # Info shall be used as an ordinary logging facility, for every desired output.
-    def info(self, txt, **kwargs):
-        ProxyLogger.out(txt, self.options['log'], 'info', **kwargs)
+    def info(self, txt, forced = False, **kwargs):
+        if forced or (self.options['verbose'] or self.options['debug'] or self.options['trace']):
+            ProxyLogger.out(txt, self.options['log'], 'info', **kwargs)
 
     # Trace by default does not uses [TRACE] prefix. Shall be used
     # for dumping packets, headers, metadata and longer technical output.

requirements.txt

@@ -0,0 +1 @@
+brotli