diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..04529356 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*.py] +charset = utf-8 +indent_style = space +indent_size = 4 +insert_final_newline = true +end_of_line = lf + +[*.{yaml,yml}] +indent_size = 2 diff --git a/.gitignore b/.gitignore index 1f7b029b..ef2ccce3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,5 @@ __pycache__ .coverage .tox/ .cache/ + +/hyper/version.py diff --git a/.travis/install.sh b/.travis/install.sh index d7423ce7..56bfca2d 100755 --- a/.travis/install.sh +++ b/.travis/install.sh @@ -7,7 +7,9 @@ if [[ "$HYPER_FAST_PARSE" = true ]]; then pip install pycohttpparser~=1.0 fi -pip install -U setuptools -pip install . +pip install -U pip +pip install -U setuptools wheel build +python3 -m build -nwx . +pip install --upgrade ./dist/*.whl pip install -r test_requirements.txt pip install flake8 diff --git a/LICENSE b/LICENSE index 7ef4aca0..a351a2a2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,7 @@ The MIT License (MIT) Copyright (c) 2014 Cory Benfield, Google Inc +Copyright (c) 2008-2020 Andrey Petrov and urllib3 contributors (see https://github.com/urllib3/urllib3/blob/master/CONTRIBUTORS.txt) (socks5 code was borrowed from there) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.rst b/README.rst index 99dce29d..9a158be5 100644 --- a/README.rst +++ b/README.rst @@ -4,6 +4,11 @@ Hyper: HTTP/2 Client for Python **This project is no longer maintained!** +@Lukasa has long dropped its maintainment. +The last maintainer was @Kriechi. He now has an own project, `HTTPX`_, that also supports HTTP/2 and is partially based on libs that were used in hyper, and now he develops it. + +He has left the message + Please use an alternative, such as `HTTPX`_ or others. .. _HTTPX: https://www.python-httpx.org/ @@ -18,6 +23,12 @@ Potential security issues will not be addressed. ---- +and archived the project. + +This fork is "maintained" by @KOLANICH. I don't really develop it, just add the stuff I need personally. + + + .. image:: https://raw.github.com/Lukasa/hyper/development/docs/source/images/hyper.png HTTP is changing under our feet. HTTP/1.1, our old friend, is being diff --git a/hyper/__init__.py b/hyper/__init__.py index 99044881..73a7caaa 100644 --- a/hyper/__init__.py +++ b/hyper/__init__.py @@ -33,4 +33,4 @@ # Set default logging handler. logging.getLogger(__name__).addHandler(logging.NullHandler()) -__version__ = '0.8.0dev0' +from .version import __version__ diff --git a/hyper/common/connection.py b/hyper/common/connection.py index 855994f8..81a81c04 100644 --- a/hyper/common/connection.py +++ b/hyper/common/connection.py @@ -46,6 +46,7 @@ class HTTPConnection(object): :param proxy_port: (optional) The proxy port to connect to. If not provided and one also isn't provided in the ``proxy_host`` parameter, defaults to 8080. + :param proxy_type: (optional) type of the proxy to use. Allows usage of socks proxies :param proxy_headers: (optional) The headers to send to a proxy. """ def __init__(self, @@ -57,6 +58,7 @@ def __init__(self, ssl_context=None, proxy_host=None, proxy_port=None, + proxy_type=None, proxy_headers=None, timeout=None, **kwargs): @@ -66,14 +68,14 @@ def __init__(self, self._h1_kwargs = { 'secure': secure, 'ssl_context': ssl_context, 'proxy_host': proxy_host, 'proxy_port': proxy_port, - 'proxy_headers': proxy_headers, 'enable_push': enable_push, + 'proxy_headers': proxy_headers, "proxy_type": proxy_type, 'enable_push': enable_push, 'timeout': timeout } self._h2_kwargs = { 'window_manager': window_manager, 'enable_push': enable_push, 'secure': secure, 'ssl_context': ssl_context, 'proxy_host': proxy_host, 'proxy_port': proxy_port, - 'proxy_headers': proxy_headers, + 'proxy_headers': proxy_headers, "proxy_type": proxy_type, 'timeout': timeout } @@ -150,6 +152,17 @@ def get_response(self, *args, **kwargs): return self._conn.get_response(1) + def reanimate(self): + """Reanimate connection reset because of proxy""" + if hasattr(self, "streams"): + for stream in list(self.streams.values()): + stream.remote_closed = True + stream.local_closed = True + self._conn.close() + self._conn = HTTP11Connection( + self._host, self._port, **self._h1_kwargs, + ) + # The following two methods are the implementation of the context manager # protocol. def __enter__(self): # pragma: no cover diff --git a/hyper/contrib.py b/hyper/contrib.py index 79aa7d12..e1abf0b3 100644 --- a/hyper/contrib.py +++ b/hyper/contrib.py @@ -56,11 +56,24 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, ssl_context = init_context(cert_path=verify, cert=cert) if proxy: + proxy = prepend_scheme_if_needed(proxy, 'http') proxy_headers = self.proxy_headers(proxy) - proxy_netloc = urlparse(proxy).netloc + parsed = urlparse(proxy) + proxy_netloc = parsed.netloc + proxy_host_port = proxy_netloc.split(":") + if len(proxy_host_port) == 2: + proxy_host, proxy_port = proxy_host_port + proxy_port = int(proxy_port) + elif len(proxy_port_host) == 1: + raise ValueError("Specify proxy port!") + else: + raise ValueError("Invalid proxy netloc: ", repr(proxy_netloc)) + proxy_type = parsed.scheme else: proxy_headers = None - proxy_netloc = None + proxy_host = None + proxy_port = None + proxy_type = None # We put proxy headers in the connection_key, because # ``proxy_headers`` method might be overridden, so we can't @@ -68,7 +81,7 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, proxy_headers_key = (frozenset(proxy_headers.items()) if proxy_headers else None) connection_key = (host, port, scheme, cert, verify, - proxy_netloc, proxy_headers_key) + proxy_host, proxy_port, proxy_type, proxy_headers_key) try: conn = self.connections[connection_key] except KeyError: @@ -78,9 +91,13 @@ def get_connection(self, host, port, scheme, cert=None, verify=True, secure=secure, window_manager=self.window_manager, ssl_context=ssl_context, - proxy_host=proxy_netloc, + proxy_host=proxy_host, + proxy_port=proxy_port, proxy_headers=proxy_headers, - timeout=timeout) + proxy_type=proxy_type, + timeout=timeout, + enable_push=self.enable_push + ) self.connections[connection_key] = conn return conn @@ -95,6 +112,12 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, proxy = prepend_scheme_if_needed(proxy, 'http') parsed = urlparse(request.url) + + # Build the selector. + selector = parsed.path + selector += '?' + parsed.query if parsed.query else '' + selector += '#' + parsed.fragment if parsed.fragment else '' + conn = self.get_connection( parsed.hostname, parsed.port, @@ -104,18 +127,27 @@ def send(self, request, stream=False, cert=None, verify=True, proxies=None, proxy=proxy, timeout=timeout) - # Build the selector. - selector = parsed.path - selector += '?' + parsed.query if parsed.query else '' - selector += '#' + parsed.fragment if parsed.fragment else '' - - conn.request( - request.method, - selector, - request.body, - request.headers - ) - resp = conn.get_response() + def do_req(): + conn.request( + request.method, + selector, + request.body, + request.headers + ) + resp = conn.get_response() + return conn, resp + + retried = 0 + max_retries = 1 + while True: + try: + conn, resp = do_req() + break + except ConnectionAbortedError as e: + if retried < max_retries: + conn.reanimate() + else: + raise r = self.build_response(request, resp) diff --git a/hyper/http11/connection.py b/hyper/http11/connection.py index 4311d307..cba86b33 100644 --- a/hyper/http11/connection.py +++ b/hyper/http11/connection.py @@ -10,7 +10,7 @@ import socket import base64 -from collections import Iterable, Mapping +from collections.abc import Iterable, Mapping import collections from hyperframe.frame import SettingsFrame @@ -101,7 +101,7 @@ class HTTP11Connection(object): def __init__(self, host, port=None, secure=None, ssl_context=None, proxy_host=None, proxy_port=None, proxy_headers=None, - timeout=None, **kwargs): + proxy_type=None, timeout=None, **kwargs): if port is None: self.host, self.port = to_host_port_tuple(host, default_port=80) else: @@ -133,11 +133,14 @@ def __init__(self, host, port=None, secure=None, ssl_context=None, self.proxy_host, self.proxy_port = to_host_port_tuple( proxy_host, default_port=8080 ) + self.proxy_type = proxy_type elif proxy_host: - self.proxy_host, self.proxy_port = proxy_host, proxy_port + self.proxy_host, self.proxy_port, self.proxy_type = proxy_host, proxy_port, proxy_type else: self.proxy_host = None self.proxy_port = None + self.proxy_type = None + raise ValueError("No proxy was set!") self.proxy_headers = proxy_headers #: The size of the in-memory buffer used to store data from the @@ -169,22 +172,48 @@ def connect(self): connect_timeout = self._timeout read_timeout = self._timeout - if self.proxy_host and self.secure: - # Send http CONNECT method to a proxy and acquire the socket - sock = _create_tunnel( - self.proxy_host, - self.proxy_port, - self.host, - self.port, - proxy_headers=self.proxy_headers, - timeout=self._timeout - ) - elif self.proxy_host: - # Simple http proxy - sock = socket.create_connection( - (self.proxy_host, self.proxy_port), - timeout=connect_timeout - ) + if self.proxy_host: + if self.proxy_type.startswith("socks"): + import socks + rdns = (self.proxy_type[-1]=="h") + # any error will result in silently connecting without a proxy. + # IDK why it is done this way + if not rdns: + raise ValueError("RDNS is disabled. Proxying dns queries is disabled. NSA is spying you.") + if rdns and self.proxy_type.startswith("socks4"): + raise ValueError("RDNS is not supported for socks4. socks.create_connection ignores it silently.") + if not isinstance(self.proxy_host, str): + raise ValueError("self.proxy_host", repr(self.proxy_host), "is not str") + if not isinstance(self.proxy_port, int): + raise ValueError("self.proxy_port", repr(self.proxy_port), "is not int") + socks_version_char = self.proxy_type[5] + sock = socks.create_connection( + (self.host, self.port), + proxy_type=getattr(socks, "PROXY_TYPE_SOCKS" + socks_version_char), + proxy_addr=self.proxy_host, + proxy_port=self.proxy_port, + #proxy_username=username, + #proxy_password=password, + proxy_rdns=rdns, + ) + elif self.proxy_host and self.secure: + # Send http CONNECT method to a proxy and acquire the socket + sock = _create_tunnel( + self.proxy_host, + self.proxy_port, + self.host, + self.port, + proxy_headers=self.proxy_headers, + timeout=self._timeout + ) + elif self.proxy_host: + # Simple http proxy + sock = socket.create_connection( + (self.proxy_host, self.proxy_port), + timeout=connect_timeout + ) + else: + raise Exception("Unsupported proxy type: "+repr(proxy_type)) else: sock = socket.create_connection((self.host, self.port), timeout=connect_timeout) @@ -454,6 +483,7 @@ def _send_file_like_obj(self, fobj): return + def close(self): """ Closes the connection. This closes the socket and then abandons the diff --git a/hyper/http20/connection.py b/hyper/http20/connection.py index b8be292b..0d2981fa 100644 --- a/hyper/http20/connection.py +++ b/hyper/http20/connection.py @@ -101,7 +101,7 @@ class HTTP20Connection(object): def __init__(self, host, port=None, secure=None, window_manager=None, enable_push=False, ssl_context=None, proxy_host=None, - proxy_port=None, force_proto=None, proxy_headers=None, + proxy_port=None, proxy_type=None, force_proto=None, proxy_headers=None, timeout=None, **kwargs): """ Creates an HTTP/2 connection to a specific server. @@ -126,11 +126,13 @@ def __init__(self, host, port=None, secure=None, window_manager=None, self.proxy_host, self.proxy_port = to_host_port_tuple( proxy_host, default_port=8080 ) + self.proxy_type = proxy_type elif proxy_host: - self.proxy_host, self.proxy_port = proxy_host, proxy_port + self.proxy_host, self.proxy_port, self.proxy_type = proxy_host, proxy_port, proxy_type else: self.proxy_host = None self.proxy_port = None + self.proxy_type = None self.proxy_headers = proxy_headers #: The size of the in-memory buffer used to store data from the @@ -353,22 +355,49 @@ def connect(self): connect_timeout = self._timeout read_timeout = self._timeout - if self.proxy_host and self.secure: - # Send http CONNECT method to a proxy and acquire the socket - sock = _create_tunnel( - self.proxy_host, - self.proxy_port, - self.host, - self.port, - proxy_headers=self.proxy_headers, - timeout=self._timeout - ) - elif self.proxy_host: - # Simple http proxy - sock = socket.create_connection( - (self.proxy_host, self.proxy_port), - timeout=connect_timeout - ) + if self.proxy_host: + if self.proxy_type.startswith("socks"): + import socks + rdns = (self.proxy_type[-1]=="h") + # any error will result in silently connecting without a proxy. + # IDK why it is done this way. + if not rdns: + raise ValueError("RDNS is disabled. Proxying dns queries is disabled. NSA is spying you.") + if rdns and self.proxy_type.startswith("socks4"): + raise ValueError("RDNS is not supported for socks4. socks.create_connection ignores it silently.") + if not isinstance(self.proxy_host, str): + raise ValueError("self.proxy_host", repr(self.proxy_host), "is not str") + if not isinstance(self.proxy_port, int): + raise ValueError("self.proxy_port", repr(self.proxy_port), "is not int") + socks_version_char = self.proxy_type[5] + sock = socks.create_connection( + (self.host, self.port), + proxy_type=getattr(socks, "PROXY_TYPE_SOCKS" + socks_version_char), + proxy_addr=self.proxy_host, + proxy_port=self.proxy_port, + #proxy_username=username, + #proxy_password=password, + proxy_rdns=rdns, + ) + #sock.getsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) + elif self.proxy_host and self.secure: + # Send http CONNECT method to a proxy and acquire the socket + sock = _create_tunnel( + self.proxy_host, + self.proxy_port, + self.host, + self.port, + proxy_headers=self.proxy_headers, + timeout=self._timeout + ) + elif self.proxy_host: + # Simple http proxy + sock = socket.create_connection( + (self.proxy_host, self.proxy_port), + timeout=connect_timeout + ) + else: + raise Exception("Unsupported proxy type: "+repr(proxy_type)) else: sock = socket.create_connection((self.host, self.port), timeout=connect_timeout) @@ -403,7 +432,7 @@ def _connect_upgrade(self, sock): with self._conn as conn: conn.initiate_upgrade_connection() conn.update_settings( - {h2.settings.ENABLE_PUSH: int(self._enable_push)} + {h2.settings.SettingCodes.ENABLE_PUSH: int(self._enable_push)} ) self._send_outstanding_data() @@ -424,7 +453,7 @@ def _send_preamble(self): with self._conn as conn: conn.initiate_connection() conn.update_settings( - {h2.settings.ENABLE_PUSH: int(self._enable_push)} + {h2.settings.SettingCodes.ENABLE_PUSH: int(self._enable_push)} ) self._send_outstanding_data() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..63f88577 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = ["setuptools>=44", "wheel", "setuptools_scm[toml]>=3.4.3"] +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] +write_to = "hyper/version.py" +write_to_template = "__version__ = '{version}'\n" diff --git a/setup.cfg b/setup.cfg index 53d397a8..3edc5385 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,55 @@ +[metadata] +name = hyper +version = attr: setup.__version__ +author = Cory Benfield +author_email = cory@lukasa.co.uk +license = MIT License +description = HTTP/2 Client for Python +long_description = file: README.rst, HISTORY.rst +url = http://hyper.rtfd.org +classifiers = + Development Status :: 3 - Alpha + Intended Audience :: Developers + License :: OSI Approved :: MIT License + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: Implementation :: CPython + +[options] +python_requires = >= 3.4 +packages = + hyper + hyper.http20 + hyper.common + hyper.http11 + +install_requires = + h2>=2.4 # git+https://github.com/python-hyper/h2.git + hyperframe>=3.2 # git+https://github.com/python-hyper/hyperframe.git + rfc3986>=1.1.0 # git+https://github.com/python-hyper/rfc3986.git + brotlipy>=0.7.0 # git+https://github.com/python-hyper/brotlicffi.git + +include_package_data = True +tests_require = pytest; requests; mock + +[options.entry_points] +console_scripts = hyper = hyper.cli:main + +[options.extras_require] +fast = pycohttpparser + +[options.package_data] +"" = + LICENSE + README.rst + CONTRIBUTORS.rst + HISTORY.rst + NOTICES + [wheel] universal = 1 diff --git a/setup.py b/setup.py deleted file mode 100644 index 94cd8d21..00000000 --- a/setup.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import os -import re -import sys - -from setuptools import setup -from setuptools.command.test import test as TestCommand - - -class PyTest(TestCommand): - user_options = [('pytest-args=', 'a', "Arguments to pass to py.test")] - - def initialize_options(self): - TestCommand.initialize_options(self) - self.pytest_args = ['test/'] - - def finalize_options(self): - TestCommand.finalize_options(self) - self.test_args = [] - self.test_suite = True - - def run_tests(self): - # import here, cause outside the eggs aren't loaded - import pytest - errno = pytest.main(self.pytest_args) - sys.exit(errno) - - -# Get the version -version_regex = r'__version__ = ["\']([^"\']*)["\']' -with open('hyper/__init__.py', 'r') as f: - text = f.read() - match = re.search(version_regex, text) - - if match: - version = match.group(1) - else: - raise RuntimeError("No version number found!") - -# Stealing this from Kenneth Reitz -if sys.argv[-1] == 'publish': - os.system('python setup.py sdist upload') - sys.exit() - - -packages = [ - 'hyper', - 'hyper.http20', - 'hyper.common', - 'hyper.http11', -] - -setup( - name='hyper', - version=version, - description='HTTP/2 Client for Python', - long_description=open('README.rst').read() + '\n\n' + open('HISTORY.rst').read(), - author='Cory Benfield', - author_email='cory@lukasa.co.uk', - url='http://hyper.rtfd.org', - packages=packages, - package_data={'': ['LICENSE', 'README.rst', 'CONTRIBUTORS.rst', 'HISTORY.rst', 'NOTICES']}, - package_dir={'hyper': 'hyper'}, - include_package_data=True, - license='MIT License', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.4', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: Implementation :: CPython', - ], - install_requires=[ - 'h2>=2.4,<3.0,!=2.5.0', 'hyperframe>=3.2,<4.0', 'rfc3986>=1.1.0,<2.0', 'brotlipy>=0.7.0,<1.0' - ], - tests_require=['pytest', 'requests', 'mock'], - cmdclass={'test': PyTest}, - entry_points={ - 'console_scripts': [ - 'hyper = hyper.cli:main', - ], - }, - extras_require={ - 'fast': ['pycohttpparser'], - # Fallback to good SSL on bad Python versions. - ':python_full_version < "2.7.9"': [ - 'pyOpenSSL>=0.15', 'service_identity>=14.0.0' - ], - # PyPy with bad SSL modules will likely also need the cryptography - # module at lower than 1.0, because it doesn't support CFFI v1.0 yet. - ':platform_python_implementation == "PyPy" and python_full_version < "2.7.9"': [ - 'cryptography<1.0' - ], - ':python_version == "2.7" or python_version == "3.3"': ['enum34>=1.0.4, <2'] - } -)