diff --git a/.travis.yml b/.travis.yml index ecc235a..99aae7b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,19 @@ +branches: + - master + language: python +dist: xenial python: - "2.7" -# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: - - pip install -r requirements-dev.txt - - pip install coveralls -# command to run tests, e.g. python setup.py test -script: - PYTHONPATH=.:$PYTHONPATH py.test --cov ./cabby -# coverage run --source=opentaxii setup.py test + - "3.5" + - "3.6" + - "3.7" + +install: + - pip install coveralls -r requirements-dev.txt + +script: + - py.test --cov cabby + after_success: - coveralls + - coveralls diff --git a/CHANGES.rst b/CHANGES.rst index d1bd844..2b4949f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,6 +1,17 @@ Changelog ========= +0.1.20 (2018-03-29) +------------------- + +* Only include relevant files in release packages. + +0.1.19 (2018-03-29) +------------------- + +* Enable client key passphrase for JWT token authentication request + (`pr#50 `_) + 0.1.18 (2017-06-19) ------------------- * Dependencies upgraded (`changes `_). diff --git a/Dockerfile b/Dockerfile index a99416d..1c02927 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,62 +1,12 @@ -# Set the base image to Python -FROM alpine:3.4 -MAINTAINER EclecticIQ - -# Volume for possible input -VOLUME [ "/input" ] - -# Create the working dir and set the working directory -WORKDIR / - -# Setup Python -RUN apk add --no-cache -U \ - ca-certificates \ - build-base \ - libxml2 \ - libxml2-dev \ - libxslt \ - libxslt-dev \ - make \ - python-dev \ - py-pip \ - python \ - && pip install --upgrade pip setuptools - - -# Setup Requirements -COPY ./ /cabby -RUN pip install -r /cabby/requirements.txt \ - && cd /cabby \ - && python setup.py install - -# Cleanup -RUN apk del build-base \ - libxml2-dev \ - libxslt-dev \ - python-dev \ - build-base \ - && rm -rf /var/cache/apk/* \ - && rm -r /root/.cache \ - && rm -f requirements.txt \ - && rm -rf /cabby - -RUN { echo '#!/bin/sh';\ - echo 'echo "';\ - echo ' Commands to be run:';\ - echo ' taxii-discovery ';\ - echo ' taxii-poll';\ - echo ' taxii-collections';\ - echo ' taxii-push ';\ - echo ' taxii-subscription';\ - echo ' taxii-proxy';\ - echo '';\ - echo 'e.g. docker run -ti eclecticiq/cabby taxii-discovery --path https://test.taxiistand.com/read-write/services/discovery';\ - echo '';\ - echo 'More information available at: http://cabby.readthedocs.org';\ - echo 'Or you can choose to drop back into a shell by providing: bash as the command:';\ - echo '';\ - echo 'docker run -ti cabby bash"'; } > /help.sh && chmod 750 /help.sh - -# Give help, unless command is given -CMD [ "/help.sh" ] - +FROM python:3-slim-stretch +LABEL maintainer="EclecticIQ " +RUN python3 -m venv --system-site-packages /venv +ENV PATH=/venv/bin:$PATH + +COPY ./requirements.txt ./requirements-dev.txt /cabby/ +RUN pip install -r /cabby/requirements-dev.txt +COPY . /cabby +RUN pip install -e /cabby + +RUN sh -c "cat /cabby/docker-help.sh >> /root/.bashrc" +CMD ["/cabby/docker-help.sh"] diff --git a/MANIFEST.in b/MANIFEST.in index e273b19..7796a18 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,5 @@ include LICENSE.rst include CHANGES.rst include README.rst include requirements.txt -recursive-include docs * +recursive-include docs *.rst global-exclude *.pyc diff --git a/README.rst b/README.rst index 7c0e77d..439a414 100644 --- a/README.rst +++ b/README.rst @@ -3,29 +3,26 @@ cabby Python TAXII client implementation from `EclecticIQ `_. -:Source: https://github.com/EclecticIQ/cabby +:Source: https://github.com/eclecticiq/cabby :Documentation: http://cabby.readthedocs.org :Information: https://www.eclecticiq.com :Download: https://pypi.python.org/pypi/cabby/ -|travis badge| |landscape.io badge| |coveralls.io badge| |docs badge| |requirements badge| +|travis badge| |coveralls.io badge| |docs badge| |requirements badge| -.. |travis badge| image:: https://travis-ci.org/EclecticIQ/cabby.svg?branch=master - :target: https://travis-ci.org/EclecticIQ/cabby +.. |travis badge| image:: https://travis-ci.org/eclecticiq/cabby.svg?branch=master + :target: https://travis-ci.org/eclecticiq/cabby :alt: Build Status -.. |landscape.io badge| image:: https://landscape.io/github/EclecticIQ/cabby/master/landscape.svg?style=flat - :target: https://landscape.io/github/EclecticIQ/cabby/master - :alt: Code Health -.. |coveralls.io badge| image:: https://coveralls.io/repos/EclecticIQ/cabby/badge.svg - :target: https://coveralls.io/r/EclecticIQ/cabby +.. |coveralls.io badge| image:: https://coveralls.io/repos/eclecticiq/cabby/badge.svg + :target: https://coveralls.io/r/eclecticiq/cabby :alt: Coverage Status .. |docs badge| image:: https://readthedocs.org/projects/cabby/badge/?version=latest - :alt: Documentation Status - :scale: 100% - :target: https://readthedocs.org/projects/cabby/ -.. |requirements badge| image:: https://requires.io/github/EclecticIQ/cabby/requirements.svg?branch=master - :target: https://requires.io/github/EclecticIQ/cabby/requirements/?branch=master - :alt: Requirements Status + :alt: Documentation Status + :scale: 100% + :target: https://readthedocs.org/projects/cabby/ +.. |requirements badge| image:: https://requires.io/github/eclecticiq/cabby/requirements.svg?branch=master + :target: https://requires.io/github/eclecticiq/cabby/requirements/?branch=master + :alt: Requirements Status A simple Python library for interacting with TAXII servers. @@ -33,11 +30,9 @@ A simple Python library for interacting with TAXII servers. Docker -------- -From version 0.1.13, the docker image is based on 'Alpine' linux. This means the size of the Image was reduced from 311MB to 74MB - To run cabby using docker, execute the following: - docker run --rm=true eclecticiq/cabby:latest taxii-discovery --path https://test.taxiistand.com/read-only/services/discovery + docker run --rm eclecticiq/cabby taxii-discovery --path https://test.taxiistand.com/read-only/services/discovery Feedback -------- diff --git a/cabby/_version.py b/cabby/_version.py index 5cbfc84..a3561a3 100644 --- a/cabby/_version.py +++ b/cabby/_version.py @@ -3,4 +3,4 @@ This module defines the package version for use in __init__.py and setup.py. """ -__version__ = '0.1.19a1' +__version__ = '0.1.20' diff --git a/cabby/abstract.py b/cabby/abstract.py index 4577a61..dc6a953 100644 --- a/cabby/abstract.py +++ b/cabby/abstract.py @@ -152,7 +152,9 @@ def prepare_generic_session(self): password=self.password if not self.jwt_url else None, cert_file=self.cert_file, key_file=self.key_file, - verify_ssl=(self.ca_cert or self.verify_ssl)) + key_password=self.key_password, + ca_cert=self.ca_cert, + verify_ssl=self.verify_ssl) def _execute_request(self, request, uri=None, service_type=None): ''' @@ -161,7 +163,6 @@ def _execute_request(self, request, uri=None, service_type=None): A service is defined by ``uri`` parameter or is chosen from pre-cached services by ``service_type``. ''' - if not uri and not service_type: raise NoURIProvidedError('URI or service_type needed') elif not uri: @@ -181,28 +182,12 @@ def _execute_request(self, request, uri=None, service_type=None): self.refresh_jwt_token(session=session) session = dispatcher.set_jwt_token(session, self.jwt_token) - if self.key_password: - # If key_password is provided - message = dispatcher.send_taxii_request( - session, - self._prepare_url(uri), - request, - taxii_binding=self.taxii_binding, - # Details in case key_password is provided - tls_details={ - 'cert_file': self.cert_file, - 'key_file': self.key_file, - 'key_password': self.key_password, - 'ca_cert': self.ca_cert - }, - timeout=self.timeout) - else: - message = dispatcher.send_taxii_request( - session, - self._prepare_url(uri), - request, - taxii_binding=self.taxii_binding, - timeout=self.timeout) + message = dispatcher.send_taxii_request( + session, + self._prepare_url(uri), + request, + taxii_binding=self.taxii_binding, + timeout=self.timeout) return message diff --git a/cabby/dispatcher.py b/cabby/dispatcher.py index b4dff46..16d38fe 100644 --- a/cabby/dispatcher.py +++ b/cabby/dispatcher.py @@ -1,4 +1,5 @@ from collections import namedtuple +import json import os import ssl import sys @@ -33,8 +34,8 @@ def raise_http_error(status_code, response_stream=None): raise HTTPError(status_code) -def send_taxii_request(session, url, request, taxii_binding=None, - tls_details=None, timeout=None): +def send_taxii_request( + session, url, request, taxii_binding=None, timeout=None): ''' Send XML message to a TAXII service and parse a response. ''' @@ -50,12 +51,28 @@ def send_taxii_request(session, url, request, taxii_binding=None, url_scheme=furl.furl(url).scheme, message_binding=taxii_binding) - if tls_details and tls_details.get('key_password'): + stream, headers = request_stream(session, url, request_body, timeout) + + gen = _parse_response(stream, headers, version=request.version) + obj = next(gen) + + if obj == const.STREAM_MARKER: + return gen + elif hasattr(obj, 'status_type'): + if obj.status_type != 'SUCCESS': + raise UnsuccessfulStatusError(obj) + else: + return None + return obj + + +def request_stream(session, url, request_body, timeout, headers=None): + if session._cabby_key_password: # Workaround until # https://github.com/kennethreitz/requests/issues/2519 is fixed try: - response = get_response_using_key_pass( - url, request_body, session, timeout=timeout, **tls_details) + response = request_with_key_password( + session, url, request_body, timeout, headers) except urllib.error.HTTPError as e: log.error( "Error while connecting to {}".format(url), @@ -64,8 +81,12 @@ def send_taxii_request(session, url, request, taxii_binding=None, stream, headers = response, response.headers else: - response = session.post(url, data=request_body, stream=True, - timeout=timeout) + response = session.post( + url, + data=request_body, + stream=True, + timeout=timeout, + headers=headers) if not response.ok: raise_http_error(response.status_code, response.raw) @@ -81,17 +102,7 @@ def send_taxii_request(session, url, request, taxii_binding=None, stream = gzip.GzipFile(fileobj=stream) - gen = _parse_response(stream, headers, version=request.version) - obj = next(gen) - - if obj == const.STREAM_MARKER: - return gen - elif hasattr(obj, 'status_type'): - if obj.status_type != 'SUCCESS': - raise UnsuccessfulStatusError(obj) - else: - return None - return obj + return stream, headers def _cleanup_batch(curr_elem, batch): @@ -292,28 +303,32 @@ def __call__(self, r): return r -def get_generic_session(proxies=None, headers=None, - username=None, password=None, - cert_file=None, key_file=None, - verify_ssl=True): +def get_generic_session( + proxies=None, + headers=None, + username=None, + password=None, + cert_file=None, + key_file=None, + key_password=None, + ca_cert=None, + verify_ssl=True): session = requests.Session() - session.verify = verify_ssl - + if ca_cert: + session.verify = ca_cert + else: + session.verify = verify_ssl if proxies: session.proxies = proxies - if headers: session.headers.update(headers) - + session.headers['User-Agent'] = 'Cabby {}'.format(cabby_version) if username and password: session.auth = HTTPBasicAuth(username, password) - - session.headers['User-Agent'] = 'Cabby {}'.format(cabby_version) - if cert_file and key_file: session.cert = (cert_file, key_file) - + session._cabby_key_password = key_password return session @@ -351,27 +366,26 @@ def get_taxii_session(session, url_scheme='https', content_type=None, return session -def obtain_jwt_token(session, jwt_url, username, password): +def obtain_jwt_token(session, jwt_url, username, password, timeout=None): log.info("Obtaining JWT token from {}".format(jwt_url)) - response = session.post(jwt_url, json={ - 'username': username, - 'password': password - }) + request_data = json.dumps({'username': username, 'password': password}) + request_body = request_data.encode('utf-8') + headers = {'Content-Type': 'application/json'} - if not response.ok: - raise_http_error(response.status_code, response.raw) + stream, headers = request_stream( + session, jwt_url, request_body, timeout, headers) + response_body = stream.read().decode('utf-8') + response_data = json.loads(response_body) - body = response.json() - if 'token' not in body: - log.debug("Incorrect JWT response:\n{}".format(body)) + if 'token' not in response_data: + log.debug("Incorrect JWT response:\n{}".format(response_body)) raise ValueError("No token found in JWT auth response") - return body['token'] - + return response_data['token'] -def get_response_using_key_pass(url, data, session, cert_file, key_file, - key_password, ca_cert=None, timeout=None): +def request_with_key_password( + session, url, request_body, timeout=None, headers=None): if sys.version_info < (2, 7, 9): raise ValueError( 'Key password specification is not supported in Python < v2.7.9') @@ -379,32 +393,40 @@ def get_response_using_key_pass(url, data, session, cert_file, key_file, if session.auth: # Using Requests Session's auth handlers to fill in proper headers DummyRequest = namedtuple('DummyRequest', ['headers']) - headers = session.auth(DummyRequest(headers=session.headers)).headers + request_headers = session.auth( + DummyRequest(headers=session.headers)).headers else: - headers = session.headers + request_headers = session.headers + if headers: + request_headers.update(headers) + + # Take the TLS details from the session object and use them with urllib. + # See also 'get_generic_session' which sets many of these attributes. + # session 'verify' attribute can be a bool or a path to a CA bundle: + ca_cert = None + if not isinstance(session.verify, bool): + ca_cert = session.verify context = ssl.create_default_context( ssl.Purpose.CLIENT_AUTH, cafile=ca_cert) + cert_file, key_file = session.cert + key_password = session._cabby_key_password context.load_cert_chain(cert_file, key_file, password=key_password) - if not session.verify and not ca_cert: - context.verify_mode = ssl.CERT_NONE - elif session.verify: + if session.verify: context.verify_mode = ssl.CERT_REQUIRED - if not ca_cert: context.set_default_verify_paths() + else: + context.verify_mode = ssl.CERT_NONE handlers = [urllib.request.HTTPSHandler(context=context)] - if session.proxies: - handlers.append( - urllib.request.ProxyHandler(session.proxies)) + handlers.append(urllib.request.ProxyHandler(session.proxies)) opener = urllib.request.build_opener(*handlers) - - request = urllib.request.Request(url, data, headers) + request = urllib.request.Request(url, request_body, request_headers) if timeout: return opener.open(request, timeout=timeout) diff --git a/cabby/exceptions.py b/cabby/exceptions.py index 9133aba..9aca3ad 100644 --- a/cabby/exceptions.py +++ b/cabby/exceptions.py @@ -18,12 +18,11 @@ class InvalidResponseError(ClientException): class UnsuccessfulStatusError(ClientException): def __init__(self, taxii_status, *args, **kwargs): - super(UnsuccessfulStatusError, self).__init__( - _status_to_message(taxii_status), *args, **kwargs) + msg = "Server Error: {}".format(_status_to_message(self.raw)) + super(UnsuccessfulStatusError, self).__init__(msg, *args, **kwargs) self.status = taxii_status.status_type self.text = taxii_status.to_text() - self.raw = taxii_status diff --git a/docker-help.sh b/docker-help.sh new file mode 100755 index 0000000..c30d058 --- /dev/null +++ b/docker-help.sh @@ -0,0 +1,33 @@ +#!/bin/sh + +_cmd="" +if [ $0 = '/cabby/docker-help.sh' ]; then + _cmd="docker run --rm eclecticiq/cabby " +fi + +cat <