From 68391d630f9e992d040c3834aae6bd98b19130e0 Mon Sep 17 00:00:00 2001 From: Russell Jones Date: Tue, 16 Sep 2014 23:43:08 +0000 Subject: [PATCH] Initial commit. --- MANIFEST.in | 4 + README.rst | 148 ++++++++++++++++++++ lemma.py | 301 ++++++++++++++++++++++++++++++++++++++++ setup.py | 29 ++++ tests/__init__.py | 19 +++ tests/fixtures/test.key | 1 + tests/lemma_test.py | 244 ++++++++++++++++++++++++++++++++ 7 files changed, 746 insertions(+) create mode 100644 MANIFEST.in create mode 100644 README.rst create mode 100644 lemma.py create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/test.key create mode 100644 tests/lemma_test.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..43f48f0 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include tests *.py +include LICENSE +include MANIFEST.in +include README.rst diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..1a1b050 --- /dev/null +++ b/README.rst @@ -0,0 +1,148 @@ +******* +pylemma +******* + +Mailgun Cryptographic Tools + +.. code-block:: py + + Warning! Still being actively developed and not ready for production use! + +**Overview** + +An keyed-hash message authentication code (HMAC) is used to provide integrity and +authenticity of a message between web services. The following elements are input +into the HMAC. Only the items in bold are required to be passed in by the user, the +other elements are either optional or build by pylemma for you. + +* **Shared secret**, a randomly generated number from a CSPRNG. +* Timestamp in epoch time (number of seconds since January 1, 1970 UTC). +* Nonce, a randomly generated number from a CSPRNG. +* **Request body.** +* Optionally the HTTP Verb and HTTP Request URI. +* Optionally an additional headers to sign. + +Each request element is delimited with the character `|` and each request element is +preceded by it's length. A simple example with only the required parameters: + +.. code-block:: py + + shared_secret = '042DAD12E0BE4625AC0B2C3F7172DBA8' + timestamp = '1330837567' + nonce = '000102030405060708090a0b0c0d0e0f' + request_body = '{"hello": "world"}' + + signature = HMAC('042DAD12E0BE4625AC0B2C3F7172DBA8', + '10|1330837567|32|000102030405060708090a0b0c0d0e0f|18|{"hello": "world"}') + +The timestamp, nonce, signature, and signature version are set as headers for the +HTTP request to be signed. They are then verified on receiving side by running the +same algorithm and verifying that the signatures match. + +Note: By default the service can securely handle authenticating 5,000 requests per +second. If you need to authenticate more, increase the capacity of the nonce +cache when initializing the package. + +**Examples** + +*Signing a Request* + +.. code-block:: py + + import lemma + import requests + + lemma.initialize('/path/to/key.file') + + [...] + + # sign request + request_body = '{"hello": "world"}' + timestamp, nonce, signature, signature_version = \ + lemma.sign_request(request_body) + + # build and submit request + request_headers = { + 'Content-type': 'application/json', + 'X-Mailgun-Timestamp': timestamp, + 'X-Mailgun-Nonce': nonce, + 'X-Mailgun-Signature': signature, + 'X-Mailgun-Signature-Version': signature_version} + requests.post(url, headers=request_headers, data=request_body) + +*Signing a Request with Headers* + +.. code-block:: py + + import lemma + import requests + + lemma.initialize('/path/to/key.file') + + [...] + + # sign request + request_body = '{"hello": "world"}' + timestamp, nonce, signature, signature_version = \ + lemma.sign_request(request_body, headers={'X-Mailgun-Header': 'foobar'}) + + # build and submit request + request_headers = { + 'Content-type': 'application/json', + 'X-Mailgun-Header': 'foobar' + 'X-Mailgun-Timestamp': timestamp, + 'X-Mailgun-Nonce': nonce, + 'X-Mailgun-Signature': signature, + 'X-Mailgun-Signature-Version': signature_version} + requests.post(url, headers=request_headers, data=request_body) + +*Signing a Request with HTTP Verb and URI* + +.. code-block:: py + + import lemma + import requests + + lemma.initialize('/path/to/key.file') + + [...] + + # sign request + request_body = '{"hello": "world"}' + timestamp, nonce, signature, signature_version = \ + lemma.sign_request(request_body, + http_verb='GET', http_request_uri='/path?key=value#fragment') + + # build and submit request + request_headers = { + 'Content-type': 'application/json', + 'X-Mailgun-Timestamp': timestamp, + 'X-Mailgun-Nonce': nonce, + 'X-Mailgun-Signature': signature, + 'X-Mailgun-Signature-Version': signature_version} + requests.post(url, headers=request_headers, data=request_body) + +*Authenticating a Request* + +.. code-block:: py + + from flask import Flask + from flask import request + import lemma + + [...] + + @app.route("/", methods=['POST']) + def process_webhook(): + + # extract headers and body + timestamp = request.headers.get('X-Mailgun-Timestamp', '') + nonce = request.headers.get('X-Mailgun-Nonce', '') + signature = request.headers.get('X-Mailgun-Signature', '') + request_body = request.data + + if not lemma.authenticate_request(timestamp, nonce, request_body, signature): + return 'Invalid request.' + + return 'Valid request.' + diff --git a/lemma.py b/lemma.py new file mode 100644 index 0000000..a161b98 --- /dev/null +++ b/lemma.py @@ -0,0 +1,301 @@ +''' +Module auth provides tools for signing and authenticating HTTP requests between +web services. See README.rst for more details. +''' + +import base64 +import hashlib +import hmac +import os +import threading +import time + +from cryptography.hazmat.primitives import constant_time +from expiringdict import ExpiringDict + +# constants +MAX_SKEW_SEC = 5 # 5 sec +CACHE_TIMEOUT = 30 # 30 sec +CACHE_CAPACITY = 5000 * CACHE_TIMEOUT # 5,000 msg/sec * 30 sec = 150,000 msg +SIGNATURE_VERSION = "2" + +# module level variables +LOCK = threading.RLock() +SHARED_SECRET = None +NONCE_CACHE = None + + +def initialize(keypath, cache_capacity=CACHE_CAPACITY, + cache_timeout=CACHE_TIMEOUT): + ''' + Initializes module by loading a shared key as well as nonce cache. This + module can handle authenticating 5,000 requests/second, if you need to + authenticate more requests than that, set the cache capacity and timeout + accordingly. + ''' + + global SHARED_SECRET, NONCE_CACHE + + # load shared secret from disk + try: + SHARED_SECRET = open(keypath).read().strip('\n') + except IOError, ioe: + SHARED_SECRET = None + + # configure nonce cache + NONCE_CACHE = ExpiringDict( + max_len=cache_capacity, max_age_seconds=cache_timeout) + + +def sign_request(request_body, + http_verb=None, http_resource_uri=None, headers=None, key=None): + ''' + Given a request body, signs request using an HMAC. Optional parameters are: + + 1. http_verb and http_resource_uri. http_verb is an HTTP verb and + http_resource_uri is the URI of the HTTP request. For example, if you are + performing a GET request on http://www.example.com/path?key=value#fragment + then http_verb would be "GET" and http_resource_uri would be + "/path?key=value#fragment". + 2. headers. headers is a dictonary of headers to also sign along with the + request. For example, the dictonary may look like: + {"X-Mailgun-Custom-Header": "foobar"} + 3. key. key is provided if you wish to override the key this module was + initialized with and sign with a different key. + + Returns tuple of timestamp, nonce, message signature, and signature version. + + >>> sign_request('{"hello": "world"}') + ('...', '...', '...', '...') + >>> sign_request('{"hello": "world"}', headers={"X-Custom-Header": "foo"}) + ('...', '...', '...', '...') + >>> sign_request('{"hello": "world"}', http_verb="GET", http_resource_uri="/path") + ('...', '...', '...', '...') + ''' + + # if shared secret or nonce cache not loaded, don't sign anything + if not SHARED_SECRET: + if not key: + raise AuthenticationException('No shared secret provided.') + if NONCE_CACHE is None: + raise AuthenticationException('Nonce cache not configured.') + + # make request body an empty string if it doesn't exist (GET request). + if not request_body: + request_body = '' + + # get 128-bit random number from /dev/urandom and base16 encode it + nonce = _generate_nonce(128) + + # get current timestamp + timestamp = _get_timestamp() + + # if we are passed in a key use it, otherwise use the global SHARED_SECRET + if key: + shared_secret = key + else: + shared_secret = SHARED_SECRET + + # get hmac over timestamp, nonce, and request body + signature = _compute_mac(shared_secret, timestamp, nonce, request_body, + http_verb=http_verb, http_resource_uri=http_resource_uri, headers=headers) + + return timestamp, nonce, signature, SIGNATURE_VERSION + + +def authenticate_request(timestamp, nonce, request_body, signature, + signature_version="2", http_verb=None, http_resource_uri=None, headers=None, key=None): + ''' + Given a timestamp, nonce, request body, signature, and optionally (signature_version, + http_verb and http_resource_uri, headers, and key: + + 1. Computes HMAC to ensure it matches given HMAC. + 2. Checks the timestamp to see if its within allowable timewindow. + 3. Check if the nonce has been seen in the cache before. + + If any of the optional parameters are passed in, they are used computing the + signature of the request. + + If a key is passed in, that key is used instead of the one the module was + initialized with. + + Returns a boolean. + ''' + + # if shared secret or nonce cache not loaded, don't authenticate anything + if not SHARED_SECRET: + if not key: + raise AuthenticationException('No shared secret provided.') + if NONCE_CACHE is None: + raise AuthenticationException('Nonce cache not configured.') + + # if any parameters are missing, return false + if not timestamp or not nonce or not signature: + return False + + # make request body an empty string if it doesn't exist (GET request). + if not request_body: + request_body = '' + + # if we are passed in a key use it, otherwise use the global SHARED_SECRET + if key: + shared_secret = key + else: + shared_secret = SHARED_SECRET + + # check the hmac + if not _check_mac(shared_secret, timestamp, nonce, request_body, signature, + http_verb=http_verb, http_resource_uri=http_resource_uri, headers=headers): + return False + + # check timestamp + if not _check_timestamp(timestamp): + return False + + # check to see if we have seen nonce before + if _nonce_in_cache(nonce): + return False + + # all checks pass, valid request + return True + + +def _generate_nonce(n): + ''' + Uses operating system source of randomness to generate an n-bit integer + to use as nonce value. Returns a hex-encoded version of the random number. + + >>> _generate_nonce() + 919368ACF548EE2BF635B071657B0B6F + ''' + + return base64.b16encode(os.urandom(n/8)) + + +def _get_timestamp(): + ''' + Returns a Unix timestamp string which denotes the number of seconds that + have elapsed since January 1, 1970 in UTC. + + >>> _get_timestamp() + 1406847690 + ''' + + return int(time.time()) + + +def _check_timestamp(timestamp): + ''' + Checks if given timestamp is within a valid time range. Returns a boolean. + ''' + + now = int(time.time()) + timestamp = int(timestamp) + + # if timestamp is from the future, it's invalid + if timestamp >= now + MAX_SKEW_SEC: + return False + + # if the timestamp is older than ttl - skew, it's invalid + if timestamp <= now - (CACHE_TIMEOUT - MAX_SKEW_SEC): + return False + + return True + + +def _nonce_in_cache(nonce): + ''' + Checks if the nonce has been seen before. Returns a boolean. + ''' + + with LOCK: + # if nonce has been seen before, it's invalid, otherwise add to cache. + if nonce in NONCE_CACHE: + return True + else: + NONCE_CACHE[nonce] = True + + return False + + raise AuthenticationException('Unable to obtain lock!') + + +def _compute_mac(shared_secret, timestamp, nonce, body, + http_verb=None, http_resource_uri=None, headers=None): + ''' + Given a timestamp, nonce, body, and optionally headers, returns hmac of + those values concatenated with each other along with the shared secret. + + >>> _compute_mac('1406847690', '919368ACF548EE2BF635B071657B0B6F', 'hi') + 50b828e3c9fdf849c5e6ee572604b00bc32663dce0c74fdf0f5b5d3261680efa + ''' + + # convert all to utf-8 + t = to_utf8(timestamp) + n = to_utf8(nonce) + b = to_utf8(body) + h = to_utf8(http_verb) + r = to_utf8(http_resource_uri) + + # requred parameters (timestamp, nonce, and body) + message = '{0}|{1}|{2}|{3}|{4}|{5}'.format(len(t), t, len(n), n, len(b), b) + + # optional parameters (http_verb, http_resource_uri) + if http_verb and http_resource_uri: + part = '|{0}|{1}|{2}|{3}'.format(len(h), h, len(r), r) + message = ''.join([message, part]) + + # optional parameters (headers) + if headers: + parts = [] + for k, v in headers.iteritems(): + # convert to utf-8, then build string + hv = to_utf8(v) + parts.append('|{0}|{1}'.format(len(hv), hv)) + message = ''.join([message] + parts) + + # return hmac-sha256 hex digest of the hmac + return hmac.new( + key=shared_secret, + msg=message, + digestmod=hashlib.sha256).hexdigest() + + +def _check_mac(shared_secret, timestamp, nonce, body, message_hmac, + http_verb=None, http_resource_uri=None, headers=None): + ''' + Computes HMAC and compares expected and obtained values. Performs constant + time comparison. Returns a boolean. + ''' + + # compute the expected hmac + expected_hmac = _compute_mac(shared_secret, timestamp, nonce, body, + http_verb=http_verb, http_resource_uri=http_resource_uri, headers=headers) + + # constant time check of expected againstreceived hmac + return constant_time.bytes_eq(to_utf8(message_hmac), expected_hmac) + + +def to_utf8(str_or_unicode): + ''' + Safely returns a UTF-8 version of a given string + + >>> utils.to_utf8(u'hi') + 'hi' + ''' + + if isinstance(str_or_unicode, unicode): + return str_or_unicode.encode("utf-8", "ignore") + return str(str_or_unicode) + + +class AuthenticationException(Exception): + ''' + Raised whenever some errors occurs while authenticating a request. + ''' + + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..bbffd47 --- /dev/null +++ b/setup.py @@ -0,0 +1,29 @@ +# coding:utf-8 + +import sys +from setuptools import setup, find_packages + + +setup(name='pylemma', + version='1.0.3', + description='Mailgun Cryptographic Tools', + long_description=open('README.rst').read(), + classifiers=[], + keywords='', + author='Mailgun Inc.', + author_email='admin@mailgunhq.com', + url='http://www.mailgun.net', + license='Apache 2', + packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + py_modules=['lemma'], + include_package_data=True, + zip_safe=True, + tests_require=[ + 'nose', + 'mock' + ], + install_requires=[ + 'expiringdict>=1.1.3', + 'cryptography>=0.5.1', + ], + ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..f576035 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,19 @@ +from os.path import join, abspath, dirname, exists + + +def fixtures_path(): + return join(abspath(dirname(__file__)), "fixtures") + + +def fixture_file(name): + return join(fixtures_path(), name) + + +def skip_if_asked(): + from nose import SkipTest + import sys + if "--no-skip" not in sys.argv: + raise SkipTest() + + +TEST_KEY = fixture_file("test.key") diff --git a/tests/fixtures/test.key b/tests/fixtures/test.key new file mode 100644 index 0000000..deabc2b --- /dev/null +++ b/tests/fixtures/test.key @@ -0,0 +1 @@ +042DAD12E0BE4625AC0B2C3F7172DBA8 diff --git a/tests/lemma_test.py b/tests/lemma_test.py new file mode 100644 index 0000000..128c49d --- /dev/null +++ b/tests/lemma_test.py @@ -0,0 +1,244 @@ +import time + +import lemma +import expiringdict + +from . import * + +from mock import patch +from nose.tools import assert_equal, assert_not_equal +from nose.tools import nottest + + +def test_initialize(): + # setup + lemma.initialize(TEST_KEY) + + # check + assert_equal(lemma.SHARED_SECRET, '042DAD12E0BE4625AC0B2C3F7172DBA8') + assert_not_equal(lemma.NONCE_CACHE, None) + + +@patch('lemma._get_timestamp') +@patch('lemma._generate_nonce') +def test_sign_request(gn, gt): + # setup + lemma.initialize(TEST_KEY) + + # mock _generate_nonce and _get_timestamp function return values + gn.return_value = '000102030405060708090a0b0c0d0e0f' + gt.return_value = '1330837567' + + # setup values we expect + expected_timestamp = '1330837567' + expected_nonce = '000102030405060708090a0b0c0d0e0f' + expected_signature = \ + '5a42c21371e8b3a2b50ca1ad72869dc7882aa83a6a2fb13db1bf108d92c6f05f' + expected_signature_version = "2" + + # test + got_timestamp, got_nonce, got_signature, got_signature_version = \ + lemma.sign_request('{"hello": "world"}') + + # check + assert_equal(expected_timestamp, got_timestamp) + assert_equal(expected_nonce, got_nonce) + assert_equal(expected_signature, got_signature) + assert_equal(expected_signature_version, got_signature_version) + + +@patch('time.time') +@patch('lemma._get_timestamp') +@patch('lemma._generate_nonce') +def test_authenticate_request(gn, gt, tm): + # setup + lemma.initialize(TEST_KEY) + + # mock _generate_nonce and _get_timestamp function return values + gn.return_value = '000102030405060708090a0b0c0d0e0f' + gt.return_value = '1330837567' + tm.return_value = 1330837567.0 + + # setup values we want to test and results + auth_tests = [ + { + # valid request + "input": { + "timestamp": "1330837567", + "nonce": "00000000000000000000000000000001", + "signature": "23de59b61ad7317e7f4c75a55c0970ad706d688624f30b468ec9f9fe7e9903d7", + "body": "{\"hello\": \"world\"}", + "http_verb": None, + "http_resource_uri": None, + "headers": None, + "key": None + }, + "output": True + }, + { + # valid request (with headers) + "input": { + "timestamp": "1330837567", + "nonce": "00000000000000000000000000000002", + "signature": "9f8c2c6d44e54a4a2fe921a734af5083dff27429529d9af8b359d3b6181ca39c", + "body": "{\"hello\": \"world\"}", + "http_verb": None, + "http_resource_uri": None, + "headers": {"foo": "bar"}, + "key": None + }, + "output": True + }, + { + # valid request (with with http_verb and http_resource_uri) + "input": { + "timestamp": "1330837567", + "nonce": "00000000000000000000000000000003", + "signature": "4f6415b3dfd306470617c14abc487807ba8e5bf26e0b57858bbbc9bb19de2923", + "body": "{\"hello\": \"world\"}", + "http_verb": "GET", + "http_resource_uri": "/path?key=value#fragment", + "headers": None, + "key": None + }, + "output": True + }, + { + # valid request (with key) + "input": { + "timestamp": "1330837567", + "nonce": "00000000000000000000000000000004", + "signature": "c8c9a91f00427f95a165eec6a7ccb0ad68d2655decb39aa3c66515b01b86eab4", + "body": "{\"hello\": \"world\"}", + "http_verb": None, + "http_resource_uri": None, + "headers": None, + "key": "abc" + }, + "output": True + }, + { + # forged signature + "input": { + "timestamp": "1330837567", + "nonce": "00000000000000000000000000000005", + "signature": "0000000000000000000000000000000000000000000000000000000000000000", + "body": "{\"hello\": \"world\"}", + "http_verb": None, + "http_resource_uri": None, + "headers": None, + "key": None + }, + "output": False + }, + { + # missing param + "input": { + "timestamp": "1330837567", + "nonce": "00000000000000000000000000000006", + "signature": None, + "body": "{\"hello\": \"world\"}", + "http_verb": None, + "http_resource_uri": None, + "headers": None, + "key": None + }, + "output": False + }] + + # check + for i, test in enumerate(auth_tests): + print 'Testing Input {}: {}'.format(i, test['input']) + testOutput = lemma.authenticate_request(test['input']['timestamp'], + test['input']['nonce'], + test['input']['body'], + test['input']['signature'], + http_verb=test['input']['http_verb'], + http_resource_uri=test['input']['http_resource_uri'], + headers=test['input']['headers'], + key=test['input']['key']) + assert_equal(test['output'], testOutput) + + +@patch('time.time') +def test_check_timestamp(tm): + # mock timestamp + tm.return_value = 1330837567.0 + + # setup values we want to test and results + auth_tests = [ + { + # goldilocks (perfect) timestamp + "input": { + "timestamp": "1330837567", + }, + "output": True + }, + { + # old timestamp + "input": { + "timestamp": "1330837517", + }, + "output": False + }, + { + # timestamp from future + "input": { + "timestamp": "1330837587", + }, + "output": False + }] + + # check + for test in auth_tests: + testOutput = lemma._check_timestamp(test['input']['timestamp']) + assert_equal(test['output'], testOutput) + + +@patch('time.time') +def test_check_nonce(tm): + # setup + lemma.initialize(TEST_KEY) + + # setup values we want to test, mock, and results + auth_tests = [ + { + # havn't seen before, should not be in cache. + "input": { + "nonce": "0", + "mock_time": 1330837567, + }, + "output": False + }, + { + # seen before, should be in cache. + "input": { + "nonce": "0", + "mock_time": 1330837567, + }, + "output": True + }, + { + # different value, should not be in cache. + "input": { + "nonce": "1", + "mock_time": 1330837567, + }, + "output": False + }, + { + # aged off first value, should not be in cache. + "input": { + "nonce": "0", + "mock_time": 1330837597, + }, + "output": False + }] + + # check + for test in auth_tests: + # mock time + tm.return_value = test['input']['mock_time'] + + testOutput = lemma._nonce_in_cache(test['input']['nonce']) + assert_equal(test['output'], testOutput)