From be0603d8050ffa725e2dc7b969739519872c06cb Mon Sep 17 00:00:00 2001 From: Yann Mahe Date: Tue, 8 Dec 2015 13:22:14 -0500 Subject: [PATCH 1/6] [util] `get_version` to resolve package version Create a `get_version` util method to resolve `datadog` package version. For compatibility purpose with Google App Engine, handle the case where `pkg_resources` is undefined. --- datadog/__init__.py | 15 ++------------- datadog/dogshell/__init__.py | 5 ++--- datadog/dogshell/wrap.py | 4 ++-- datadog/util/compat.py | 9 +++++++-- datadog/util/config.py | 27 ++++++++++++++++++++++++++- 5 files changed, 39 insertions(+), 21 deletions(-) diff --git a/datadog/__init__.py b/datadog/__init__.py index 4d02cf9c1..60885b4e3 100644 --- a/datadog/__init__.py +++ b/datadog/__init__.py @@ -8,7 +8,6 @@ * datadog.dogshell: a command-line tool, wrapping datadog.api, to interact with Datadog REST API. """ # stdlib -from pkg_resources import get_distribution, DistributionNotFound import os import os.path @@ -18,20 +17,10 @@ from datadog.threadstats import ThreadStats # noqa from datadog.util.hostname import get_hostname from datadog.util.compat import iteritems +from datadog.util.config import get_version -try: - _dist = get_distribution("datadog") - # Normalize case for Windows systems - dist_loc = os.path.normcase(_dist.location) - here = os.path.normcase(__file__) - if not here.startswith(os.path.join(dist_loc, __name__)): - # not installed, but there is another version that *is*e - raise DistributionNotFound -except DistributionNotFound: - __version__ = 'Please install datadog with setup.py' -else: - __version__ = _dist.version +__version__ = get_version() def initialize(api_key=None, app_key=None, host_name=None, api_host=None, diff --git a/datadog/dogshell/__init__.py b/datadog/dogshell/__init__.py index 3df5392f7..a5361a2b4 100644 --- a/datadog/dogshell/__init__.py +++ b/datadog/dogshell/__init__.py @@ -1,7 +1,6 @@ # stdlib import logging import os -import pkg_resources as pkg # 3p import argparse @@ -20,6 +19,7 @@ from datadog.dogshell.service_check import ServiceCheckClient from datadog.dogshell.tag import TagClient from datadog.dogshell.timeboard import TimeboardClient +from datadog.util.config import get_version logging.getLogger('dd.datadogpy').setLevel(logging.CRITICAL) @@ -44,8 +44,7 @@ def main(): parser.add_argument('--timeout', help="time to wait in seconds before timing" " out an API call (default 10)", default=10, type=int) parser.add_argument('-v', '--version', help='Dog API version', action='version', - version='%(prog)s {version}' - .format(version=pkg.require("datadog")[0].version)) + version='%(prog)s {0}'.format(get_version())) config = DogshellConfig() diff --git a/datadog/dogshell/wrap.py b/datadog/dogshell/wrap.py index 203a2f34c..241f628f5 100644 --- a/datadog/dogshell/wrap.py +++ b/datadog/dogshell/wrap.py @@ -19,7 +19,6 @@ ''' # stdlib import optparse -import pkg_resources as pkg import subprocess import sys import threading @@ -27,6 +26,7 @@ # datadog from datadog import initialize, api +from datadog.util.config import get_version SUCCESS = 'success' @@ -223,7 +223,7 @@ def main(): quotes to prevent python as soon as there is a space in your command. \n \nNOTICE: In normal \ mode, the whole stderr is printed before stdout, in flush_live mode they will be mixed but there \ is not guarantee that messages sent by the command on both stderr and stdout are printed in the \ -order they were sent.", version="%prog {0}".format(pkg.require("datadog")[0].version)) +order they were sent.", version="%prog {0}".format(get_version())) parser.add_option('-n', '--name', action='store', type='string', help="the name of the event \ as it should appear on your Datadog stream") diff --git a/datadog/util/compat.py b/datadog/util/compat.py index 13273e715..b3d45d230 100644 --- a/datadog/util/compat.py +++ b/datadog/util/compat.py @@ -1,6 +1,6 @@ # flake8: noqa - -""" Imports for compatibility with Py2 and Py3 +""" +Imports for compatibility with Py2, Py3 and Google App Engine. """ import sys import logging @@ -51,3 +51,8 @@ def iternext(iter): from urllib.parse import urlparse except ImportError: from urlparse import urlparse + +try: + import pkg_resources as pkg +except ImportError: + pkg = None diff --git a/datadog/util/config.py b/datadog/util/config.py index c303d9e29..e5b9b9d4a 100644 --- a/datadog/util/config.py +++ b/datadog/util/config.py @@ -3,7 +3,8 @@ import string import sys -from datadog.util.compat import configparser, StringIO, is_p3k +# datadog +from datadog.util.compat import configparser, StringIO, is_p3k, pkg # CONSTANTS DATADOG_CONF = "datadog.conf" @@ -125,3 +126,27 @@ def get_config(cfg_path=None, options=None): raise CfgNotFound return agentConfig + + +def get_version(): + """ + Resolve `datadog` package version. + """ + version = u"unknown" + + if not pkg: + return version + + try: + dist = pkg.get_distribution("datadog") + # Normalize case for Windows systems + dist_loc = os.path.normcase(dist.location) + here = os.path.normcase(__file__) + if not here.startswith(dist_loc): + # not installed, but there is another version that *is* + raise pkg.DistributionNotFound + version = dist.version + except pkg.DistributionNotFound: + version = u"Please install `datadog` with setup.py" + + return version From fdad3add1a34240e25b172bf5596e446ea022e88 Mon Sep 17 00:00:00 2001 From: Yann Mahe Date: Tue, 8 Dec 2015 13:41:55 -0500 Subject: [PATCH 2/6] [deps] `simplejson` fall back to stdlib `json` Enhance compatibility with Google App Engine: fall back `simplejson` to stdlib `json`. --- datadog/api/base.py | 5 +---- datadog/dogshell/comment.py | 6 ++---- datadog/dogshell/downtime.py | 2 +- datadog/dogshell/event.py | 4 +--- datadog/dogshell/host.py | 2 +- datadog/dogshell/monitor.py | 2 +- datadog/dogshell/screenboard.py | 2 +- datadog/dogshell/search.py | 4 +--- datadog/dogshell/service_check.py | 2 +- datadog/dogshell/tag.py | 4 +--- datadog/dogshell/timeboard.py | 4 ++-- datadog/util/compat.py | 6 ++++++ datadog/util/format.py | 4 ++-- datadog/util/hostname.py | 5 +---- tests/unit/api/helper.py | 3 +-- 15 files changed, 23 insertions(+), 32 deletions(-) diff --git a/datadog/api/base.py b/datadog/api/base.py index 839f3f018..f089e0d9d 100644 --- a/datadog/api/base.py +++ b/datadog/api/base.py @@ -3,14 +3,11 @@ import logging import requests -# 3p -import simplejson as json - # datadog from datadog.api.exceptions import ClientError, ApiError, HttpBackoff, \ HttpTimeout, ApiNotInitialized from datadog.api import _api_version, _max_timeouts, _backoff_period -from datadog.util.compat import is_p3k +from datadog.util.compat import json, is_p3k log = logging.getLogger('dd.datadogpy') diff --git a/datadog/dogshell/comment.py b/datadog/dogshell/comment.py index 4c7259c5c..c67ba7b35 100644 --- a/datadog/dogshell/comment.py +++ b/datadog/dogshell/comment.py @@ -1,12 +1,10 @@ # stdlib import sys -# 3p -import simplejson as json - # datadog -from datadog.dogshell.common import report_errors, report_warnings from datadog import api +from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json class CommentClient(object): diff --git a/datadog/dogshell/downtime.py b/datadog/dogshell/downtime.py index 34fffcd22..c99cc74f5 100644 --- a/datadog/dogshell/downtime.py +++ b/datadog/dogshell/downtime.py @@ -1,10 +1,10 @@ # 3p -import simplejson as json from datadog.util.format import pretty_json # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json class DowntimeClient(object): diff --git a/datadog/dogshell/event.py b/datadog/dogshell/event.py index 5b6accefe..3b0b53e77 100644 --- a/datadog/dogshell/event.py +++ b/datadog/dogshell/event.py @@ -4,12 +4,10 @@ import re import sys -# 3p -import simplejson as json - # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json def prettyprint_event(event): diff --git a/datadog/dogshell/host.py b/datadog/dogshell/host.py index 531c3804e..26a43547c 100644 --- a/datadog/dogshell/host.py +++ b/datadog/dogshell/host.py @@ -1,10 +1,10 @@ # 3p -import simplejson as json from datadog.util.format import pretty_json # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json class HostClient(object): diff --git a/datadog/dogshell/monitor.py b/datadog/dogshell/monitor.py index bf19c3c14..ae9d2139a 100644 --- a/datadog/dogshell/monitor.py +++ b/datadog/dogshell/monitor.py @@ -1,10 +1,10 @@ # 3p -import simplejson as json from datadog.util.format import pretty_json # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json class MonitorClient(object): diff --git a/datadog/dogshell/screenboard.py b/datadog/dogshell/screenboard.py index fc215ae68..f31dd62ea 100644 --- a/datadog/dogshell/screenboard.py +++ b/datadog/dogshell/screenboard.py @@ -5,12 +5,12 @@ import webbrowser # 3p -import simplejson as json from datadog.util.format import pretty_json # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings, print_err +from datadog.util.compat import json from datetime import datetime diff --git a/datadog/dogshell/search.py b/datadog/dogshell/search.py index 4903fd296..113a58984 100644 --- a/datadog/dogshell/search.py +++ b/datadog/dogshell/search.py @@ -1,9 +1,7 @@ -# 3p -import simplejson as json - # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json # TODO IS there a test ? diff --git a/datadog/dogshell/service_check.py b/datadog/dogshell/service_check.py index e6489abab..40623c967 100644 --- a/datadog/dogshell/service_check.py +++ b/datadog/dogshell/service_check.py @@ -1,10 +1,10 @@ # 3p -import simplejson as json from datadog.util.format import pretty_json # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json class ServiceCheckClient(object): diff --git a/datadog/dogshell/tag.py b/datadog/dogshell/tag.py index 83f881149..3205ea72e 100644 --- a/datadog/dogshell/tag.py +++ b/datadog/dogshell/tag.py @@ -1,9 +1,7 @@ -# 3p -import simplejson as json - # datadog from datadog import api from datadog.dogshell.common import report_errors, report_warnings +from datadog.util.compat import json class TagClient(object): diff --git a/datadog/dogshell/timeboard.py b/datadog/dogshell/timeboard.py index b0bd6b604..553988a11 100644 --- a/datadog/dogshell/timeboard.py +++ b/datadog/dogshell/timeboard.py @@ -6,12 +6,12 @@ # 3p import argparse -import simplejson as json # datadog from datadog import api -from datadog.util.format import pretty_json from datadog.dogshell.common import report_errors, report_warnings, print_err +from datadog.util.compat import json +from datadog.util.format import pretty_json from datetime import datetime diff --git a/datadog/util/compat.py b/datadog/util/compat.py index b3d45d230..aa4cb2840 100644 --- a/datadog/util/compat.py +++ b/datadog/util/compat.py @@ -56,3 +56,9 @@ def iternext(iter): import pkg_resources as pkg except ImportError: pkg = None + +# Prefer `simplejson` but fall back to stdlib `json` +try: + import simplejson as json +except ImportError: + import json diff --git a/datadog/util/format.py b/datadog/util/format.py index 996719158..19ab8187a 100644 --- a/datadog/util/format.py +++ b/datadog/util/format.py @@ -1,5 +1,5 @@ -# 3p -import simplejson as json +# datadog +from datadog.util.compat import json def pretty_json(obj): diff --git a/datadog/util/hostname.py b/datadog/util/hostname.py index 497ee9581..278c79c3c 100644 --- a/datadog/util/hostname.py +++ b/datadog/util/hostname.py @@ -5,11 +5,8 @@ import subprocess import types -# 3p -import simplejson as json - # datadog -from datadog.util.compat import url_lib, is_p3k, iteritems +from datadog.util.compat import url_lib, is_p3k, iteritems, json from datadog.util.config import get_config, get_os, CfgNotFound VALID_HOSTNAME_RFC_1123_PATTERN = re.compile(r"^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$") # noqa diff --git a/tests/unit/api/helper.py b/tests/unit/api/helper.py index ebfed1a31..ed7477a96 100644 --- a/tests/unit/api/helper.py +++ b/tests/unit/api/helper.py @@ -4,14 +4,13 @@ # 3p from mock import patch, Mock import requests -import simplejson as json # datadog from datadog import initialize, api from datadog.api.base import CreateableAPIResource, UpdatableAPIResource, DeletableAPIResource,\ GetableAPIResource, ListableAPIResource, ActionAPIResource from datadog.api.exceptions import ApiError -from datadog.util.compat import iteritems +from datadog.util.compat import iteritems, json API_KEY = "apikey" From 03fa45434b80a3da1fb776b788860bb34dc72ac0 Mon Sep 17 00:00:00 2001 From: Yann Mahe Date: Tue, 8 Dec 2015 18:16:36 -0500 Subject: [PATCH 3/6] [api] split api and http clients Redesign the `api` module to dissociate the API from the HTTP client. Make HTTP clients pluggable. **Changes** Two new modules: * `api_client`. * `APIClient`. Defines * `http_client`. * `HTTPClient`. Defines a pluggable HTTP client skeleton. * `RequestClient`. HTTP client based on `requests` 3p module. One module renamed: * `api.base` module is now `api.resources` --- datadog/__init__.py | 2 +- datadog/api/api_client.py | 213 +++++++++++++++++ datadog/api/base.py | 439 ---------------------------------- datadog/api/comments.py | 2 +- datadog/api/downtimes.py | 2 +- datadog/api/events.py | 4 +- datadog/api/exceptions.py | 67 +++--- datadog/api/graphs.py | 2 +- datadog/api/hosts.py | 2 +- datadog/api/http_client.py | 88 +++++++ datadog/api/infrastructure.py | 2 +- datadog/api/metrics.py | 4 +- datadog/api/monitors.py | 2 +- datadog/api/resources.py | 223 +++++++++++++++++ datadog/api/screenboards.py | 2 +- datadog/api/service_checks.py | 4 +- datadog/api/tags.py | 2 +- datadog/api/timeboards.py | 2 +- datadog/api/users.py | 2 +- tests/unit/api/helper.py | 4 +- 20 files changed, 584 insertions(+), 484 deletions(-) create mode 100644 datadog/api/api_client.py delete mode 100644 datadog/api/base.py create mode 100644 datadog/api/http_client.py create mode 100644 datadog/api/resources.py diff --git a/datadog/__init__.py b/datadog/__init__.py index 60885b4e3..cb4fb60e4 100644 --- a/datadog/__init__.py +++ b/datadog/__init__.py @@ -15,9 +15,9 @@ from datadog import api from datadog.dogstatsd import DogStatsd, statsd # noqa from datadog.threadstats import ThreadStats # noqa -from datadog.util.hostname import get_hostname from datadog.util.compat import iteritems from datadog.util.config import get_version +from datadog.util.hostname import get_hostname __version__ = get_version() diff --git a/datadog/api/api_client.py b/datadog/api/api_client.py new file mode 100644 index 000000000..973624d25 --- /dev/null +++ b/datadog/api/api_client.py @@ -0,0 +1,213 @@ +# stdlib +import logging +import time + +# datadog +from datadog.api import _api_version, _max_timeouts, _backoff_period +from datadog.api.exceptions import ClientError, ApiError, HttpBackoff, \ + HttpTimeout, ApiNotInitialized +from datadog.api.http_client import RequestClient +from datadog.util.compat import json, is_p3k + + +log = logging.getLogger('dd.datadogpy') + + +class APIClient(object): + """ + """ + # HTTP transport parameters + _backoff_period = _backoff_period + _max_timeouts = _max_timeouts + _backoff_timestamp = None + _timeout_counter = 0 + _api_version = _api_version + + def __init__(self): + """ + Instantiate a HTTP Client. + """ + self._http_client = self._get_http_client() + + @staticmethod + def _get_http_client(): + """ + Resolve the appropriate HTTP client based on priority and user environment. + """ + + return RequestClient + + def submit(self, method, path, body=None, attach_host_name=False, response_formatter=None, + error_formatter=None, **params): + """ + Make an HTTP API request + + :param method: HTTP method to use to contact API endpoint + :type method: HTTP method string + + :param path: API endpoint url + :type path: url + + :param body: dictionary to be sent in the body of the request + :type body: dictionary + + :param response_formatter: function to format JSON response from HTTP API request + :type response_formatter: JSON input function + + :param error_formatter: function to format JSON error response from HTTP API request + :type error_formatter: JSON input function + + :param attach_host_name: link the new resource object to the host name + :type attach_host_name: bool + + :param params: dictionary to be sent in the query string of the request + :type params: dictionary + + :returns: JSON or formated response from HTTP API request + """ + try: + # Check if it's ok to submit + if not self._should_submit(): + _, backoff_time_left = self._backoff_status() + raise HttpBackoff(backoff_time_left) + + # Import API, User and HTTP settings + from datadog.api import _api_key, _application_key, _api_host, \ + _mute, _host_name, _proxies, _max_retries, _timeout, \ + _cacert + + # Check keys and add then to params + if _api_key is None: + raise ApiNotInitialized("API key is not set." + " Please run 'initialize' method first.") + params['api_key'] = _api_key + if _application_key: + params['application_key'] = _application_key + + # Construct the url + url = "%s/api/%s/%s" % (_api_host, self._api_version, path.lstrip("/")) + + # Attach host name to body + if attach_host_name and body: + # Is it a 'series' list of objects ? + if 'series' in body: + # Adding the host name to all objects + for obj_params in body['series']: + if obj_params.get('host', "") == "": + obj_params['host'] = _host_name + else: + if body.get('host', "") == "": + body['host'] = _host_name + + # If defined, make sure tags are defined as a comma-separated string + if 'tags' in params and isinstance(params['tags'], list): + params['tags'] = ','.join(params['tags']) + + # Process the body, if necessary + headers = {} + if isinstance(body, dict): + body = json.dumps(body) + headers['Content-Type'] = 'application/json' + + # Process requesting + start_time = time.time() + + result = self._http_client.request( + method=method, url=url, + headers=headers, params=params, data=body, + timeout=_timeout, max_retries=_max_retries, + proxies=_proxies, verify=_cacert + ) + + # Request succeeded: log it and reset the timeout counter + duration = round((time.time() - start_time) * 1000., 4) + log.info("%s %s %s (%sms)" % (result.status_code, method, url, duration)) + self._timeout_counter = 0 + + # Format response content + content = result.content + + if content: + try: + if is_p3k(): + response_obj = json.loads(content.decode('utf-8')) + else: + response_obj = json.loads(content) + except ValueError: + raise ValueError('Invalid JSON response: {0}'.format(content)) + + if response_obj and 'errors' in response_obj: + raise ApiError(response_obj) + else: + response_obj = None + if response_formatter is None: + return response_obj + else: + return response_formatter(response_obj) + + except ClientError as e: + if _mute: + log.error(str(e)) + if error_formatter is None: + return {'errors': e.args[0]} + else: + return error_formatter({'errors': e.args[0]}) + else: + raise + except ApiError as e: + if _mute: + for error in e.args[0]['errors']: + log.error(str(error)) + if error_formatter is None: + return e.args[0] + else: + return error_formatter(e.args[0]) + else: + raise + + @classmethod + def _should_submit(cls): + """ + Returns True if we're in a state where we should make a request + (backoff expired, no backoff in effect), false otherwise. + """ + now = time.time() + should_submit = False + + # If we're not backing off, but the timeout counter exceeds the max + # number of timeouts, then enter the backoff state, recording the time + # we started backing off + if not cls._backoff_timestamp and cls._timeout_counter >= cls._max_timeouts: + log.info("Max number of datadog timeouts exceeded, backing off for {0} seconds" + .format(cls._backoff_period)) + cls._backoff_timestamp = now + should_submit = False + + # If we are backing off but the we've waiting sufficiently long enough + # (backoff_retry_age), exit the backoff state and reset the timeout + # counter so that we try submitting metrics again + elif cls._backoff_timestamp: + backed_off_time, backoff_time_left = cls._backoff_status() + if backoff_time_left < 0: + log.info("Exiting backoff state after {0} seconds, will try to submit metrics again" + .format(backed_off_time)) + cls._backoff_timestamp = None + cls._timeout_counter = 0 + should_submit = True + else: + log.info("In backoff state, won't submit metrics for another {0} seconds" + .format(backoff_time_left)) + should_submit = False + else: + should_submit = True + + return should_submit + + @classmethod + def _backoff_status(cls): + """ + """ + now = time.time() + backed_off_time = now - cls._backoff_timestamp + backoff_time_left = cls._backoff_period - backed_off_time + return round(backed_off_time, 2), round(backoff_time_left, 2) diff --git a/datadog/api/base.py b/datadog/api/base.py deleted file mode 100644 index f089e0d9d..000000000 --- a/datadog/api/base.py +++ /dev/null @@ -1,439 +0,0 @@ -# stdlib -import time -import logging -import requests - -# datadog -from datadog.api.exceptions import ClientError, ApiError, HttpBackoff, \ - HttpTimeout, ApiNotInitialized -from datadog.api import _api_version, _max_timeouts, _backoff_period -from datadog.util.compat import json, is_p3k - -log = logging.getLogger('dd.datadogpy') - - -class HTTPClient(object): - """ - HTTP client based on Requests library for Datadog API calls - """ - # http transport params - _backoff_period = _backoff_period - _max_timeouts = _max_timeouts - _backoff_timestamp = None - _timeout_counter = 0 - _api_version = _api_version - - @classmethod - def request(cls, method, path, body=None, attach_host_name=False, response_formatter=None, - error_formatter=None, **params): - """ - Make an HTTP API request - - :param method: HTTP method to use to contact API endpoint - :type method: HTTP method string - - :param path: API endpoint url - :type path: url - - :param body: dictionary to be sent in the body of the request - :type body: dictionary - - :param response_formatter: function to format JSON response from HTTP API request - :type response_formatter: JSON input function - - :param error_formatter: function to format JSON error response from HTTP API request - :type error_formatter: JSON input function - - :param attach_host_name: link the new resource object to the host name - :type attach_host_name: bool - - :param params: dictionary to be sent in the query string of the request - :type params: dictionary - - :returns: JSON or formated response from HTTP API request - """ - - try: - # Check if it's ok to submit - if not cls._should_submit(): - raise HttpBackoff("Too many timeouts. Won't try again for {1} seconds." - .format(*cls._backoff_status())) - - # Import API, User and HTTP settings - from datadog.api import _api_key, _application_key, _api_host, \ - _mute, _host_name, _proxies, _max_retries, _timeout, \ - _cacert - - # Check keys and add then to params - if _api_key is None: - raise ApiNotInitialized("API key is not set." - " Please run 'initialize' method first.") - params['api_key'] = _api_key - if _application_key: - params['application_key'] = _application_key - - # Construct the url - url = "%s/api/%s/%s" % (_api_host, cls._api_version, path.lstrip("/")) - - # Attach host name to body - if attach_host_name and body: - # Is it a 'series' list of objects ? - if 'series' in body: - # Adding the host name to all objects - for obj_params in body['series']: - if obj_params.get('host', "") == "": - obj_params['host'] = _host_name - else: - if body.get('host', "") == "": - body['host'] = _host_name - - # If defined, make sure tags are defined as a comma-separated string - if 'tags' in params and isinstance(params['tags'], list): - params['tags'] = ','.join(params['tags']) - - # Process the body, if necessary - headers = {} - if isinstance(body, dict): - body = json.dumps(body) - headers['Content-Type'] = 'application/json' - - # Process requesting - start_time = time.time() - try: - # Use a session to set a max_retries parameters - s = requests.Session() - http_adapter = requests.adapters.HTTPAdapter(max_retries=_max_retries) - s.mount('https://', http_adapter) - - # Request - result = s.request( - method, - url, - headers=headers, - params=params, - data=body, - timeout=_timeout, - proxies=_proxies, - verify=_cacert) - - result.raise_for_status() - except requests.ConnectionError as e: - raise ClientError("Could not request %s %s%s: %s" % (method, _api_host, url, e)) - except requests.exceptions.Timeout as e: - cls._timeout_counter += 1 - raise HttpTimeout('%s %s timed out after %d seconds.' % (method, url, _timeout)) - except requests.exceptions.HTTPError as e: - if e.response.status_code in (400, 403, 404, 409): - # This gets caught afterwards and raises an ApiError exception - pass - else: - raise - except TypeError as e: - raise TypeError( - "Your installed version of 'requests' library seems not compatible with" - "Datadog's usage. We recommand upgrading it ('pip install -U requests')." - "If you need help or have any question, please contact support@datadoghq.com") - - # Request succeeded: log it and reset the timeout counter - duration = round((time.time() - start_time) * 1000., 4) - log.info("%s %s %s (%sms)" % (result.status_code, method, url, duration)) - cls._timeout_counter = 0 - - # Format response content - content = result.content - - if content: - try: - if is_p3k(): - response_obj = json.loads(content.decode('utf-8')) - else: - response_obj = json.loads(content) - except ValueError: - raise ValueError('Invalid JSON response: {0}'.format(content)) - - if response_obj and 'errors' in response_obj: - raise ApiError(response_obj) - else: - response_obj = None - if response_formatter is None: - return response_obj - else: - return response_formatter(response_obj) - - except ClientError as e: - if _mute: - log.error(str(e)) - if error_formatter is None: - return {'errors': e.args[0]} - else: - return error_formatter({'errors': e.args[0]}) - else: - raise - except ApiError as e: - if _mute: - for error in e.args[0]['errors']: - log.error(str(error)) - if error_formatter is None: - return e.args[0] - else: - return error_formatter(e.args[0]) - else: - raise - - # Private functions - @classmethod - def _should_submit(cls): - """ Returns True if we're in a state where we should make a request - (backoff expired, no backoff in effect), false otherwise. - """ - now = time.time() - should_submit = False - - # If we're not backing off, but the timeout counter exceeds the max - # number of timeouts, then enter the backoff state, recording the time - # we started backing off - if not cls._backoff_timestamp and cls._timeout_counter >= cls._max_timeouts: - log.info("Max number of datadog timeouts exceeded, backing off for {0} seconds" - .format(cls._backoff_period)) - cls._backoff_timestamp = now - should_submit = False - - # If we are backing off but the we've waiting sufficiently long enough - # (backoff_retry_age), exit the backoff state and reset the timeout - # counter so that we try submitting metrics again - elif cls._backoff_timestamp: - backed_off_time, backoff_time_left = cls._backoff_status() - if backoff_time_left < 0: - log.info("Exiting backoff state after {0} seconds, will try to submit metrics again" - .format(backed_off_time)) - cls._backoff_timestamp = None - cls._timeout_counter = 0 - should_submit = True - else: - log.info("In backoff state, won't submit metrics for another {0} seconds" - .format(backoff_time_left)) - should_submit = False - else: - should_submit = True - - return should_submit - - @classmethod - def _backoff_status(cls): - now = time.time() - backed_off_time = now - cls._backoff_timestamp - backoff_time_left = cls._backoff_period - backed_off_time - return round(backed_off_time, 2), round(backoff_time_left, 2) - - -# API Resource types are listed below -class CreateableAPIResource(object): - """ - Creatable API Resource - """ - @classmethod - def create(cls, attach_host_name=False, method='POST', id=None, params=None, **body): - """ - Create a new API resource object - - :param attach_host_name: link the new resource object to the host name - :type attach_host_name: bool - - :param method: HTTP method to use to contact API endpoint - :type method: HTTP method string - - :param id: create a new resource object as a child of the given object - :type id: id - - :param params: new resource object source - :type params: dictionary - - :param body: new resource object attributes - :type body: dictionary - - :returns: JSON response from HTTP API request - """ - if params is None: - params = {} - if method == 'GET': - return HTTPClient.request('GET', cls._class_url, **body) - if id is None: - return HTTPClient.request('POST', cls._class_url, body, - attach_host_name=attach_host_name, **params) - else: - return HTTPClient.request('POST', cls._class_url + "/" + str(id), body, - attach_host_name=attach_host_name, **params) - - -class SendableAPIResource(object): - """ - Fork of CreateableAPIResource class with different method names - """ - @classmethod - def send(cls, attach_host_name=False, id=None, **body): - """ - Create an API resource object - - :param attach_host_name: link the new resource object to the host name - :type attach_host_name: bool - - :param id: create a new resource object as a child of the given object - :type id: id - - :param body: new resource object attributes - :type body: dictionary - - :returns: JSON response from HTTP API request - """ - if id is None: - return HTTPClient.request('POST', cls._class_url, body, - attach_host_name=attach_host_name) - else: - return HTTPClient.request('POST', cls._class_url + "/" + str(id), body, - attach_host_name=attach_host_name) - - -class UpdatableAPIResource(object): - """ - Updatable API Resource - """ - @classmethod - def update(cls, id, params=None, **body): - """ - Update an API resource object - - :param params: updated resource object source - :type params: dictionary - - :param body: updated resource object attributes - :type body: dictionary - - :returns: JSON response from HTTP API request - """ - if params is None: - params = {} - return HTTPClient.request('PUT', cls._class_url + "/" + str(id), body, **params) - - -class DeletableAPIResource(object): - """ - Deletable API Resource - """ - @classmethod - def delete(cls, id, **params): - """ - Delete an API resource object - - :param id: resource object to delete - :type id: id - - :returns: JSON response from HTTP API request - """ - return HTTPClient.request('DELETE', cls._class_url + "/" + str(id), **params) - - -class GetableAPIResource(object): - """ - Getable API Resource - """ - @classmethod - def get(cls, id, **params): - """ - Get information about an API resource object - - :param id: resource object id to retrieve - :type id: id - - :param params: parameters to filter API resource stream - :type params: dictionary - - :returns: JSON response from HTTP API request - """ - return HTTPClient.request('GET', cls._class_url + "/" + str(id), **params) - - -class ListableAPIResource(object): - """ - Listable API Resource - """ - @classmethod - def get_all(cls, **params): - """ - List API resource objects - - :param params: parameters to filter API resource stream - :type params: dictionary - - :returns: JSON response from HTTP API request - """ - return HTTPClient.request('GET', cls._class_url, **params) - - -class SearchableAPIResource(object): - """ - Fork of ListableAPIResource class with different method names - """ - @classmethod - def _search(cls, **params): - """ - Query an API resource stream - - :param params: parameters to filter API resource stream - :type params: dictionary - - :returns: JSON response from HTTP API request - """ - return HTTPClient.request('GET', cls._class_url, **params) - - -class ActionAPIResource(object): - """ - Actionable API Resource - """ - @classmethod - def _trigger_class_action(cls, method, name, id=None, **params): - """ - Trigger an action - - :param method: HTTP method to use to contact API endpoint - :type method: HTTP method string - - :param name: action name - :type name: string - - :param id: trigger the action for the specified resource object - :type id: id - - :param params: action parameters - :type params: dictionary - - :returns: JSON response from HTTP API request - """ - if id is None: - return HTTPClient.request(method, cls._class_url + "/" + name, params) - else: - return HTTPClient.request(method, cls._class_url + "/" + str(id) + "/" + name, params) - - @classmethod - def _trigger_action(cls, method, name, id=None, **params): - """ - Trigger an action - - :param method: HTTP method to use to contact API endpoint - :type method: HTTP method string - - :param name: action name - :type name: string - - :param id: trigger the action for the specified resource object - :type id: id - - :param params: action parameters - :type params: dictionary - - :returns: JSON response from HTTP API request - """ - if id is None: - return HTTPClient.request(method, name, params) - else: - return HTTPClient.request(method, name + "/" + str(id), params) diff --git a/datadog/api/comments.py b/datadog/api/comments.py index f0f4c01d3..39b9e4fc3 100644 --- a/datadog/api/comments.py +++ b/datadog/api/comments.py @@ -1,4 +1,4 @@ -from datadog.api.base import CreateableAPIResource, UpdatableAPIResource, \ +from datadog.api.resources import CreateableAPIResource, UpdatableAPIResource, \ DeletableAPIResource diff --git a/datadog/api/downtimes.py b/datadog/api/downtimes.py index 9af703feb..82a84fd63 100644 --- a/datadog/api/downtimes.py +++ b/datadog/api/downtimes.py @@ -1,4 +1,4 @@ -from datadog.api.base import GetableAPIResource, CreateableAPIResource,\ +from datadog.api.resources import GetableAPIResource, CreateableAPIResource,\ UpdatableAPIResource, ListableAPIResource, DeletableAPIResource diff --git a/datadog/api/events.py b/datadog/api/events.py index c3569d15b..831d0b8d6 100644 --- a/datadog/api/events.py +++ b/datadog/api/events.py @@ -1,6 +1,6 @@ -from datadog.util.compat import iteritems -from datadog.api.base import GetableAPIResource, CreateableAPIResource, \ +from datadog.api.resources import GetableAPIResource, CreateableAPIResource, \ SearchableAPIResource +from datadog.util.compat import iteritems class Event(GetableAPIResource, CreateableAPIResource, SearchableAPIResource): diff --git a/datadog/api/exceptions.py b/datadog/api/exceptions.py index 82dfd27fb..8cd7f5763 100644 --- a/datadog/api/exceptions.py +++ b/datadog/api/exceptions.py @@ -1,34 +1,47 @@ -""" Module containing all the possible exceptions that datadog can raise. """ -__all__ = [ - 'DatadogException', - 'ClientError', - 'HttpTimeout', - 'HttpBackoff', - 'ApiError', - 'ApiNotInitialized', -] - - -class DatadogException(Exception): - pass - - -class ClientError(DatadogException): - "When HTTP connection to Datadog endpoint is not possible" - - -class HttpTimeout(DatadogException): - "HTTP connection timeout" - - -class HttpBackoff(DatadogException): - "Backing off after too many timeouts" +Module containing all the possible exceptions that `datadog` can raise. +""" -class ApiError(DatadogException): +class ClientError(Exception): + """ + When HTTP connection to Datadog endpoint is not possible. + """ + def __init__(self, method, url, exception): + message = u"Could not request {method} {url}: {exception}. "\ + u"Please check the network connection or try again later. "\ + u"If the problem persists, please contact support@datadoghq.com".format( + method=method, url=url, exception=exception + ) + super(ClientError, self).__init__(message) + + +class HttpTimeout(Exception): + """ + HTTP connection timeout. + """ + def __init__(self, method, url, timeout): + message = u"{method} {url} timed out after {timeout}. "\ + u"Please try again later. "\ + u"If the problem persists, please contact support@datadoghq.com".format( + method=method, url=url, timeout=timeout + ) + super(HttpTimeout, self).__init__(message) + + +class HttpBackoff(Exception): + """ + Backing off after too many timeouts. + """ + def __init__(self, backoff_period): + message = u"Too many timeouts. Won't try again for {backoff_period} seconds. ".format( + backoff_period=backoff_period) + super(HttpBackoff, self).__init__(message) + + +class ApiError(Exception): "Datadog API is returning an error" -class ApiNotInitialized(DatadogException): +class ApiNotInitialized(Exception): "No API key is set" diff --git a/datadog/api/graphs.py b/datadog/api/graphs.py index 9a247f7d8..d938f0fe2 100644 --- a/datadog/api/graphs.py +++ b/datadog/api/graphs.py @@ -1,5 +1,5 @@ from datadog.util.compat import urlparse -from datadog.api.base import ( +from datadog.api.resources import ( CreateableAPIResource, ActionAPIResource, GetableAPIResource, diff --git a/datadog/api/hosts.py b/datadog/api/hosts.py index f14432c1f..90eab3d97 100644 --- a/datadog/api/hosts.py +++ b/datadog/api/hosts.py @@ -1,4 +1,4 @@ -from datadog.api.base import ActionAPIResource +from datadog.api.resources import ActionAPIResource class Host(ActionAPIResource): diff --git a/datadog/api/http_client.py b/datadog/api/http_client.py new file mode 100644 index 000000000..f95a65b00 --- /dev/null +++ b/datadog/api/http_client.py @@ -0,0 +1,88 @@ +""" +Available HTTP Client for Datadog API client. + +1. Priority to `requests` +2. Fall back to `urlfetch` module on Google App Engine +""" +# stdlib +import logging + +# 3p +import requests + +# datadog +from datadog.api.exceptions import ClientError, HttpTimeout + + +log = logging.getLogger('dd.datadogpy') + + +class HTTPClient(object): + """ + An abstract generic HTTP client. Subclasses must implement the `request` methods. + """ + _CORE = NotImplemented + + @classmethod + def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries): + """ + """ + raise NotImplementedError( + u"Must be implemented by HTTPClient subclasses." + ) + + +class RequestClient(HTTPClient): + """ + HTTP client based on 3rd party `requests` module. + """ + @classmethod + def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries): + """ + """ + # Use a session to set a max_retries parameters + s = requests.Session() + http_adapter = requests.adapters.HTTPAdapter(max_retries=max_retries) + s.mount('https://', http_adapter) + + try: + # Request + result = s.request( + method, + url, + headers=headers, + params=params, + data=data, + timeout=timeout, + proxies=proxies, + verify=verify) + + # Raise on status + result.raise_for_status() + + except requests.ConnectionError as e: + raise ClientError(method, url, e) + except requests.exceptions.Timeout as e: + cls._timeout_counter += 1 + raise HttpTimeout(method, url, timeout) + except requests.exceptions.HTTPError as e: + if e.response.status_code in (400, 403, 404, 409): + # This gets caught afterwards and raises an ApiError exception + pass + else: + raise + except TypeError as e: + raise TypeError( + u"Your installed version of `requests` library seems not compatible with" + u"Datadog's usage. We recommand upgrading it ('pip install -U requests')." + u"If you need help or have any question, please contact support@datadoghq.com" + ) + + return result + + +class URLFetchClient(HTTPClient): + """ + HTTP client based on Google App Engine `urlfetch` module. + """ + pass diff --git a/datadog/api/infrastructure.py b/datadog/api/infrastructure.py index 63b324df7..01f8c9da4 100644 --- a/datadog/api/infrastructure.py +++ b/datadog/api/infrastructure.py @@ -1,4 +1,4 @@ -from datadog.api.base import SearchableAPIResource +from datadog.api.resources import SearchableAPIResource class Infrastructure(SearchableAPIResource): diff --git a/datadog/api/metrics.py b/datadog/api/metrics.py index f0853f6ec..b175e4d45 100644 --- a/datadog/api/metrics.py +++ b/datadog/api/metrics.py @@ -1,8 +1,10 @@ +# stdlib import time from numbers import Number -from datadog.api.base import SearchableAPIResource, SendableAPIResource +# datadog from datadog.api.exceptions import ApiError +from datadog.api.resources import SearchableAPIResource, SendableAPIResource class Metric(SearchableAPIResource, SendableAPIResource): diff --git a/datadog/api/monitors.py b/datadog/api/monitors.py index 5c4487905..b8cf1a44a 100644 --- a/datadog/api/monitors.py +++ b/datadog/api/monitors.py @@ -1,4 +1,4 @@ -from datadog.api.base import GetableAPIResource, CreateableAPIResource, \ +from datadog.api.resources import GetableAPIResource, CreateableAPIResource, \ UpdatableAPIResource, ListableAPIResource, DeletableAPIResource, \ ActionAPIResource diff --git a/datadog/api/resources.py b/datadog/api/resources.py new file mode 100644 index 000000000..6a0cf264f --- /dev/null +++ b/datadog/api/resources.py @@ -0,0 +1,223 @@ +""" +Datadog API resources. +""" +# stdlib +import logging + +# datadog +from datadog.api.api_client import APIClient + + +log = logging.getLogger('dd.datadogpy') + + +class CreateableAPIResource(object): + """ + Creatable API Resource + """ + @classmethod + def create(cls, attach_host_name=False, method='POST', id=None, params=None, **body): + """ + Create a new API resource object + + :param attach_host_name: link the new resource object to the host name + :type attach_host_name: bool + + :param method: HTTP method to use to contact API endpoint + :type method: HTTP method string + + :param id: create a new resource object as a child of the given object + :type id: id + + :param params: new resource object source + :type params: dictionary + + :param body: new resource object attributes + :type body: dictionary + + :returns: JSON response from HTTP API request + """ + if params is None: + params = {} + if method == 'GET': + return APIClient().submit('GET', cls._class_url, **body) + if id is None: + return APIClient().submit('POST', cls._class_url, body, + attach_host_name=attach_host_name, **params) + else: + return APIClient().submit('POST', cls._class_url + "/" + str(id), body, + attach_host_name=attach_host_name, **params) + + +class SendableAPIResource(object): + """ + Fork of CreateableAPIResource class with different method names + """ + @classmethod + def send(cls, attach_host_name=False, id=None, **body): + """ + Create an API resource object + + :param attach_host_name: link the new resource object to the host name + :type attach_host_name: bool + + :param id: create a new resource object as a child of the given object + :type id: id + + :param body: new resource object attributes + :type body: dictionary + + :returns: JSON response from HTTP API request + """ + if id is None: + return APIClient().submit('POST', cls._class_url, body, + attach_host_name=attach_host_name) + else: + return APIClient().submit('POST', cls._class_url + "/" + str(id), body, + attach_host_name=attach_host_name) + + +class UpdatableAPIResource(object): + """ + Updatable API Resource + """ + @classmethod + def update(cls, id, params=None, **body): + """ + Update an API resource object + + :param params: updated resource object source + :type params: dictionary + + :param body: updated resource object attributes + :type body: dictionary + + :returns: JSON response from HTTP API request + """ + if params is None: + params = {} + return APIClient().submit('PUT', cls._class_url + "/" + str(id), body, **params) + + +class DeletableAPIResource(object): + """ + Deletable API Resource + """ + @classmethod + def delete(cls, id, **params): + """ + Delete an API resource object + + :param id: resource object to delete + :type id: id + + :returns: JSON response from HTTP API request + """ + return APIClient().submit('DELETE', cls._class_url + "/" + str(id), **params) + + +class GetableAPIResource(object): + """ + Getable API Resource + """ + @classmethod + def get(cls, id, **params): + """ + Get information about an API resource object + + :param id: resource object id to retrieve + :type id: id + + :param params: parameters to filter API resource stream + :type params: dictionary + + :returns: JSON response from HTTP API request + """ + return APIClient().submit('GET', cls._class_url + "/" + str(id), **params) + + +class ListableAPIResource(object): + """ + Listable API Resource + """ + @classmethod + def get_all(cls, **params): + """ + List API resource objects + + :param params: parameters to filter API resource stream + :type params: dictionary + + :returns: JSON response from HTTP API request + """ + return APIClient().submit('GET', cls._class_url, **params) + + +class SearchableAPIResource(object): + """ + Fork of ListableAPIResource class with different method names + """ + @classmethod + def _search(cls, **params): + """ + Query an API resource stream + + :param params: parameters to filter API resource stream + :type params: dictionary + + :returns: JSON response from HTTP API request + """ + return APIClient().submit('GET', cls._class_url, **params) + + +class ActionAPIResource(object): + """ + Actionable API Resource + """ + @classmethod + def _trigger_class_action(cls, method, name, id=None, **params): + """ + Trigger an action + + :param method: HTTP method to use to contact API endpoint + :type method: HTTP method string + + :param name: action name + :type name: string + + :param id: trigger the action for the specified resource object + :type id: id + + :param params: action parameters + :type params: dictionary + + :returns: JSON response from HTTP API request + """ + if id is None: + return APIClient().submit(method, cls._class_url + "/" + name, params) + else: + return APIClient().submit(method, cls._class_url + "/" + str(id) + "/" + name, params) + + @classmethod + def _trigger_action(cls, method, name, id=None, **params): + """ + Trigger an action + + :param method: HTTP method to use to contact API endpoint + :type method: HTTP method string + + :param name: action name + :type name: string + + :param id: trigger the action for the specified resource object + :type id: id + + :param params: action parameters + :type params: dictionary + + :returns: JSON response from HTTP API request + """ + if id is None: + return APIClient().submit(method, name, params) + else: + return APIClient().submit(method, name + "/" + str(id), params) diff --git a/datadog/api/screenboards.py b/datadog/api/screenboards.py index c8d2df031..a5cd3c64b 100644 --- a/datadog/api/screenboards.py +++ b/datadog/api/screenboards.py @@ -1,4 +1,4 @@ -from datadog.api.base import GetableAPIResource, CreateableAPIResource, \ +from datadog.api.resources import GetableAPIResource, CreateableAPIResource, \ UpdatableAPIResource, DeletableAPIResource, ActionAPIResource, ListableAPIResource diff --git a/datadog/api/service_checks.py b/datadog/api/service_checks.py index 6bb33c63e..524f277e0 100644 --- a/datadog/api/service_checks.py +++ b/datadog/api/service_checks.py @@ -1,6 +1,6 @@ -from datadog.api.base import ActionAPIResource -from datadog.api.exceptions import ApiError from datadog.api.constants import CheckStatus +from datadog.api.exceptions import ApiError +from datadog.api.resources import ActionAPIResource class ServiceCheck(ActionAPIResource): diff --git a/datadog/api/tags.py b/datadog/api/tags.py index d03369f22..213168e08 100644 --- a/datadog/api/tags.py +++ b/datadog/api/tags.py @@ -1,4 +1,4 @@ -from datadog.api.base import CreateableAPIResource, UpdatableAPIResource,\ +from datadog.api.resources import CreateableAPIResource, UpdatableAPIResource,\ DeletableAPIResource, GetableAPIResource, ListableAPIResource diff --git a/datadog/api/timeboards.py b/datadog/api/timeboards.py index f8877abab..de6d444a4 100644 --- a/datadog/api/timeboards.py +++ b/datadog/api/timeboards.py @@ -1,4 +1,4 @@ -from datadog.api.base import GetableAPIResource, CreateableAPIResource, \ +from datadog.api.resources import GetableAPIResource, CreateableAPIResource, \ UpdatableAPIResource, ListableAPIResource, DeletableAPIResource diff --git a/datadog/api/users.py b/datadog/api/users.py index cd6ac9554..57e7588bd 100644 --- a/datadog/api/users.py +++ b/datadog/api/users.py @@ -1,4 +1,4 @@ -from datadog.api.base import ActionAPIResource, GetableAPIResource, \ +from datadog.api.resources import ActionAPIResource, GetableAPIResource, \ CreateableAPIResource, UpdatableAPIResource, ListableAPIResource, \ DeletableAPIResource diff --git a/tests/unit/api/helper.py b/tests/unit/api/helper.py index ed7477a96..dc46edaff 100644 --- a/tests/unit/api/helper.py +++ b/tests/unit/api/helper.py @@ -7,9 +7,9 @@ # datadog from datadog import initialize, api -from datadog.api.base import CreateableAPIResource, UpdatableAPIResource, DeletableAPIResource,\ - GetableAPIResource, ListableAPIResource, ActionAPIResource from datadog.api.exceptions import ApiError +from datadog.api.resources import CreateableAPIResource, UpdatableAPIResource, DeletableAPIResource,\ + GetableAPIResource, ListableAPIResource, ActionAPIResource from datadog.util.compat import iteritems, json From dbb42851990130495d5e90b9c4ade2ea23e6e083 Mon Sep 17 00:00:00 2001 From: Yann Mahe Date: Wed, 9 Dec 2015 11:29:03 -0500 Subject: [PATCH 4/6] [api] `urlfetch` based http client Create a new HTTP client, `URLFetchClient`, based on `urlfetch` Google App Engine 3p module. --- datadog/api/api_client.py | 30 +++++----- datadog/api/exceptions.py | 26 +++++++- datadog/api/http_client.py | 119 ++++++++++++++++++++++++++++++------- 3 files changed, 137 insertions(+), 38 deletions(-) diff --git a/datadog/api/api_client.py b/datadog/api/api_client.py index 973624d25..2ae124738 100644 --- a/datadog/api/api_client.py +++ b/datadog/api/api_client.py @@ -6,7 +6,7 @@ from datadog.api import _api_version, _max_timeouts, _backoff_period from datadog.api.exceptions import ClientError, ApiError, HttpBackoff, \ HttpTimeout, ApiNotInitialized -from datadog.api.http_client import RequestClient +from datadog.api.http_client import get_http_client from datadog.util.compat import json, is_p3k @@ -15,6 +15,8 @@ class APIClient(object): """ + Datadog API client: format and submit API calls to Datadog. + Embeds a HTTP client. """ # HTTP transport parameters _backoff_period = _backoff_period @@ -25,17 +27,9 @@ class APIClient(object): def __init__(self): """ - Instantiate a HTTP Client. + Instantiate the client. Plug a HTTP client. """ - self._http_client = self._get_http_client() - - @staticmethod - def _get_http_client(): - """ - Resolve the appropriate HTTP client based on priority and user environment. - """ - - return RequestClient + self._http_client = get_http_client() def submit(self, method, path, body=None, attach_host_name=False, response_formatter=None, error_formatter=None, **params): @@ -84,9 +78,6 @@ def submit(self, method, path, body=None, attach_host_name=False, response_forma if _application_key: params['application_key'] = _application_key - # Construct the url - url = "%s/api/%s/%s" % (_api_host, self._api_version, path.lstrip("/")) - # Attach host name to body if attach_host_name and body: # Is it a 'series' list of objects ? @@ -109,6 +100,13 @@ def submit(self, method, path, body=None, attach_host_name=False, response_forma body = json.dumps(body) headers['Content-Type'] = 'application/json' + # Construct the URL + url = "{api_host}/api/{api_version}/{path}".format( + api_host=_api_host, + api_version=self._api_version, + path=path.lstrip("/"), + ) + # Process requesting start_time = time.time() @@ -145,6 +143,9 @@ def submit(self, method, path, body=None, attach_host_name=False, response_forma else: return response_formatter(response_obj) + except HttpTimeout: + self._timeout_counter += 1 + raise except ClientError as e: if _mute: log.error(str(e)) @@ -206,6 +207,7 @@ def _should_submit(cls): @classmethod def _backoff_status(cls): """ + Get a backoff report, i.e. backoff total and remaining time. """ now = time.time() backed_off_time = now - cls._backoff_timestamp diff --git a/datadog/api/exceptions.py b/datadog/api/exceptions.py index 8cd7f5763..b0d8e26c7 100644 --- a/datadog/api/exceptions.py +++ b/datadog/api/exceptions.py @@ -1,11 +1,11 @@ """ -Module containing all the possible exceptions that `datadog` can raise. +API & HTTP Clients exceptions. """ class ClientError(Exception): """ - When HTTP connection to Datadog endpoint is not possible. + HTTP connection to Datadog endpoint is not possible. """ def __init__(self, method, url, exception): message = u"Could not request {method} {url}: {exception}. "\ @@ -39,8 +39,28 @@ def __init__(self, backoff_period): super(HttpBackoff, self).__init__(message) +class HTTPError(Exception): + """ + Datadog returned a HTTP error. + """ + def __init__(self, status_code=None, reason=None): + reason = u" - {reason}".format(reason=reason) if reason else u"" + message = u"Datadog returned a bad HTTP response code: {status_code}{reason}. "\ + u"Please try again later. "\ + u"If the problem persists, please contact support@datadoghq.com".format( + status_code=status_code, + reason=reason, + ) + + super(HTTPError, self).__init__(message) + + class ApiError(Exception): - "Datadog API is returning an error" + """ + Datadog returned an API error (known HTTPError). + + Matches the following status codes: 400, 403, 404, 409. + """ class ApiNotInitialized(Exception): diff --git a/datadog/api/http_client.py b/datadog/api/http_client.py index f95a65b00..a0f169ba6 100644 --- a/datadog/api/http_client.py +++ b/datadog/api/http_client.py @@ -1,18 +1,20 @@ """ Available HTTP Client for Datadog API client. -1. Priority to `requests` -2. Fall back to `urlfetch` module on Google App Engine +Priority: +1. `requests` 3p module +2. `urlfetch` 3p module - Google App Engine only """ # stdlib import logging - -# 3p -import requests +import urllib # datadog -from datadog.api.exceptions import ClientError, HttpTimeout +from datadog.api.exceptions import ClientError, HTTPError, HttpTimeout +requests = None +urlfetch = None +urlfetch_errors = None log = logging.getLogger('dd.datadogpy') @@ -21,11 +23,19 @@ class HTTPClient(object): """ An abstract generic HTTP client. Subclasses must implement the `request` methods. """ - _CORE = NotImplemented - @classmethod def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries): """ + Main method to be implemented by HTTP clients. + + The returned data structure has the following fields: + * `content`: string containing the response from the server + * `status_code`: HTTP status code returned by the server + + Can raise the following exceptions: + * `ClientError`: server cannot be contacted + * `HttpTimeout`: connection timed out + * `HTTPError`: unexpected HTTP response code """ raise NotImplementedError( u"Must be implemented by HTTPClient subclasses." @@ -46,31 +56,24 @@ def request(cls, method, url, headers, params, data, timeout, proxies, verify, m s.mount('https://', http_adapter) try: - # Request result = s.request( - method, - url, - headers=headers, - params=params, - data=data, + method, url, + headers=headers, params=params, data=data, timeout=timeout, - proxies=proxies, - verify=verify) + proxies=proxies, verify=verify) - # Raise on status result.raise_for_status() except requests.ConnectionError as e: raise ClientError(method, url, e) - except requests.exceptions.Timeout as e: - cls._timeout_counter += 1 + except requests.exceptions.Timeout: raise HttpTimeout(method, url, timeout) except requests.exceptions.HTTPError as e: if e.response.status_code in (400, 403, 404, 409): # This gets caught afterwards and raises an ApiError exception pass else: - raise + raise HTTPError(e.response.status_code, result.reason) except TypeError as e: raise TypeError( u"Your installed version of `requests` library seems not compatible with" @@ -85,4 +88,78 @@ class URLFetchClient(HTTPClient): """ HTTP client based on Google App Engine `urlfetch` module. """ - pass + @classmethod + def request(cls, method, url, headers, params, data, timeout, proxies, verify, max_retries): + """ + Wrapper around `urlfetch.fetch` method. + + TO IMPLEMENT: + * `max_retries` + """ + # No local certificate file can be used on Google App Engine + validate_certificate = True if verify else False + + # Encode parameters in the url + url_with_params = "{url}?{params}".format( + url=url, + params=urllib.urlencode(params) + ) + + try: + result = urlfetch.fetch( + url=url_with_params, + method=method, + headers=headers, + validate_certificate=validate_certificate, + deadline=timeout, + payload=data + ) + + cls.raise_on_status(result) + + except urlfetch.DownloadError as e: + raise ClientError(method, url, e) + except urlfetch_errors.DeadlineExceededError: + raise HttpTimeout(method, url, timeout) + + return result + + @classmethod + def raise_on_status(cls, result): + """ + Raise on HTTP status code errors. + """ + status_code = result.status_code + + if (status_code / 100) != 2: + if status_code in (400, 403, 404, 409): + pass + else: + raise HTTPError(status_code) + + +HTTP_CLIENTS = [RequestClient, URLFetchClient] + + +def get_http_client(): + """ + Return the appropriate HTTP client based the defined priority and user environment. + """ + global requests + global urlfetch + global urlfetch_errors + + try: + import requests + return RequestClient + except ImportError: + pass + + try: + from google.appengine.api import urlfetch, urlfetch_errors + return URLFetchClient + except ImportError: + raise ImportError( + u"Datadog API client was unable to resolve a HTTP client. " + u" Please install `requests` library." + ) From 49cf300cdf3c69157855e509b8712bf597a349c7 Mon Sep 17 00:00:00 2001 From: Yann Mahe Date: Wed, 9 Dec 2015 11:36:56 -0500 Subject: [PATCH 5/6] [util] don't raise on hostname resolution failures `hostname` is an optional API parameter. Thus, `initialize` method should not raise when unable to resolve it. --- datadog/util/hostname.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/datadog/util/hostname.py b/datadog/util/hostname.py index 278c79c3c..33f90ab79 100644 --- a/datadog/util/hostname.py +++ b/datadog/util/hostname.py @@ -101,12 +101,12 @@ def _get_hostname_unix(): hostname = socket_hostname if hostname is None: - log.critical("Unable to reliably determine host name. You can define one" - " in datadog.conf or in your hosts file") - raise Exception("Unable to reliably determine host name. You can define" - " one in datadog.conf or in your hosts file") - else: - return hostname + log.warning( + u"Unable to reliably determine host name. You can define one in your `hosts` file, " + u"or in `datadog.conf` file if you have Datadog Agent installed." + ) + + return hostname def get_ec2_instance_id(): From 844e0b8232a1d6c454e4a5aaca808e11d01a0315 Mon Sep 17 00:00:00 2001 From: Yann Mahe Date: Thu, 26 May 2016 17:16:22 -0400 Subject: [PATCH 6/6] [api] static `APIClient` class Following @degemer's feedback: * Make `APIClient` class 100% static * Rewrite the `resolve_http_client` logic in the `HTTPClient` class --- datadog/api/api_client.py | 30 +++++++++++++++---------- datadog/api/http_client.py | 45 +++++++++++++++++++------------------- datadog/api/resources.py | 36 +++++++++++++++--------------- 3 files changed, 59 insertions(+), 52 deletions(-) diff --git a/datadog/api/api_client.py b/datadog/api/api_client.py index 2ae124738..335bd22fc 100644 --- a/datadog/api/api_client.py +++ b/datadog/api/api_client.py @@ -6,7 +6,7 @@ from datadog.api import _api_version, _max_timeouts, _backoff_period from datadog.api.exceptions import ClientError, ApiError, HttpBackoff, \ HttpTimeout, ApiNotInitialized -from datadog.api.http_client import get_http_client +from datadog.api.http_client import resolve_http_client from datadog.util.compat import json, is_p3k @@ -25,13 +25,21 @@ class APIClient(object): _timeout_counter = 0 _api_version = _api_version - def __init__(self): + # Plugged HTTP client + _http_client = None + + @classmethod + def _get_http_client(cls): """ - Instantiate the client. Plug a HTTP client. + Getter for the embedded HTTP client. """ - self._http_client = get_http_client() + if not cls._http_client: + cls._http_client = resolve_http_client() - def submit(self, method, path, body=None, attach_host_name=False, response_formatter=None, + return cls._http_client + + @classmethod + def submit(cls, method, path, body=None, attach_host_name=False, response_formatter=None, error_formatter=None, **params): """ Make an HTTP API request @@ -61,8 +69,8 @@ def submit(self, method, path, body=None, attach_host_name=False, response_forma """ try: # Check if it's ok to submit - if not self._should_submit(): - _, backoff_time_left = self._backoff_status() + if not cls._should_submit(): + _, backoff_time_left = cls._backoff_status() raise HttpBackoff(backoff_time_left) # Import API, User and HTTP settings @@ -103,14 +111,14 @@ def submit(self, method, path, body=None, attach_host_name=False, response_forma # Construct the URL url = "{api_host}/api/{api_version}/{path}".format( api_host=_api_host, - api_version=self._api_version, + api_version=cls._api_version, path=path.lstrip("/"), ) # Process requesting start_time = time.time() - result = self._http_client.request( + result = cls._get_http_client().request( method=method, url=url, headers=headers, params=params, data=body, timeout=_timeout, max_retries=_max_retries, @@ -120,7 +128,7 @@ def submit(self, method, path, body=None, attach_host_name=False, response_forma # Request succeeded: log it and reset the timeout counter duration = round((time.time() - start_time) * 1000., 4) log.info("%s %s %s (%sms)" % (result.status_code, method, url, duration)) - self._timeout_counter = 0 + cls._timeout_counter = 0 # Format response content content = result.content @@ -144,7 +152,7 @@ def submit(self, method, path, body=None, attach_host_name=False, response_forma return response_formatter(response_obj) except HttpTimeout: - self._timeout_counter += 1 + cls._timeout_counter += 1 raise except ClientError as e: if _mute: diff --git a/datadog/api/http_client.py b/datadog/api/http_client.py index a0f169ba6..2b67829c0 100644 --- a/datadog/api/http_client.py +++ b/datadog/api/http_client.py @@ -9,12 +9,20 @@ import logging import urllib +# 3p +try: + import requests +except ImportError: + requests = None + +try: + from google.appengine.api import urlfetch, urlfetch_errors +except ImportError: + urlfetch, urlfetch_errors = None, None + # datadog from datadog.api.exceptions import ClientError, HTTPError, HttpTimeout -requests = None -urlfetch = None -urlfetch_errors = None log = logging.getLogger('dd.datadogpy') @@ -138,28 +146,19 @@ def raise_on_status(cls, result): raise HTTPError(status_code) -HTTP_CLIENTS = [RequestClient, URLFetchClient] - - -def get_http_client(): +def resolve_http_client(): """ - Return the appropriate HTTP client based the defined priority and user environment. + Resolve an appropriate HTTP client based the defined priority and user environment. """ - global requests - global urlfetch - global urlfetch_errors - - try: - import requests + if requests: + log.debug(u"Use `requests` based HTTP client.") return RequestClient - except ImportError: - pass - try: - from google.appengine.api import urlfetch, urlfetch_errors + if urlfetch and urlfetch_errors: + log.debug(u"Use `urlfetch` based HTTP client.") return URLFetchClient - except ImportError: - raise ImportError( - u"Datadog API client was unable to resolve a HTTP client. " - u" Please install `requests` library." - ) + + raise ImportError( + u"Datadog API client was unable to resolve a HTTP client. " + u" Please install `requests` library." + ) diff --git a/datadog/api/resources.py b/datadog/api/resources.py index 6a0cf264f..9dbe8a0ef 100644 --- a/datadog/api/resources.py +++ b/datadog/api/resources.py @@ -40,13 +40,13 @@ def create(cls, attach_host_name=False, method='POST', id=None, params=None, **b if params is None: params = {} if method == 'GET': - return APIClient().submit('GET', cls._class_url, **body) + return APIClient.submit('GET', cls._class_url, **body) if id is None: - return APIClient().submit('POST', cls._class_url, body, - attach_host_name=attach_host_name, **params) + return APIClient.submit('POST', cls._class_url, body, + attach_host_name=attach_host_name, **params) else: - return APIClient().submit('POST', cls._class_url + "/" + str(id), body, - attach_host_name=attach_host_name, **params) + return APIClient.submit('POST', cls._class_url + "/" + str(id), body, + attach_host_name=attach_host_name, **params) class SendableAPIResource(object): @@ -70,11 +70,11 @@ def send(cls, attach_host_name=False, id=None, **body): :returns: JSON response from HTTP API request """ if id is None: - return APIClient().submit('POST', cls._class_url, body, - attach_host_name=attach_host_name) + return APIClient.submit('POST', cls._class_url, body, + attach_host_name=attach_host_name) else: - return APIClient().submit('POST', cls._class_url + "/" + str(id), body, - attach_host_name=attach_host_name) + return APIClient.submit('POST', cls._class_url + "/" + str(id), body, + attach_host_name=attach_host_name) class UpdatableAPIResource(object): @@ -96,7 +96,7 @@ def update(cls, id, params=None, **body): """ if params is None: params = {} - return APIClient().submit('PUT', cls._class_url + "/" + str(id), body, **params) + return APIClient.submit('PUT', cls._class_url + "/" + str(id), body, **params) class DeletableAPIResource(object): @@ -113,7 +113,7 @@ def delete(cls, id, **params): :returns: JSON response from HTTP API request """ - return APIClient().submit('DELETE', cls._class_url + "/" + str(id), **params) + return APIClient.submit('DELETE', cls._class_url + "/" + str(id), **params) class GetableAPIResource(object): @@ -133,7 +133,7 @@ def get(cls, id, **params): :returns: JSON response from HTTP API request """ - return APIClient().submit('GET', cls._class_url + "/" + str(id), **params) + return APIClient.submit('GET', cls._class_url + "/" + str(id), **params) class ListableAPIResource(object): @@ -150,7 +150,7 @@ def get_all(cls, **params): :returns: JSON response from HTTP API request """ - return APIClient().submit('GET', cls._class_url, **params) + return APIClient.submit('GET', cls._class_url, **params) class SearchableAPIResource(object): @@ -167,7 +167,7 @@ def _search(cls, **params): :returns: JSON response from HTTP API request """ - return APIClient().submit('GET', cls._class_url, **params) + return APIClient.submit('GET', cls._class_url, **params) class ActionAPIResource(object): @@ -194,9 +194,9 @@ def _trigger_class_action(cls, method, name, id=None, **params): :returns: JSON response from HTTP API request """ if id is None: - return APIClient().submit(method, cls._class_url + "/" + name, params) + return APIClient.submit(method, cls._class_url + "/" + name, params) else: - return APIClient().submit(method, cls._class_url + "/" + str(id) + "/" + name, params) + return APIClient.submit(method, cls._class_url + "/" + str(id) + "/" + name, params) @classmethod def _trigger_action(cls, method, name, id=None, **params): @@ -218,6 +218,6 @@ def _trigger_action(cls, method, name, id=None, **params): :returns: JSON response from HTTP API request """ if id is None: - return APIClient().submit(method, name, params) + return APIClient.submit(method, name, params) else: - return APIClient().submit(method, name + "/" + str(id), params) + return APIClient.submit(method, name + "/" + str(id), params)