-
Notifications
You must be signed in to change notification settings - Fork 331
/
Copy path_http_client.py
150 lines (118 loc) · 5.47 KB
/
_http_client.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Internal HTTP client module.
This module provides utilities for making HTTP calls using the requests library.
"""
from google.auth import transport # type: ignore
import requests
from requests.packages.urllib3.util import retry # type: ignore # pylint: disable=import-error
if hasattr(retry.Retry.DEFAULT, 'allowed_methods'):
_ANY_METHOD = {'allowed_methods': None}
else:
_ANY_METHOD = {'method_whitelist': None}
# Default retry configuration: Retries once on low-level connection and socket read errors.
# Retries up to 4 times on HTTP 500 and 503 errors, with exponential backoff. Returns the
# last response upon exhausting all retries.
DEFAULT_RETRY_CONFIG = retry.Retry(
connect=1, read=1, status=4, status_forcelist=[500, 503],
raise_on_status=False, backoff_factor=0.5, **_ANY_METHOD)
DEFAULT_TIMEOUT_SECONDS = 120
class HttpClient:
"""Base HTTP client used to make HTTP calls.
HttpClient maintains an HTTP session, and handles request authentication and retries if
necessary.
"""
def __init__(
self, credential=None, session=None, base_url='', headers=None,
retries=DEFAULT_RETRY_CONFIG, timeout=DEFAULT_TIMEOUT_SECONDS):
"""Creates a new HttpClient instance from the provided arguments.
If a credential is provided, initializes a new HTTP session authorized with it. If neither
a credential nor a session is provided, initializes a new unauthorized session.
Args:
credential: A Google credential that can be used to authenticate requests (optional).
session: A custom HTTP session (optional).
base_url: A URL prefix to be added to all outgoing requests (optional).
headers: A map of headers to be added to all outgoing requests (optional).
retries: A urllib retry configuration. Default settings would retry once for low-level
connection and socket read errors, and up to 4 times for HTTP 500 and 503 errors.
Pass a False value to disable retries (optional).
timeout: HTTP timeout in seconds. Defaults to 120 seconds when not specified. Set to
None to disable timeouts (optional).
"""
if credential:
self._session = transport.requests.AuthorizedSession(credential)
elif session:
self._session = session
else:
self._session = requests.Session() # pylint: disable=redefined-variable-type
if headers:
self._session.headers.update(headers)
if retries:
self._session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
self._session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
self._base_url = base_url
self._timeout = timeout
@property
def session(self):
return self._session
@property
def base_url(self):
return self._base_url
@property
def timeout(self):
return self._timeout
def parse_body(self, resp):
raise NotImplementedError
def request(self, method, url, **kwargs):
"""Makes an HTTP call using the Python requests library.
This is the sole entry point to the requests library. All other helper methods in this
class call this method to send HTTP requests out. Refer to
http://docs.python-requests.org/en/master/api/ for more information on supported options
and features.
Args:
method: HTTP method name as a string (e.g. get, post).
url: URL of the remote endpoint.
**kwargs: An additional set of keyword arguments to be passed into the requests API
(e.g. json, params, timeout).
Returns:
Response: An HTTP response object.
Raises:
RequestException: Any requests exceptions encountered while making the HTTP call.
"""
if 'timeout' not in kwargs:
kwargs['timeout'] = self.timeout
resp = self._session.request(method, self.base_url + url, **kwargs)
resp.raise_for_status()
return resp
def headers(self, method, url, **kwargs):
resp = self.request(method, url, **kwargs)
return resp.headers
def body_and_response(self, method, url, **kwargs):
resp = self.request(method, url, **kwargs)
return self.parse_body(resp), resp
def body(self, method, url, **kwargs):
resp = self.request(method, url, **kwargs)
return self.parse_body(resp)
def headers_and_body(self, method, url, **kwargs):
resp = self.request(method, url, **kwargs)
return resp.headers, self.parse_body(resp)
def close(self):
self._session.close()
self._session = None
class JsonHttpClient(HttpClient):
"""An HTTP client that parses response messages as JSON."""
def __init__(self, **kwargs):
HttpClient.__init__(self, **kwargs)
def parse_body(self, resp):
return resp.json()