Add forecast classes - #1115
Merged
Merged
GitHub Actions / Petals / Unit Test Results (3.11)
failed
Jun 8, 2026 in 0s
3 fail, 3 skipped, 338 pass in 7m 0s
Annotations
Check warning on line 0 in climada_petals.entity.exposures.test.test_black_marble.TestEconIndices
github-actions / Petals / Unit Test Results (3.11)
test_fill_econ_indicators_na_pass (climada_petals.entity.exposures.test.test_black_marble.TestEconIndices) failed
climada_petals/tests_xml/tests.xml [took 30s]
Raw output
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.worldbank.org', port=443): Read timed out. (read timeout=30)
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7ff9bd83ad10>
conn = <HTTPSConnection(host='api.worldbank.org', port=443) at 0x7ff9bd8100d0>
method = 'GET'
url = '/v2/countries/ZMB/indicators/NY.GDP.MKTP.CD?format=json&page=1'
body = None
headers = {'User-Agent': 'python-requests/2.34.2', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': 'keep-alive'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=30, read=30, total=None), chunked = False
response_conn = <HTTPSConnection(host='api.worldbank.org', port=443) at 0x7ff9bd8100d0>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
raise new_e
# conn.request() calls http.client.*.request, not the method in
# urllib3.request. It also calls makefile (recv) on the socket.
try:
conn.request(
method,
url,
body=body,
headers=headers,
chunked=chunked,
preload_content=preload_content,
decode_content=decode_content,
enforce_content_length=enforce_content_length,
)
# We are swallowing BrokenPipeError (errno.EPIPE) since the server is
# legitimately able to close the connection after sending a valid response.
# With this behaviour, the received response is still readable.
except BrokenPipeError:
pass
except OSError as e:
# MacOS/Linux
# EPROTOTYPE and ECONNRESET are needed on macOS
# https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
# Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
raise
# Reset the timeout for the recv() on the socket
read_timeout = timeout_obj.read_timeout
if not conn.is_closed:
# In Python 3 socket.py will catch EAGAIN and return None when you
# try and read into the file pointer created by http.client, which
# instead raises a BadStatusLine exception. Instead of catching
# the exception and assuming all BadStatusLine exceptions are read
# timeouts, check for a zero timeout before making the request.
if read_timeout == 0:
raise ReadTimeoutError(
self, url, f"Read timed out. (read timeout={read_timeout})"
)
conn.timeout = read_timeout
# Receive the response from the server
try:
> response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:534:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:571: in getresponse
httplib_response = super().getresponse()
^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/http/client.py:1415: in getresponse
response.begin()
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/http/client.py:330: in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/http/client.py:291: in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/socket.py:718: in readinto
return self._sock.recv_into(b)
^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1314: in recv_into
return self.read(nbytes, buffer)
^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, len = 8192
buffer = <memory at 0x7ff9c13fc880>
def read(self, len=1024, buffer=None):
"""Read up to LEN bytes and return them.
Return zero-length string on EOF."""
self._checkClosed()
if self._sslobj is None:
raise ValueError("Read on closed or unwrapped SSL socket.")
try:
if buffer is not None:
> return self._sslobj.read(len, buffer)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E TimeoutError: The read operation timed out
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1166: TimeoutError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7ff9c9698d50>
request = <PreparedRequest [GET]>, stream = False, timeout = 30, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:498: in increment
raise reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/util.py:39: in reraise
raise value
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788: in urlopen
response = self._make_request(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:536: in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7ff9bd83ad10>
err = TimeoutError('The read operation timed out')
url = '/v2/countries/ZMB/indicators/NY.GDP.MKTP.CD?format=json&page=1'
timeout_value = 30
def _raise_timeout(
self,
err: BaseSSLError | OSError | SocketTimeout,
url: str,
timeout_value: _TYPE_TIMEOUT | None,
) -> None:
"""Is the error actually a timeout? Will raise a ReadTimeout or pass"""
if isinstance(err, SocketTimeout):
> raise ReadTimeoutError(
self, url, f"Read timed out. (read timeout={timeout_value})"
) from err
E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='api.worldbank.org', port=443): Read timed out. (read timeout=30)
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError
During handling of the above exception, another exception occurred:
self = <climada_petals.entity.exposures.test.test_black_marble.TestEconIndices testMethod=test_fill_econ_indicators_na_pass>
def test_fill_econ_indicators_na_pass(self):
"""Test fill_econ_indicators with '' inputs."""
ref_year = 2019
country_isos = {'CHE': [1, 'Switzerland', 'che_geom'],
'ZMB': [2, 'Zambia', 'zmb_geom']
}
gdp = {'CHE': 1.2 * 1e20, 'ZMB': ''}
inc_grp = {'CHE': '', 'ZMB': 4}
kwargs = {'gdp': gdp, 'inc_grp': inc_grp}
> fill_econ_indicators(ref_year, country_isos, SHP_FILE, **kwargs)
climada_petals/entity/exposures/test/test_black_marble.py:301:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/black_marble.py:270: in fill_econ_indicators
_, gdp_val = gdp(cntry_iso, ref_year, shp_file)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:171: in gdp
close_year, close_val = world_bank(cntry_iso, ref_year, "NY.GDP.MKTP.CD")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:277: in world_bank
cntry_gdp = download_world_bank_indicator(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:209: in download_world_bank_indicator
response = requests.get(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7ff9c9698d50>
request = <PreparedRequest [GET]>, stream = False, timeout = 30, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
raise SSLError(e, request=request)
raise ConnectionError(e, request=request)
except ClosedPoolError as e:
raise ConnectionError(e, request=request)
except _ProxyError as e:
raise ProxyError(e)
except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
> raise ReadTimeout(e, request=request)
E requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.worldbank.org', port=443): Read timed out. (read timeout=30)
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:742: ReadTimeout
Check warning on line 0 in climada_petals.entity.exposures.test.test_black_marble.TestEconIndices
github-actions / Petals / Unit Test Results (3.11)
test_fill_econ_indicators_pass (climada_petals.entity.exposures.test.test_black_marble.TestEconIndices) failed
climada_petals/tests_xml/tests.xml [took 30s]
Raw output
requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.worldbank.org', port=443): Read timed out. (read timeout=30)
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7ff9bd9cf450>
conn = <HTTPSConnection(host='api.worldbank.org', port=443) at 0x7ff9bd9cf250>
method = 'GET'
url = '/v2/countries/ZMB/indicators/NY.GDP.MKTP.CD?format=json&page=1'
body = None
headers = {'User-Agent': 'python-requests/2.34.2', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': 'keep-alive'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=30, read=30, total=None), chunked = False
response_conn = <HTTPSConnection(host='api.worldbank.org', port=443) at 0x7ff9bd9cf250>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
raise new_e
# conn.request() calls http.client.*.request, not the method in
# urllib3.request. It also calls makefile (recv) on the socket.
try:
conn.request(
method,
url,
body=body,
headers=headers,
chunked=chunked,
preload_content=preload_content,
decode_content=decode_content,
enforce_content_length=enforce_content_length,
)
# We are swallowing BrokenPipeError (errno.EPIPE) since the server is
# legitimately able to close the connection after sending a valid response.
# With this behaviour, the received response is still readable.
except BrokenPipeError:
pass
except OSError as e:
# MacOS/Linux
# EPROTOTYPE and ECONNRESET are needed on macOS
# https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/
# Condition changed later to emit ECONNRESET instead of only EPROTOTYPE.
if e.errno != errno.EPROTOTYPE and e.errno != errno.ECONNRESET:
raise
# Reset the timeout for the recv() on the socket
read_timeout = timeout_obj.read_timeout
if not conn.is_closed:
# In Python 3 socket.py will catch EAGAIN and return None when you
# try and read into the file pointer created by http.client, which
# instead raises a BadStatusLine exception. Instead of catching
# the exception and assuming all BadStatusLine exceptions are read
# timeouts, check for a zero timeout before making the request.
if read_timeout == 0:
raise ReadTimeoutError(
self, url, f"Read timed out. (read timeout={read_timeout})"
)
conn.timeout = read_timeout
# Receive the response from the server
try:
> response = conn.getresponse()
^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:534:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:571: in getresponse
httplib_response = super().getresponse()
^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/http/client.py:1415: in getresponse
response.begin()
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/http/client.py:330: in begin
version, status, reason = self._read_status()
^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/http/client.py:291: in _read_status
line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/socket.py:718: in readinto
return self._sock.recv_into(b)
^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1314: in recv_into
return self.read(nbytes, buffer)
^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, len = 8192
buffer = <memory at 0x7ff9c13fc400>
def read(self, len=1024, buffer=None):
"""Read up to LEN bytes and return them.
Return zero-length string on EOF."""
self._checkClosed()
if self._sslobj is None:
raise ValueError("Read on closed or unwrapped SSL socket.")
try:
if buffer is not None:
> return self._sslobj.read(len, buffer)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E TimeoutError: The read operation timed out
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1166: TimeoutError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7ff9bd9cfed0>
request = <PreparedRequest [GET]>, stream = False, timeout = 30, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:498: in increment
raise reraise(type(error), error, _stacktrace)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/util.py:39: in reraise
raise value
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788: in urlopen
response = self._make_request(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:536: in _make_request
self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7ff9bd9cf450>
err = TimeoutError('The read operation timed out')
url = '/v2/countries/ZMB/indicators/NY.GDP.MKTP.CD?format=json&page=1'
timeout_value = 30
def _raise_timeout(
self,
err: BaseSSLError | OSError | SocketTimeout,
url: str,
timeout_value: _TYPE_TIMEOUT | None,
) -> None:
"""Is the error actually a timeout? Will raise a ReadTimeout or pass"""
if isinstance(err, SocketTimeout):
> raise ReadTimeoutError(
self, url, f"Read timed out. (read timeout={timeout_value})"
) from err
E urllib3.exceptions.ReadTimeoutError: HTTPSConnectionPool(host='api.worldbank.org', port=443): Read timed out. (read timeout=30)
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:367: ReadTimeoutError
During handling of the above exception, another exception occurred:
self = <climada_petals.entity.exposures.test.test_black_marble.TestEconIndices testMethod=test_fill_econ_indicators_pass>
def test_fill_econ_indicators_pass(self):
"""Test fill_econ_indicators CHE, ZMB."""
ref_year = 2015
country_isos = {'CHE': [1, 'Switzerland', 'che_geom'],
'ZMB': [2, 'Zambia', 'zmb_geom']
}
> fill_econ_indicators(ref_year, country_isos, SHP_FILE)
climada_petals/entity/exposures/test/test_black_marble.py:264:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/black_marble.py:270: in fill_econ_indicators
_, gdp_val = gdp(cntry_iso, ref_year, shp_file)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:171: in gdp
close_year, close_val = world_bank(cntry_iso, ref_year, "NY.GDP.MKTP.CD")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:277: in world_bank
cntry_gdp = download_world_bank_indicator(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:209: in download_world_bank_indicator
response = requests.get(
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7ff9bd9cfed0>
request = <PreparedRequest [GET]>, stream = False, timeout = 30, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
raise SSLError(e, request=request)
raise ConnectionError(e, request=request)
except ClosedPoolError as e:
raise ConnectionError(e, request=request)
except _ProxyError as e:
raise ProxyError(e)
except (_SSLError, _HTTPError) as e:
if isinstance(e, _SSLError):
# This branch is for urllib3 versions earlier than v1.22
raise SSLError(e, request=request)
elif isinstance(e, ReadTimeoutError):
> raise ReadTimeout(e, request=request)
E requests.exceptions.ReadTimeout: HTTPSConnectionPool(host='api.worldbank.org', port=443): Read timed out. (read timeout=30)
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:742: ReadTimeout
Check warning on line 0 in climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery
github-actions / Petals / Unit Test Results (3.11)
test_get_data_overpass (climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery) failed
climada_petals/tests_xml/tests.xml [took 2m 9s]
Raw output
Exception: The Overpass API is consistently unavailable
self = <climada_petals.entity.exposures.osm_dataloader.OSMApiQuery object at 0x7ff9c134ee90>
query_clause = '[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
read_chunk_size = 100000, end_of_patience = 127
def _insistent_osm_api_query(self, query_clause, read_chunk_size=100000,
end_of_patience=127):
"""Runs a single Overpass API query through overpy.Overpass.query.
In case of failure it tries again after an ever increasing waiting period.
If the waiting period surpasses a given limit an exception is raised.
Parameters:
query_clause (str): the query
read_chunk_size (int): paramter passed over to overpy.Overpass.query
end_of_patience (int): upper limit for the next waiting period to proceed.
Returns:
result as returned by overpy.Overpass.query
"""
api = overpy.Overpass(read_chunk_size=read_chunk_size)
waiting_period = 1
while True:
try:
> return api.query(query_clause)
^^^^^^^^^^^^^^^^^^^^^^^
climada_petals/entity/exposures/osm_dataloader.py:102:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <overpy.Overpass object at 0x7ff9bfd22b90>
query = b'[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
def query(self, query: Union[bytes, str]) -> "Result":
"""
Query the Overpass API
:param query: The query string in Overpass QL
:return: The parsed result
"""
if not isinstance(query, bytes):
query = query.encode("utf-8")
retry_num: int = 0
retry_exceptions: List[exception.OverPyException] = []
do_retry: bool = True if self.max_retry_count > 0 else False
while retry_num <= self.max_retry_count:
if retry_num > 0:
time.sleep(self.retry_timeout)
retry_num += 1
try:
f = urlopen(self.url, query)
except HTTPError as e:
f = e
response = f.read(self.read_chunk_size)
while True:
data = f.read(self.read_chunk_size)
if len(data) == 0:
break
response = response + data
f.close()
current_exception: exception.OverPyException
if f.code == 200:
content_type = f.getheader("Content-Type")
if content_type == "application/json":
return self.parse_json(response)
if content_type == "application/osm3s+xml":
return self.parse_xml(response)
current_exception = exception.OverpassUnknownContentType(content_type)
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 400:
msgs: List[str] = []
for msg_raw in self._regex_extract_error_msg.finditer(response):
msg_clean_bytes = self._regex_remove_tag.sub(b"", msg_raw.group("msg"))
try:
msg = msg_clean_bytes.decode("utf-8")
except UnicodeDecodeError:
msg = repr(msg_clean_bytes)
msgs.append(msg)
current_exception = exception.OverpassBadRequest(
query,
msgs=msgs
)
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 429:
current_exception = exception.OverpassTooManyRequests()
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 504:
current_exception = exception.OverpassGatewayTimeout()
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
current_exception = exception.OverpassUnknownHTTPStatusCode(f.code)
if not do_retry:
> raise current_exception
E overpy.exception.OverpassUnknownHTTPStatusCode: Unknown/Unhandled status code: 406
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/overpy/__init__.py:195: OverpassUnknownHTTPStatusCode
During handling of the above exception, another exception occurred:
self = <climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery testMethod=test_get_data_overpass>
def test_get_data_overpass(self):
"""test methods of OSMApiQuery"""
area_bbox = (8.5327506, 47.368260, 8.5486078, 47.376877)
area_poly = shapely.geometry.Polygon(
[(8.5327506, 47.368260),
(8.5486078, 47.376877),
(8.5486078, 47.39)])
condition_building = '["building"]'
condition_church = '["amenity"="place_of_worship"]'
gdf1 = osm_dl.OSMApiQuery.from_bounding_box(
> area_bbox, condition_building).get_data_overpass()
^^^^^^^^^^^^^^^^^^^
climada_petals/entity/exposures/test/test_osm_dataloader.py:123:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/osm_dataloader.py:368: in get_data_overpass
result = self._insistent_osm_api_query(query_clause)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <climada_petals.entity.exposures.osm_dataloader.OSMApiQuery object at 0x7ff9c134ee90>
query_clause = '[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
read_chunk_size = 100000, end_of_patience = 127
def _insistent_osm_api_query(self, query_clause, read_chunk_size=100000,
end_of_patience=127):
"""Runs a single Overpass API query through overpy.Overpass.query.
In case of failure it tries again after an ever increasing waiting period.
If the waiting period surpasses a given limit an exception is raised.
Parameters:
query_clause (str): the query
read_chunk_size (int): paramter passed over to overpy.Overpass.query
end_of_patience (int): upper limit for the next waiting period to proceed.
Returns:
result as returned by overpy.Overpass.query
"""
api = overpy.Overpass(read_chunk_size=read_chunk_size)
waiting_period = 1
while True:
try:
return api.query(query_clause)
except overpy.exception.OverpassTooManyRequests:
if waiting_period < end_of_patience:
LOGGER.warning("""Too many Overpass API requests -
trying again in {waiting_period} seconds """)
else:
raise Exception("Overpass API is consistently unavailable")
except Exception as exc:
if waiting_period < end_of_patience:
LOGGER.warning(f"""{exc}
Trying again in {waiting_period} seconds""")
else:
> raise Exception(
"The Overpass API is consistently unavailable")
E Exception: The Overpass API is consistently unavailable
climada_petals/entity/exposures/osm_dataloader.py:114: Exception
Loading