From c79adecf7eb2e506d9d4bf8ff3bca75ba803acf3 Mon Sep 17 00:00:00 2001 From: Melvin Klein Date: Tue, 29 Oct 2024 14:11:58 +0100 Subject: [PATCH 1/7] mark Environment variable names as private --- src/stackit/core/configuration.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/stackit/core/configuration.py b/src/stackit/core/configuration.py index d639f03..65d5dcc 100644 --- a/src/stackit/core/configuration.py +++ b/src/stackit/core/configuration.py @@ -2,22 +2,22 @@ class EnvironmentVariables: - SERVICE_ACCOUNT_EMAIL_ENV = "STACKIT_SERVICE_ACCOUNT_EMAIL" - SERVICE_ACCOUNT_TOKEN_ENV = "STACKIT_SERVICE_ACCOUNT_TOKEN" # noqa: S105 false positive - SERVICE_ACCOUNT_KEY_PATH_ENV = "STACKIT_SERVICE_ACCOUNT_KEY_PATH" - PRIVATE_KEY_PATH_ENV = "STACKIT_PRIVATE_KEY_PATH" - TOKEN_BASEURL_ENV = "STACKIT_TOKEN_BASEURL" # noqa: S105 false positive - CREDENTIALS_PATH_ENV = "STACKIT_CREDENTIALS_PATH" - REGION_ENV = "STACKIT_REGION" + _SERVICE_ACCOUNT_EMAIL_ENV = "STACKIT_SERVICE_ACCOUNT_EMAIL" + _SERVICE_ACCOUNT_TOKEN_ENV = "STACKIT_SERVICE_ACCOUNT_TOKEN" # noqa: S105 false positive + _SERVICE_ACCOUNT_KEY_PATH_ENV = "STACKIT_SERVICE_ACCOUNT_KEY_PATH" + _PRIVATE_KEY_PATH_ENV = "STACKIT_PRIVATE_KEY_PATH" + _TOKEN_BASEURL_ENV = "STACKIT_TOKEN_BASEURL" # noqa: S105 false positive + _CREDENTIALS_PATH_ENV = "STACKIT_CREDENTIALS_PATH" + _REGION_ENV = "STACKIT_REGION" def __init__(self): - self.account_email = os.environ.get(self.SERVICE_ACCOUNT_EMAIL_ENV) - self.service_account_token = os.environ.get(self.SERVICE_ACCOUNT_TOKEN_ENV) - self.account_key_path = os.environ.get(self.SERVICE_ACCOUNT_KEY_PATH_ENV) - self.private_key_path = os.environ.get(self.PRIVATE_KEY_PATH_ENV) - self.token_baseurl = os.environ.get(self.TOKEN_BASEURL_ENV) - self.credentials_path = os.environ.get(self.CREDENTIALS_PATH_ENV) - self.region = os.environ.get(self.REGION_ENV) + self.account_email = os.environ.get(self._SERVICE_ACCOUNT_EMAIL_ENV) + self.service_account_token = os.environ.get(self._SERVICE_ACCOUNT_TOKEN_ENV) + self.account_key_path = os.environ.get(self._SERVICE_ACCOUNT_KEY_PATH_ENV) + self.private_key_path = os.environ.get(self._PRIVATE_KEY_PATH_ENV) + self.token_baseurl = os.environ.get(self._TOKEN_BASEURL_ENV) + self.credentials_path = os.environ.get(self._CREDENTIALS_PATH_ENV) + self.region = os.environ.get(self._REGION_ENV) class Configuration: From 90fbfd40151f5f43bab89b61ebf65af2700a5969 Mon Sep 17 00:00:00 2001 From: Melvin Klein Date: Wed, 30 Oct 2024 10:26:13 +0100 Subject: [PATCH 2/7] add wait functionality --- src/stackit/core/wait.py | 60 +++++++++++++++++++++++++++++ tests/core/test_auth.py | 5 +-- tests/core/test_wait.py | 83 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 src/stackit/core/wait.py create mode 100644 tests/core/test_wait.py diff --git a/src/stackit/core/wait.py b/src/stackit/core/wait.py new file mode 100644 index 0000000..fdb915a --- /dev/null +++ b/src/stackit/core/wait.py @@ -0,0 +1,60 @@ +import time +from http import HTTPStatus + +import signal +import contextlib +from typing import Any, Callable, Tuple, Union + + +class Wait: + + RetryHttpErrorStatusCodes = [HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT] + + def __init__( + self, + check_function: Callable[[None], Tuple[bool, Union[Exception, None], Union[int, None], Any]], + sleep_before_wait: int = 0, + throttle: int = 5, + timeout: int = 30, + temp_error_retry_limit: int = 5, + ) -> None: + if throttle == 0: + raise ValueError("throttle can't be 0") + self._check_function = check_function + self._sleep_before_wait = sleep_before_wait + self._throttle = throttle + self._timeout = timeout * 60 + self._temp_error_retry_limit = temp_error_retry_limit + self.retry_http_error_status_codes = [HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT] + + @staticmethod + def _timeout_handler(signum, frame): + raise TimeoutError("Wait has timed out") + + def wait(self) -> Any: + time.sleep(self._sleep_before_wait) + + retry_temp_error_counter = 0 + + signal.signal(signal.SIGALRM, Wait._timeout_handler) + signal.alarm(self._timeout) + + while True: + + done, error, code, result = self._check_function() + if error: + retry_temp_error_counter = self._handle_error(retry_temp_error_counter, error, code) + + if done: + return result + time.sleep(self._throttle) + + def _handle_error(self, retry_temp_error_counter: int, error, code: int): + + if code in self.retry_http_error_status_codes: + retry_temp_error_counter += 1 + if retry_temp_error_counter == self._temp_error_retry_limit: + raise error + return retry_temp_error_counter + else: + raise error diff --git a/tests/core/test_auth.py b/tests/core/test_auth.py index 384fe93..f48c07f 100644 --- a/tests/core/test_auth.py +++ b/tests/core/test_auth.py @@ -1,9 +1,8 @@ +import json from pathlib import Path, PurePath +from unittest.mock import Mock, mock_open, patch import pytest -import json -from unittest.mock import patch, mock_open, Mock - from requests.auth import HTTPBasicAuth from stackit.core.auth_methods.key_auth import KeyAuth diff --git a/tests/core/test_wait.py b/tests/core/test_wait.py new file mode 100644 index 0000000..781bca1 --- /dev/null +++ b/tests/core/test_wait.py @@ -0,0 +1,83 @@ +import time +from typing import Any, Callable, List, Tuple, Union + +import pytest + +from stackit.core.wait import Wait + + +def timeout_check_function() -> Tuple[bool, Union[Exception, None], Union[int, None], Any]: + time.sleep(9999) + return True, None, None, None + + +def create_check_function( + error_codes: Union[List[int], None], tries_to_success: int, correct_return: str +) -> Callable[[None], Tuple[bool, Union[Exception, None], Union[int, None], Any]]: + error_code_counter = 0 + tries_to_success_counter = 0 + + def check_function() -> Tuple[bool, Union[Exception, None], Union[int, None], Any]: + nonlocal error_code_counter + nonlocal tries_to_success_counter + + if error_codes and error_code_counter < len(error_codes): + code = error_codes[error_code_counter] + error_code_counter += 1 + return False, Exception("Some exception"), code, None + elif tries_to_success_counter < tries_to_success: + tries_to_success_counter += 1 + return False, None, 200, None + else: + return True, None, 200, correct_return + + return check_function + + +class TestWait: + def test_timeout_throws_timeouterror(self): + wait = Wait(timeout_check_function, timeout=1) + + with pytest.raises(TimeoutError, match="Wait has timed out"): + wait.wait() + + def test_throttle_0_throws_error(self): + with pytest.raises(ValueError, match="throttle can't be 0"): + wait = Wait(lambda: (True, None, None, None), throttle=0) + + @pytest.mark.parametrize( + "check_function", + [ + create_check_function([400], 3, "Shouldn't be returned"), + + ], + ) + def test_throws_for_no_retry_status_code(self, check_function): + wait = Wait(check_function) + with pytest.raises(Exception, match="Some exception"): + wait.wait() + + @pytest.mark.parametrize( + "correct_return,error_retry_limit,check_function", + [ + ( + "This was a triumph.", + 0, + create_check_function(None, 0, "This was a triumph."), + ), + ( + "I'm making a note here: HUGE SUCCESS.", + 3, + create_check_function( + [502, 504], 3, "I'm making a note here: HUGE SUCCESS."), + ), + ], + ) + def test_return_is_correct( + self, + correct_return: str, + error_retry_limit: int, + check_function: Callable[[None], Tuple[bool, Union[Exception, None], Union[int, None], Any]], + ): + wait = Wait(check_function, temp_error_retry_limit=error_retry_limit) + assert wait.wait() == correct_return From d95374dd301881c5eb45ff5bf3166a49645851a4 Mon Sep 17 00:00:00 2001 From: Melvin Klein Date: Thu, 31 Oct 2024 14:09:16 +0100 Subject: [PATCH 3/7] adjust version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d611066..a32d6fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "stackit-core" -version = "0.0.1a" +version = "0.0.1a1" authors = [ "STACKIT Developer Tools ", ] From 2dc5d7080be6d9023aedfcdc6f7df02761626de3 Mon Sep 17 00:00:00 2001 From: Melvin Klein Date: Thu, 31 Oct 2024 13:27:38 +0000 Subject: [PATCH 4/7] black --- tests/core/test_wait.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/core/test_wait.py b/tests/core/test_wait.py index 781bca1..aee7af6 100644 --- a/tests/core/test_wait.py +++ b/tests/core/test_wait.py @@ -49,7 +49,6 @@ def test_throttle_0_throws_error(self): "check_function", [ create_check_function([400], 3, "Shouldn't be returned"), - ], ) def test_throws_for_no_retry_status_code(self, check_function): @@ -68,8 +67,7 @@ def test_throws_for_no_retry_status_code(self, check_function): ( "I'm making a note here: HUGE SUCCESS.", 3, - create_check_function( - [502, 504], 3, "I'm making a note here: HUGE SUCCESS."), + create_check_function([502, 504], 3, "I'm making a note here: HUGE SUCCESS."), ), ], ) From 68cffdd3b940b3ebe588f578cd48d087b28970ef Mon Sep 17 00:00:00 2001 From: Melvin Klein Date: Thu, 31 Oct 2024 13:29:32 +0000 Subject: [PATCH 5/7] linting --- src/stackit/core/wait.py | 4 +--- tests/core/test_wait.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/stackit/core/wait.py b/src/stackit/core/wait.py index fdb915a..6f1f3b0 100644 --- a/src/stackit/core/wait.py +++ b/src/stackit/core/wait.py @@ -1,8 +1,6 @@ +import signal import time from http import HTTPStatus - -import signal -import contextlib from typing import Any, Callable, Tuple, Union diff --git a/tests/core/test_wait.py b/tests/core/test_wait.py index aee7af6..9a9e88d 100644 --- a/tests/core/test_wait.py +++ b/tests/core/test_wait.py @@ -43,7 +43,7 @@ def test_timeout_throws_timeouterror(self): def test_throttle_0_throws_error(self): with pytest.raises(ValueError, match="throttle can't be 0"): - wait = Wait(lambda: (True, None, None, None), throttle=0) + _ = Wait(lambda: (True, None, None, None), throttle=0) @pytest.mark.parametrize( "check_function", From 61070acaedebd30da0dbb1eb77b59a4a99ca5760 Mon Sep 17 00:00:00 2001 From: Melvin Klein Date: Thu, 31 Oct 2024 15:35:45 +0100 Subject: [PATCH 6/7] adds dataclass for wait config --- src/stackit/core/wait.py | 39 +++++++++++++++++++++------------------ tests/core/test_wait.py | 8 ++++---- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/stackit/core/wait.py b/src/stackit/core/wait.py index 6f1f3b0..5b20361 100644 --- a/src/stackit/core/wait.py +++ b/src/stackit/core/wait.py @@ -1,41 +1,44 @@ +from dataclasses import dataclass, field import signal import time from http import HTTPStatus -from typing import Any, Callable, Tuple, Union +from typing import Any, Callable, List, Tuple, Union -class Wait: +@dataclass +class WaitConfig: + sleep_before_wait: int = 0 + throttle: int = 5 + timeout: int = 30 + temp_error_retry_limit: int = 5 + retry_http_error_status_codes: List[int] = field( + default_factory=lambda: [HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT] + ) + - RetryHttpErrorStatusCodes = [HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT] +class Wait: def __init__( self, check_function: Callable[[None], Tuple[bool, Union[Exception, None], Union[int, None], Any]], - sleep_before_wait: int = 0, - throttle: int = 5, - timeout: int = 30, - temp_error_retry_limit: int = 5, + config: Union[WaitConfig, None] = None, ) -> None: - if throttle == 0: + self._config = config if config else WaitConfig() + if self._config.throttle == 0: raise ValueError("throttle can't be 0") self._check_function = check_function - self._sleep_before_wait = sleep_before_wait - self._throttle = throttle - self._timeout = timeout * 60 - self._temp_error_retry_limit = temp_error_retry_limit - self.retry_http_error_status_codes = [HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT] @staticmethod def _timeout_handler(signum, frame): raise TimeoutError("Wait has timed out") def wait(self) -> Any: - time.sleep(self._sleep_before_wait) + time.sleep(self._config.sleep_before_wait) retry_temp_error_counter = 0 signal.signal(signal.SIGALRM, Wait._timeout_handler) - signal.alarm(self._timeout) + signal.alarm(self._config.timeout * 60) while True: @@ -45,13 +48,13 @@ def wait(self) -> Any: if done: return result - time.sleep(self._throttle) + time.sleep(self._config.throttle) def _handle_error(self, retry_temp_error_counter: int, error, code: int): - if code in self.retry_http_error_status_codes: + if code in self._config.retry_http_error_status_codes: retry_temp_error_counter += 1 - if retry_temp_error_counter == self._temp_error_retry_limit: + if retry_temp_error_counter == self._config.temp_error_retry_limit: raise error return retry_temp_error_counter else: diff --git a/tests/core/test_wait.py b/tests/core/test_wait.py index 9a9e88d..549e7a6 100644 --- a/tests/core/test_wait.py +++ b/tests/core/test_wait.py @@ -3,7 +3,7 @@ import pytest -from stackit.core.wait import Wait +from stackit.core.wait import Wait, WaitConfig def timeout_check_function() -> Tuple[bool, Union[Exception, None], Union[int, None], Any]: @@ -36,14 +36,14 @@ def check_function() -> Tuple[bool, Union[Exception, None], Union[int, None], An class TestWait: def test_timeout_throws_timeouterror(self): - wait = Wait(timeout_check_function, timeout=1) + wait = Wait(timeout_check_function, WaitConfig(timeout=1)) with pytest.raises(TimeoutError, match="Wait has timed out"): wait.wait() def test_throttle_0_throws_error(self): with pytest.raises(ValueError, match="throttle can't be 0"): - _ = Wait(lambda: (True, None, None, None), throttle=0) + _ = Wait(lambda: (True, None, None, None), WaitConfig(throttle=0)) @pytest.mark.parametrize( "check_function", @@ -77,5 +77,5 @@ def test_return_is_correct( error_retry_limit: int, check_function: Callable[[None], Tuple[bool, Union[Exception, None], Union[int, None], Any]], ): - wait = Wait(check_function, temp_error_retry_limit=error_retry_limit) + wait = Wait(check_function, WaitConfig(error_retry_limit)) assert wait.wait() == correct_return From 5180f910df5dbb835e3806504d628dfc398a2bc7 Mon Sep 17 00:00:00 2001 From: Melvin Klein Date: Fri, 8 Nov 2024 13:40:50 +0000 Subject: [PATCH 7/7] removes conversion of timeout to seconds --- src/stackit/core/wait.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/stackit/core/wait.py b/src/stackit/core/wait.py index 5b20361..fafa654 100644 --- a/src/stackit/core/wait.py +++ b/src/stackit/core/wait.py @@ -9,7 +9,7 @@ class WaitConfig: sleep_before_wait: int = 0 throttle: int = 5 - timeout: int = 30 + timeout: int = 1800 temp_error_retry_limit: int = 5 retry_http_error_status_codes: List[int] = field( default_factory=lambda: [HTTPStatus.BAD_GATEWAY, HTTPStatus.GATEWAY_TIMEOUT] @@ -38,7 +38,7 @@ def wait(self) -> Any: retry_temp_error_counter = 0 signal.signal(signal.SIGALRM, Wait._timeout_handler) - signal.alarm(self._config.timeout * 60) + signal.alarm(self._config.timeout) while True: