diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 156fa520a..7293da1cf 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -80,3 +80,7 @@ services: - GRAFANA_MYSQL_PASSWORD=${MYSQL_ROOT_PASSWORD} volumes: - ${GRAFANA_DATA_PATH:-./data/grafana}:/var/lib/grafana + + aibuilder-connector: + volumes: + - ./src:/app:ro diff --git a/docker-compose.yaml b/docker-compose.yaml index b55752652..974310b82 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -122,7 +122,6 @@ services: - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET - AIBUILDER_API_TOKEN=$AIBUILDER_API_TOKEN volumes: - - ./src:/app - ${DATA_PATH}/connectors:/opt/connectors/data command: > /bin/bash -c "/opt/connectors/script/entry.sh" diff --git a/docs/hosting/connectors.md b/docs/hosting/connectors.md index 63af22ff6..1615c3ee5 100644 --- a/docs/hosting/connectors.md +++ b/docs/hosting/connectors.md @@ -47,6 +47,14 @@ AI Builder's models are only accessible with authentication, and for this the AP Because we do not want to expose the API key, we obfuscate it and use `AIBUILDER_API_TOKEN` in URLs. This means that for using the url of the `same_as` field of the AIBuilder models, you will need to substitute `AIBUILDER_API_TOKEN` on the url for your actual API token value. +Separately, you can configure the AI Builder API URL and the request timeout in `src/config.default.toml` (or `config.override.toml`): + +```toml +[aibuilder] +api_url = "https://aiexp-dev.ai4europe.eu/federation" +request_timeout = 30 # seconds +``` + ## HuggingFace **Profile**: `huggingface-datasets` diff --git a/pyproject.toml b/pyproject.toml index d26e610a0..5a20a5129 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,9 @@ ignore-words-list = "checkin" [tool.ruff.lint] select = ["S", "B", "C", "E", "F", "T", "W"] +72: +73: [tool.ruff.lint.per-file-ignores] +74: "src/tests/*" = ["S101"] [tool.pytest.ini_options] filterwarnings = [ diff --git a/src/config.default.toml b/src/config.default.toml index a9b9b1ff1..c54ea9640 100644 --- a/src/config.default.toml +++ b/src/config.default.toml @@ -45,3 +45,8 @@ client_id = "aiod-api" # a private client, used by the backend client_id_swagger = "aiod-api-swagger" # a public client, used by the Swagger Frontend openid_connect_url = "http://localhost/aiod-auth/realms/aiod/.well-known/openid-configuration" scopes = "openid profile roles" + +# AI Builder connector settings +[aibuilder] +api_url = "https://aiexp-dev.ai4europe.eu/federation" +request_timeout = 30 # seconds diff --git a/src/connectors/aibuilder/aibuilder_mlmodel_connector.py b/src/connectors/aibuilder/aibuilder_mlmodel_connector.py index 0106320d4..0e7e90f93 100644 --- a/src/connectors/aibuilder/aibuilder_mlmodel_connector.py +++ b/src/connectors/aibuilder/aibuilder_mlmodel_connector.py @@ -14,7 +14,7 @@ from ratelimit import limits, sleep_and_retry from typing import Iterator, Tuple, Any -from config import REQUEST_TIMEOUT +from config import CONFIG from database.model.models_and_experiments.ml_model import MLModel from database.model.platform.platform_names import PlatformName @@ -32,7 +32,7 @@ from .aibuilder_mappings import mlmodel_mapping TOKEN = os.getenv("AIBUILDER_API_TOKEN", "") -API_URL = "https://aiexp-dev.ai4europe.eu/federation" +# API_URL is now fetched from CONFIG directly in the code GLOBAL_MAX_CALLS_MINUTE = 60 GLOBAL_MAX_CALLS_HOUR = 2000 ONE_MINUTE = 60 @@ -72,7 +72,7 @@ def get_response(self, url) -> dict | list | RecordError: `list` or `dict`structure received or a `RecordError`. """ try: - response = requests.get(url, timeout=REQUEST_TIMEOUT) + response = requests.get(url, timeout=CONFIG["aibuilder"]["request_timeout"]) except Exception as e: return RecordError(identifier=None, error=e) if response.status_code == status.HTTP_200_OK: @@ -207,7 +207,8 @@ def fetch( # noqa: C901 if not self._is_aware(to_excl): to_excl = to_excl.replace(tzinfo=pytz.UTC) - url_get_catalog_list = f"{API_URL}/get_catalog_list?apiToken={self.token}" + api_url = CONFIG["aibuilder"]["api_url"] + url_get_catalog_list = f"{api_url}/get_catalog_list?apiToken={self.token}" response = self.get_response(url_get_catalog_list) if isinstance(response, RecordError): self.is_concluded = True @@ -228,7 +229,7 @@ def fetch( # noqa: C901 for num_catalog, catalog in enumerate(catalog_list): url_get_catalog_solutions = ( - f"{API_URL}/get_catalog_solutions?catalogId={catalog}&apiToken={self.token}" + f"{api_url}/get_catalog_solutions?catalogId={catalog}&apiToken={self.token}" ) response = self.get_response(url_get_catalog_solutions) if isinstance(response, RecordError): @@ -253,9 +254,9 @@ def fetch( # noqa: C901 continue for num_solution, solution in enumerate(solutions_list): - url_get_solution = f"{API_URL}/get_solution?fullId={solution}&apiToken={self.token}" + url_get_solution = f"{api_url}/get_solution?fullId={solution}&apiToken={self.token}" url_to_show = ( - f"{API_URL}/get_solution?fullId={solution}&apiToken=AIBUILDER_API_TOKEN" + f"{api_url}/get_solution?fullId={solution}&apiToken=AIBUILDER_API_TOKEN" ) response = self.get_response(url_get_solution) if isinstance(response, RecordError): diff --git a/src/tests/connectors/aibuilder/test_aibuilder_mlmodel_connector.py b/src/tests/connectors/aibuilder/test_aibuilder_mlmodel_connector.py index f55d821cc..392e02454 100644 --- a/src/tests/connectors/aibuilder/test_aibuilder_mlmodel_connector.py +++ b/src/tests/connectors/aibuilder/test_aibuilder_mlmodel_connector.py @@ -5,8 +5,8 @@ from datetime import datetime from requests.exceptions import HTTPError +from config import CONFIG from connectors.aibuilder.aibuilder_mlmodel_connector import AIBuilderMLModelConnector -from connectors.aibuilder.aibuilder_mlmodel_connector import API_URL from connectors.resource_with_relations import ResourceWithRelations from connectors.record_error import RecordError from database.model.models_and_experiments.ml_model import MLModel @@ -17,10 +17,11 @@ TOKEN = "TEST_AIBUILDER_API_TOKEN" connector = AIBuilderMLModelConnector(f"{TOKEN}") test_resources_path = os.path.join(path_test_resources(), "connectors", "aibuilder") -catalog_list_url = f"{API_URL}/get_catalog_list?apiToken={TOKEN}" -catalog_solutions_url = f"{API_URL}/get_catalog_solutions?catalogId=1&apiToken={TOKEN}" -solution_1_url = f"{API_URL}/get_solution?fullId=1&apiToken={TOKEN}" -solution_2_url = f"{API_URL}/get_solution?fullId=2&apiToken={TOKEN}" +api_url = CONFIG["aibuilder"]["api_url"] +catalog_list_url = f"{api_url}/get_catalog_list?apiToken={TOKEN}" +catalog_solutions_url = f"{api_url}/get_catalog_solutions?catalogId=1&apiToken={TOKEN}" +solution_1_url = f"{api_url}/get_solution?fullId=1&apiToken={TOKEN}" +solution_2_url = f"{api_url}/get_solution?fullId=2&apiToken={TOKEN}" mocked_datetime_from = datetime.fromisoformat("2023-09-01T00:00:00Z") mocked_datetime_to = datetime.fromisoformat("2023-09-01T00:00:01Z")