feat: Make AI Builder API URL/Timeout configurable and secure Docker production build

docker-compose.dev.yaml

@@ -80,3 +80,7 @@ services:
       - GRAFANA_MYSQL_PASSWORD=${MYSQL_ROOT_PASSWORD}
     volumes:
       - ${GRAFANA_DATA_PATH:-./data/grafana}:/var/lib/grafana
+
+  aibuilder-connector:
+    volumes:
+      - ./src:/app:ro

docker-compose.yaml

@@ -122,7 +122,6 @@ services:
       - KEYCLOAK_CLIENT_SECRET=$KEYCLOAK_CLIENT_SECRET
       - AIBUILDER_API_TOKEN=$AIBUILDER_API_TOKEN
     volumes:
-      - ./src:/app
       - ${DATA_PATH}/connectors:/opt/connectors/data
     command: >
       /bin/bash -c "/opt/connectors/script/entry.sh"

docs/hosting/connectors.md

@@ -47,6 +47,14 @@ AI Builder's models are only accessible with authentication, and for this the AP
 Because we do not want to expose the API key, we obfuscate it and use `AIBUILDER_API_TOKEN` in URLs.
 This means that for using the url of the `same_as` field of the AIBuilder models, you will need to substitute `AIBUILDER_API_TOKEN` on the url for your actual API token value.
 
+Separately, you can configure the AI Builder API URL and the request timeout in `src/config.default.toml` (or `config.override.toml`):
+
+```toml
+[aibuilder]
+api_url = "https://aiexp-dev.ai4europe.eu/federation"
+request_timeout = 30 # seconds
+```
+
 ## HuggingFace
 
 **Profile**: `huggingface-datasets`

pyproject.toml

@@ -69,6 +69,9 @@ ignore-words-list = "checkin"
 
 [tool.ruff.lint]
 select = ["S", "B", "C", "E", "F", "T", "W"]
+72: 
+73: [tool.ruff.lint.per-file-ignores]
+74: "src/tests/*" = ["S101"]
 
 [tool.pytest.ini_options]
 filterwarnings = [

src/config.default.toml

@@ -45,3 +45,8 @@ client_id = "aiod-api"  # a private client, used by the backend
 client_id_swagger = "aiod-api-swagger"  # a public client, used by the Swagger Frontend
 openid_connect_url = "http://localhost/aiod-auth/realms/aiod/.well-known/openid-configuration"
 scopes = "openid profile roles"
+
+# AI Builder connector settings
+[aibuilder]
+api_url = "https://aiexp-dev.ai4europe.eu/federation"
+request_timeout = 30 # seconds

src/connectors/aibuilder/aibuilder_mlmodel_connector.py

@@ -14,7 +14,7 @@
 from ratelimit import limits, sleep_and_retry
 from typing import Iterator, Tuple, Any
 
-from config import REQUEST_TIMEOUT
+from config import CONFIG
 
 from database.model.models_and_experiments.ml_model import MLModel
 from database.model.platform.platform_names import PlatformName
@@ -32,7 +32,7 @@
 from .aibuilder_mappings import mlmodel_mapping
 
 TOKEN = os.getenv("AIBUILDER_API_TOKEN", "")
-API_URL = "https://aiexp-dev.ai4europe.eu/federation"
+# API_URL is now fetched from CONFIG directly in the code
 GLOBAL_MAX_CALLS_MINUTE = 60
 GLOBAL_MAX_CALLS_HOUR = 2000
 ONE_MINUTE = 60
@@ -72,7 +72,7 @@ def get_response(self, url) -> dict | list | RecordError:
         `list` or `dict`structure received or a `RecordError`.
         """
         try:
-            response = requests.get(url, timeout=REQUEST_TIMEOUT)
+            response = requests.get(url, timeout=CONFIG["aibuilder"]["request_timeout"])
         except Exception as e:
             return RecordError(identifier=None, error=e)
         if response.status_code == status.HTTP_200_OK:
@@ -207,7 +207,8 @@ def fetch(  # noqa: C901
         if not self._is_aware(to_excl):
             to_excl = to_excl.replace(tzinfo=pytz.UTC)
 
-        url_get_catalog_list = f"{API_URL}/get_catalog_list?apiToken={self.token}"
+        api_url = CONFIG["aibuilder"]["api_url"]
+        url_get_catalog_list = f"{api_url}/get_catalog_list?apiToken={self.token}"
         response = self.get_response(url_get_catalog_list)
         if isinstance(response, RecordError):
             self.is_concluded = True
@@ -228,7 +229,7 @@ def fetch(  # noqa: C901
 
         for num_catalog, catalog in enumerate(catalog_list):
             url_get_catalog_solutions = (
-                f"{API_URL}/get_catalog_solutions?catalogId={catalog}&apiToken={self.token}"
+                f"{api_url}/get_catalog_solutions?catalogId={catalog}&apiToken={self.token}"
             )
             response = self.get_response(url_get_catalog_solutions)
             if isinstance(response, RecordError):
@@ -253,9 +254,9 @@ def fetch(  # noqa: C901
                 continue
 
             for num_solution, solution in enumerate(solutions_list):
-                url_get_solution = f"{API_URL}/get_solution?fullId={solution}&apiToken={self.token}"
+                url_get_solution = f"{api_url}/get_solution?fullId={solution}&apiToken={self.token}"
                 url_to_show = (
-                    f"{API_URL}/get_solution?fullId={solution}&apiToken=AIBUILDER_API_TOKEN"
+                    f"{api_url}/get_solution?fullId={solution}&apiToken=AIBUILDER_API_TOKEN"
                 )
                 response = self.get_response(url_get_solution)
                 if isinstance(response, RecordError):

src/tests/connectors/aibuilder/test_aibuilder_mlmodel_connector.py

@@ -5,8 +5,8 @@
 from datetime import datetime
 from requests.exceptions import HTTPError
 
+from config import CONFIG
 from connectors.aibuilder.aibuilder_mlmodel_connector import AIBuilderMLModelConnector
-from connectors.aibuilder.aibuilder_mlmodel_connector import API_URL
 from connectors.resource_with_relations import ResourceWithRelations
 from connectors.record_error import RecordError
 from database.model.models_and_experiments.ml_model import MLModel
@@ -17,10 +17,11 @@
 TOKEN = "TEST_AIBUILDER_API_TOKEN"
 connector = AIBuilderMLModelConnector(f"{TOKEN}")
 test_resources_path = os.path.join(path_test_resources(), "connectors", "aibuilder")
-catalog_list_url = f"{API_URL}/get_catalog_list?apiToken={TOKEN}"
-catalog_solutions_url = f"{API_URL}/get_catalog_solutions?catalogId=1&apiToken={TOKEN}"
-solution_1_url = f"{API_URL}/get_solution?fullId=1&apiToken={TOKEN}"
-solution_2_url = f"{API_URL}/get_solution?fullId=2&apiToken={TOKEN}"
+api_url = CONFIG["aibuilder"]["api_url"]
+catalog_list_url = f"{api_url}/get_catalog_list?apiToken={TOKEN}"
+catalog_solutions_url = f"{api_url}/get_catalog_solutions?catalogId=1&apiToken={TOKEN}"
+solution_1_url = f"{api_url}/get_solution?fullId=1&apiToken={TOKEN}"
+solution_2_url = f"{api_url}/get_solution?fullId=2&apiToken={TOKEN}"
 mocked_datetime_from = datetime.fromisoformat("2023-09-01T00:00:00Z")
 mocked_datetime_to = datetime.fromisoformat("2023-09-01T00:00:01Z")