omit blank settings in requests view

.gitignore

@@ -130,5 +130,10 @@ dmypy.json
 
 .DS_Store
 
-# ignore all vscode, this is not standard configuration in this place
+# ignore all editor files, this is not standard configuration in this place
 .vscode
+*.vim
+
+# ignore worker SSH keys
+id_ed25519*
+id_rsa*

Dockerfile-api

@@ -1,4 +1,3 @@
-
 FROM python:3.12-alpine
 LABEL org.opencontainers.image.source https://github.com/openzim/zimit-frontend
 

Dockerfile-ui

@@ -1,4 +1,4 @@
-FROM node:20-alpine as ui_builder
+FROM node:20-alpine AS ui_builder
 
 RUN apk --no-cache add yarn
 WORKDIR /src/ui

api/src/zimitfrontend/constants.py

@@ -85,6 +85,9 @@ class ApiConfiguration:
     zimfarm_username = os.getenv("_ZIMFARM_USERNAME", "-")
     zimfarm_password = os.getenv("_ZIMFARM_PASSWORD", "-")
     zimit_image = os.getenv("ZIMIT_IMAGE", "openzim/zimit:1.2.0")
+    zimit_definition_version = os.getenv("ZIMIT_DEFINITION_VERSION", "")
+    if not zimit_definition_version:
+        _, zimit_definition_version = zimit_image.split(":")
 
     zimit_size_limit = _get_int_setting("ZIMIT_SIZE_LIMIT", 2**30 * 4)
     zimit_time_limit = _get_int_setting("ZIMIT_TIME_LIMIT", 3600 * 2)

api/src/zimitfrontend/main.py

@@ -8,11 +8,10 @@
 
 from zimitfrontend import __about__
 from zimitfrontend.constants import ApiConfiguration, logger
-from zimitfrontend.routes import hook, requests, tracker
+from zimitfrontend.routes import hook, offliners, requests, tracker
 
 
 class Main:
-
     def create_app(self) -> FastAPI:
         self.app = FastAPI(
             title=__about__.__api_title__,
@@ -74,6 +73,7 @@ async def internal_exception_handler(  # pyright: ignore[reportUnusedFunction]
         api.include_router(router=requests.router)
         api.include_router(router=hook.router)
         api.include_router(router=tracker.router)
+        api.include_router(router=offliners.router)
 
         self.app.mount(f"/api/{__about__.__api_version__}", api)
 

api/src/zimitfrontend/routes/offliners.py

@@ -0,0 +1,39 @@
+from http import HTTPStatus
+
+from fastapi import APIRouter, HTTPException
+
+from zimitfrontend.constants import ApiConfiguration
+from zimitfrontend.routes.schemas import OfflinerDefinitionSchema
+from zimitfrontend.zimfarm import query_api
+
+router = APIRouter(
+    prefix="/offliner-definition",
+    tags=["all"],
+)
+
+
+@router.get(
+    "",
+    summary="Get the definition for the zimit offliner",
+    status_code=200,
+    responses={
+        200: {
+            "description": "Zimit offliner definition schema",
+        },
+    },
+)
+def get_offliner_definition() -> OfflinerDefinitionSchema:
+    _, status, schema = query_api(
+        "GET", f"/offliners/zimit/{ApiConfiguration.zimit_definition_version}"
+    )
+    if status != HTTPStatus.OK:
+        raise HTTPException(
+            status_code=HTTPStatus.BAD_REQUEST,
+            detail={
+                "error": (
+                    f"Failed to get offliner defintion on Zimfarm with HTTP {status}"
+                ),
+                "zimfarm_message": schema,
+            },
+        )
+    return OfflinerDefinitionSchema.model_validate(schema)

api/src/zimitfrontend/routes/requests.py

@@ -167,6 +167,7 @@ def _cap_limit(user_limit: int, zimit_limit: int) -> int:
         "tags": [],
         "enabled": True,
         "config": config,
+        "version": ApiConfiguration.zimit_definition_version,
     }
 
     # add notification callback if email supplied

api/src/zimitfrontend/routes/schemas.py

@@ -24,6 +24,7 @@ class TaskInfo(CamelModel):
     flags: list[TaskInfoFlag]
     progress: int | None
     rank: int | None
+    offliner_definition_version: str
 
 
 class TaskCreateRequest(CamelModel):
@@ -90,6 +91,7 @@ class ZimfarmTask(BaseModel):
     # rank is populated only on GET /requested_tasks/{id}, and not on any of
     # other endpoint and not on the webhook calls
     rank: int | None = None
+    version: str
 
 
 class HookStatus(BaseModel):
@@ -101,3 +103,29 @@ class HookProcessingResult(BaseModel):
     mail_target: str | None = None
     mail_subject: str | None = None
     mail_body: str | None = None
+
+
+class Choice(BaseModel):
+    title: str
+    value: str
+
+
+class FlagSchema(BaseModel):
+    data_key: str
+    key: str
+    type: str
+    choices: list[Choice] | None = None
+    label: str
+    description: str
+    required: bool = False
+    secret: bool = False
+    min: int | None = None
+    max: int | None = None
+    min_length: int | None = None
+    max_length: int | None = None
+    pattern: str | None = None
+
+
+class OfflinerDefinitionSchema(BaseModel):
+    help: str
+    flags: list[FlagSchema]

api/src/zimitfrontend/routes/utils.py

@@ -79,6 +79,7 @@ def get_task_info(task: Any) -> TaskInfo:
             else 0
         ),
         rank=zimfarm_task.rank,
+        offliner_definition_version=zimfarm_task.version,
     )
 
 

api/tests/unit/routes/test_utils.py

@@ -48,6 +48,8 @@
                 "notification": {"ended": {"webhook": ["bla"]}},
                 "container": {"progress": {"partialZim": True}},
                 "rank": 123,
+                "offliner": "zimit",
+                "version": "initial",
             },
             TaskInfo(
                 id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -62,6 +64,7 @@
                 ],
                 progress=0,
                 rank=123,
+                offliner_definition_version="initial",
             ),
             id="full",
         ),
@@ -71,6 +74,8 @@
                 "config": {"warehouse_path": "/other", "offliner": {}},
                 "status": "blu",
                 "rank": 456,
+                "offliner": "zimit",
+                "version": "initial",
             },
             TaskInfo(
                 id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -81,6 +86,7 @@
                 flags=[],
                 progress=0,
                 rank=456,
+                offliner_definition_version="initial",
             ),
             id="simple",
         ),
@@ -91,6 +97,8 @@
                 "container": {"progress": {"partialZim": False}},
                 "status": "bla",
                 "rank": 456,
+                "offliner": "zimit",
+                "version": "initial",
             },
             TaskInfo(
                 id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -101,6 +109,7 @@
                 flags=[],
                 progress=0,
                 rank=456,
+                offliner_definition_version="initial",
             ),
             id="limit_not_hit",
         ),
@@ -111,6 +120,8 @@
                 "container": {"progress": {"overall": 100}},
                 "status": "bla",
                 "rank": 456,
+                "offliner": "zimit",
+                "version": "initial",
             },
             TaskInfo(
                 id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -121,6 +132,7 @@
                 flags=[],
                 progress=100,
                 rank=456,
+                offliner_definition_version="initial",
             ),
             id="no_limit_info",
         ),
@@ -153,6 +165,8 @@ def test_convert_zimfarm_task_to_info(task: Any, expected: TaskInfo):
         "notification": {"ended": {"webhook": ["bla"]}},
         "container": {"progress": {"partialZim": True}},
         "rank": 123,
+        "offliner": "zimit",
+        "version": "initial",
     }
 )
 

dev/README.md

@@ -2,56 +2,97 @@ This is a docker-compose configuration to be used **only** for development purpo
 almost zero security in the stack configuration.
 
 It is composed of the Zimit frontend UI and API (of course), but also a local Zimfarm DB,
-API and UI, so that you can test the whole integration locally.
+API, worker and UI, so that you can test the whole integration locally.
 
 Zimit frontend UI has two containers, one identical to production and one allowing hot reload
 of local developments.
 
 Zimit frontend API has one container, slightly modified to allow hot reload of
 most modifications done to the source code.
 
-Zimfarm UI, API and DB are deployed with official production Docker images.
+Zimfarm UI, API worker, and DB are deployed with official production Docker images.
 
 ## List of containers
 
-### zimit_ui_prod
+### zimit-ui-prod
 
 This container is Zimit frontend UI as served in production (already compiled as a static website, so not possible to live-edit)
 
-### zimit_ui_dev
+### zimit-ui-dev
 
 This container is Zimit frontend UI served in development mode (possible to live-edit)
 
-### zimit_api
+### zimit-api
 
 This container is Zimit frontend API server (only slightly modified to enable live reload of edits)
 
-## zimfarm_db
+## zimfarm-db
 
 This container is a local Zimfarm database
 
-## zimfarm_api
+## zimfarm-api
 
 This container is a local Zimfarm API
 
-## zimfarm_ui
+## zimfarm-ui
 
 This container is a local Zimfarm UI
 
+## zimfarm-worker-manager
+
+This container is a local Zimfarm worker manager. It pulls the Zimfarm task worker image to
+execute tasks
+
+## zimfarm-receiver
+
+This container stores the uploaded files/logs for each task.
+
 ## Instructions
 
-First start the Docker-Compose stack:
+### Starting the Compose Stack
 
-```sh
-cd dev
-docker compose -p zimit up -d
-```
+- To start the compose services (without the worker):
 
-If it is your first execution of the dev stack, you need to create a "virtual" worker in Zimfarm DB:
+  ```sh
+  cd dev
+  docker compose up --build
+  ```
 
-```sh
-dev/create_worker.sh
-```
+- To start the compose services (with a registered worker):
+
+  ```sh
+  cd dev
+  docker compose --profile worker up --build
+  ```
+
+- If you are running with worker profile, you will need to create warehouse paths to upload the logs and files for each task.
+  ```sh
+  docker exec -it zimfarm-receiver bash
+  /contrib/create-warehouse-paths.sh
+  ```
+
+If it is your first execution of the dev stack, you need to create offliners and a "virtual" worker in Zimfarm DB. Thus, you need to start the services without the worker
+profile till you register a worker.
+
+- To create offliners:
+
+  ```sh
+  dev/create_offliners.sh
+  ```
+
+  This pulls the latest offliner definitions from the respective offliner repositories
+  and registers them with the Zimfarm API. The versions of the offliner definitions
+  are hardcoded to "dev". This is the same as the `ZIMIT_DEFINITION_VERSION` defined in `dev/docker-compose.yml`
+
+- To register a worker
+  ```sh
+  dev/create_worker.sh
+  ```
+
+NOTE: These shell scripts have been configured withs some reasonable defaults like:
+
+- admin username is `admin` with password `admin`. This must be the same as teh `INIT_USERNAME` AND `INIT_PASSWORD` of the `zimfarm-api` service and `_ZIMFARM_USERNAME` and `_ZIMFARM_PASSWORD` of the `zimit-api` service. See the Compose file for the definition of these environment variables.
+- a worker `test_worker` with 1Gb memory, 1Gb disk and 1 CPU. These are specified in the `environment` section of the `zimfarm-worker-manager` too.
 
 If you have requested a task via Zimit UI and want to simulate a worker starting this task to observe the consequence in Zimit UI, you might use the `dev/start_first_req_task.sh`.