diff --git a/.gitignore b/.gitignore
index a76f45d..fd3b24b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -130,5 +130,10 @@ dmypy.json
.DS_Store
-# ignore all vscode, this is not standard configuration in this place
+# ignore all editor files, this is not standard configuration in this place
.vscode
+*.vim
+
+# ignore worker SSH keys
+id_ed25519*
+id_rsa*
diff --git a/Dockerfile-api b/Dockerfile-api
index 3fd46af..2786741 100644
--- a/Dockerfile-api
+++ b/Dockerfile-api
@@ -1,4 +1,3 @@
-
FROM python:3.12-alpine
LABEL org.opencontainers.image.source https://github.com/openzim/zimit-frontend
diff --git a/Dockerfile-ui b/Dockerfile-ui
index b56e33a..e27b3ba 100644
--- a/Dockerfile-ui
+++ b/Dockerfile-ui
@@ -1,4 +1,4 @@
-FROM node:20-alpine as ui_builder
+FROM node:20-alpine AS ui_builder
RUN apk --no-cache add yarn
WORKDIR /src/ui
diff --git a/api/src/zimitfrontend/constants.py b/api/src/zimitfrontend/constants.py
index 2a49cc8..250f6ae 100644
--- a/api/src/zimitfrontend/constants.py
+++ b/api/src/zimitfrontend/constants.py
@@ -85,6 +85,9 @@ class ApiConfiguration:
zimfarm_username = os.getenv("_ZIMFARM_USERNAME", "-")
zimfarm_password = os.getenv("_ZIMFARM_PASSWORD", "-")
zimit_image = os.getenv("ZIMIT_IMAGE", "openzim/zimit:1.2.0")
+ zimit_definition_version = os.getenv("ZIMIT_DEFINITION_VERSION", "")
+ if not zimit_definition_version:
+ _, zimit_definition_version = zimit_image.split(":")
zimit_size_limit = _get_int_setting("ZIMIT_SIZE_LIMIT", 2**30 * 4)
zimit_time_limit = _get_int_setting("ZIMIT_TIME_LIMIT", 3600 * 2)
diff --git a/api/src/zimitfrontend/main.py b/api/src/zimitfrontend/main.py
index 07a84ab..48016af 100644
--- a/api/src/zimitfrontend/main.py
+++ b/api/src/zimitfrontend/main.py
@@ -8,11 +8,10 @@
from zimitfrontend import __about__
from zimitfrontend.constants import ApiConfiguration, logger
-from zimitfrontend.routes import hook, requests, tracker
+from zimitfrontend.routes import hook, offliners, requests, tracker
class Main:
-
def create_app(self) -> FastAPI:
self.app = FastAPI(
title=__about__.__api_title__,
@@ -74,6 +73,7 @@ async def internal_exception_handler( # pyright: ignore[reportUnusedFunction]
api.include_router(router=requests.router)
api.include_router(router=hook.router)
api.include_router(router=tracker.router)
+ api.include_router(router=offliners.router)
self.app.mount(f"/api/{__about__.__api_version__}", api)
diff --git a/api/src/zimitfrontend/routes/offliners.py b/api/src/zimitfrontend/routes/offliners.py
new file mode 100644
index 0000000..f5f64b2
--- /dev/null
+++ b/api/src/zimitfrontend/routes/offliners.py
@@ -0,0 +1,39 @@
+from http import HTTPStatus
+
+from fastapi import APIRouter, HTTPException
+
+from zimitfrontend.constants import ApiConfiguration
+from zimitfrontend.routes.schemas import OfflinerDefinitionSchema
+from zimitfrontend.zimfarm import query_api
+
+router = APIRouter(
+ prefix="/offliner-definition",
+ tags=["all"],
+)
+
+
+@router.get(
+ "",
+ summary="Get the definition for the zimit offliner",
+ status_code=200,
+ responses={
+ 200: {
+ "description": "Zimit offliner definition schema",
+ },
+ },
+)
+def get_offliner_definition() -> OfflinerDefinitionSchema:
+ _, status, schema = query_api(
+ "GET", f"/offliners/zimit/{ApiConfiguration.zimit_definition_version}"
+ )
+ if status != HTTPStatus.OK:
+ raise HTTPException(
+ status_code=HTTPStatus.BAD_REQUEST,
+ detail={
+ "error": (
+ f"Failed to get offliner defintion on Zimfarm with HTTP {status}"
+ ),
+ "zimfarm_message": schema,
+ },
+ )
+ return OfflinerDefinitionSchema.model_validate(schema)
diff --git a/api/src/zimitfrontend/routes/requests.py b/api/src/zimitfrontend/routes/requests.py
index 4f1839f..59d534c 100644
--- a/api/src/zimitfrontend/routes/requests.py
+++ b/api/src/zimitfrontend/routes/requests.py
@@ -167,6 +167,7 @@ def _cap_limit(user_limit: int, zimit_limit: int) -> int:
"tags": [],
"enabled": True,
"config": config,
+ "version": ApiConfiguration.zimit_definition_version,
}
# add notification callback if email supplied
diff --git a/api/src/zimitfrontend/routes/schemas.py b/api/src/zimitfrontend/routes/schemas.py
index 83711fd..6a72ded 100644
--- a/api/src/zimitfrontend/routes/schemas.py
+++ b/api/src/zimitfrontend/routes/schemas.py
@@ -24,6 +24,7 @@ class TaskInfo(CamelModel):
flags: list[TaskInfoFlag]
progress: int | None
rank: int | None
+ offliner_definition_version: str
class TaskCreateRequest(CamelModel):
@@ -90,6 +91,7 @@ class ZimfarmTask(BaseModel):
# rank is populated only on GET /requested_tasks/{id}, and not on any of
# other endpoint and not on the webhook calls
rank: int | None = None
+ version: str
class HookStatus(BaseModel):
@@ -101,3 +103,29 @@ class HookProcessingResult(BaseModel):
mail_target: str | None = None
mail_subject: str | None = None
mail_body: str | None = None
+
+
+class Choice(BaseModel):
+ title: str
+ value: str
+
+
+class FlagSchema(BaseModel):
+ data_key: str
+ key: str
+ type: str
+ choices: list[Choice] | None = None
+ label: str
+ description: str
+ required: bool = False
+ secret: bool = False
+ min: int | None = None
+ max: int | None = None
+ min_length: int | None = None
+ max_length: int | None = None
+ pattern: str | None = None
+
+
+class OfflinerDefinitionSchema(BaseModel):
+ help: str
+ flags: list[FlagSchema]
diff --git a/api/src/zimitfrontend/routes/utils.py b/api/src/zimitfrontend/routes/utils.py
index 89da0ed..aabb5b8 100644
--- a/api/src/zimitfrontend/routes/utils.py
+++ b/api/src/zimitfrontend/routes/utils.py
@@ -79,6 +79,7 @@ def get_task_info(task: Any) -> TaskInfo:
else 0
),
rank=zimfarm_task.rank,
+ offliner_definition_version=zimfarm_task.version,
)
diff --git a/api/tests/unit/routes/test_utils.py b/api/tests/unit/routes/test_utils.py
index 9ea768b..a993960 100644
--- a/api/tests/unit/routes/test_utils.py
+++ b/api/tests/unit/routes/test_utils.py
@@ -48,6 +48,8 @@
"notification": {"ended": {"webhook": ["bla"]}},
"container": {"progress": {"partialZim": True}},
"rank": 123,
+ "offliner": "zimit",
+ "version": "initial",
},
TaskInfo(
id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -62,6 +64,7 @@
],
progress=0,
rank=123,
+ offliner_definition_version="initial",
),
id="full",
),
@@ -71,6 +74,8 @@
"config": {"warehouse_path": "/other", "offliner": {}},
"status": "blu",
"rank": 456,
+ "offliner": "zimit",
+ "version": "initial",
},
TaskInfo(
id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -81,6 +86,7 @@
flags=[],
progress=0,
rank=456,
+ offliner_definition_version="initial",
),
id="simple",
),
@@ -91,6 +97,8 @@
"container": {"progress": {"partialZim": False}},
"status": "bla",
"rank": 456,
+ "offliner": "zimit",
+ "version": "initial",
},
TaskInfo(
id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -101,6 +109,7 @@
flags=[],
progress=0,
rank=456,
+ offliner_definition_version="initial",
),
id="limit_not_hit",
),
@@ -111,6 +120,8 @@
"container": {"progress": {"overall": 100}},
"status": "bla",
"rank": 456,
+ "offliner": "zimit",
+ "version": "initial",
},
TaskInfo(
id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf",
@@ -121,6 +132,7 @@
flags=[],
progress=100,
rank=456,
+ offliner_definition_version="initial",
),
id="no_limit_info",
),
@@ -153,6 +165,8 @@ def test_convert_zimfarm_task_to_info(task: Any, expected: TaskInfo):
"notification": {"ended": {"webhook": ["bla"]}},
"container": {"progress": {"partialZim": True}},
"rank": 123,
+ "offliner": "zimit",
+ "version": "initial",
}
)
diff --git a/dev/README.md b/dev/README.md
index 02017b9..1004246 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -2,7 +2,7 @@ This is a docker-compose configuration to be used **only** for development purpo
almost zero security in the stack configuration.
It is composed of the Zimit frontend UI and API (of course), but also a local Zimfarm DB,
-API and UI, so that you can test the whole integration locally.
+API, worker and UI, so that you can test the whole integration locally.
Zimit frontend UI has two containers, one identical to production and one allowing hot reload
of local developments.
@@ -10,48 +10,89 @@ of local developments.
Zimit frontend API has one container, slightly modified to allow hot reload of
most modifications done to the source code.
-Zimfarm UI, API and DB are deployed with official production Docker images.
+Zimfarm UI, API worker, and DB are deployed with official production Docker images.
## List of containers
-### zimit_ui_prod
+### zimit-ui-prod
This container is Zimit frontend UI as served in production (already compiled as a static website, so not possible to live-edit)
-### zimit_ui_dev
+### zimit-ui-dev
This container is Zimit frontend UI served in development mode (possible to live-edit)
-### zimit_api
+### zimit-api
This container is Zimit frontend API server (only slightly modified to enable live reload of edits)
-## zimfarm_db
+## zimfarm-db
This container is a local Zimfarm database
-## zimfarm_api
+## zimfarm-api
This container is a local Zimfarm API
-## zimfarm_ui
+## zimfarm-ui
This container is a local Zimfarm UI
+## zimfarm-worker-manager
+
+This container is a local Zimfarm worker manager. It pulls the Zimfarm task worker image to
+execute tasks
+
+## zimfarm-receiver
+
+This container stores the uploaded files/logs for each task.
+
## Instructions
-First start the Docker-Compose stack:
+### Starting the Compose Stack
-```sh
-cd dev
-docker compose -p zimit up -d
-```
+- To start the compose services (without the worker):
-If it is your first execution of the dev stack, you need to create a "virtual" worker in Zimfarm DB:
+ ```sh
+ cd dev
+ docker compose up --pull always --build
+ ```
-```sh
-dev/create_worker.sh
-```
+- To start the compose services (with a registered worker):
+
+ ```sh
+ cd dev
+ docker compose --profile worker up --pull always --build
+ ```
+
+- If you are running with worker profile, you will need to create warehouse paths to upload the logs and files for each task.
+ ```sh
+ docker exec -it zimfarm-receiver bash
+ /contrib/create-warehouse-paths.sh
+ ```
+
+If it is your first execution of the dev stack, you need to create offliners and a "virtual" worker in Zimfarm DB. Thus, you need to start the services without the worker
+profile till you register a worker.
+
+- To create offliners:
+
+ ```sh
+ dev/create_offliners.sh
+ ```
+
+ This pulls the latest offliner definitions from the respective offliner repositories
+ and registers them with the Zimfarm API. The versions of the offliner definitions
+ are hardcoded to "dev". This is the same as the `ZIMIT_DEFINITION_VERSION` defined in `dev/docker-compose.yml`
+
+- To register a worker
+ ```sh
+ dev/create_worker.sh
+ ```
+
+NOTE: These shell scripts have been configured withs some reasonable defaults like:
+
+- admin username is `admin` with password `admin`. This must be the same as teh `INIT_USERNAME` AND `INIT_PASSWORD` of the `zimfarm-api` service and `_ZIMFARM_USERNAME` and `_ZIMFARM_PASSWORD` of the `zimit-api` service. See the Compose file for the definition of these environment variables.
+- a worker `test_worker` with 1Gb memory, 1Gb disk and 1 CPU. These are specified in the `environment` section of the `zimfarm-worker-manager` too.
If you have requested a task via Zimit UI and want to simulate a worker starting this task to observe the consequence in Zimit UI, you might use the `dev/start_first_req_task.sh`.
diff --git a/dev/create-warehouse-paths.sh b/dev/create-warehouse-paths.sh
new file mode 100755
index 0000000..593c5a6
--- /dev/null
+++ b/dev/create-warehouse-paths.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+set -e
+
+mkdir -p \
+ /jail/zim/freecodecamp \
+ /jail/zim/gutenberg \
+ /jail/zim/ifixit \
+ /jail/zim/devdocs \
+ /jail/zim/libretexts \
+ /jail/zim/mindtouch \
+ /jail/zim/mooc \
+ /jail/zim/other \
+ /jail/zim/phet \
+ /jail/zim/stack_exchange \
+ /jail/zim/ted \
+ /jail/zim/videos \
+ /jail/zim/vikidia \
+ /jail/zim/wikibooks \
+ /jail/zim/wikihow \
+ /jail/zim/wikinews \
+ /jail/zim/wikipedia \
+ /jail/zim/wikiquote \
+ /jail/zim/wikisource \
+ /jail/zim/wikiversity \
+ /jail/zim/wikivoyage \
+ /jail/zim/wiktionary \
+ /jail/zim/zimit
+
+chmod 777 \
+ /jail/zim/freecodecamp \
+ /jail/zim/gutenberg \
+ /jail/zim/ifixit \
+ /jail/zim/devdocs \
+ /jail/zim/libretexts \
+ /jail/zim/mindtouch \
+ /jail/zim/mooc \
+ /jail/zim/other \
+ /jail/zim/phet \
+ /jail/zim/stack_exchange \
+ /jail/zim/ted \
+ /jail/zim/videos \
+ /jail/zim/vikidia \
+ /jail/zim/wikibooks \
+ /jail/zim/wikihow \
+ /jail/zim/wikinews \
+ /jail/zim/wikipedia \
+ /jail/zim/wikiquote \
+ /jail/zim/wikisource \
+ /jail/zim/wikiversity \
+ /jail/zim/wikivoyage \
+ /jail/zim/wiktionary \
+ /jail/zim/zimit
diff --git a/dev/create_offliners.sh b/dev/create_offliners.sh
new file mode 100755
index 0000000..c70d927
--- /dev/null
+++ b/dev/create_offliners.sh
@@ -0,0 +1,174 @@
+#!/bin/bash
+
+# Script to register offliners in Zimfarm
+# - retrieve an admin token
+# - fetch offliner definitions from their respective GitHub repositories
+# - register each offliner and its definition with the backend API
+
+set -e
+
+# List of offliners to register
+OFFLINERS=("zimit")
+
+# Base URL for offliner definitions
+OFFLINER_DEFINITIONS_BASE_URL="https://raw.githubusercontent.com/openzim"
+
+# zimfarm API URL
+ZIMFARM_BASE_API_URL="http://localhost:8004/v2"
+
+# Offliner configuration data for API registration
+declare -A offliner_configs
+
+# Initialize offliner configurations
+setup_offliner_configs() {
+ # DashModel offliners
+ offliner_configs["zimit"]='{"base_model":"CamelModel","docker_image_name":"openzim/zimit","command_name":"zimit"}'
+}
+
+die() {
+ echo "ERROR: $1" >&2
+ exit 1
+}
+
+fetch_offliner_definition() {
+ local offliner_name="$1"
+ local url="${OFFLINER_DEFINITIONS_BASE_URL}/${offliner_name}/refs/heads/main/offliner-definition.json"
+
+ local definition
+ if ! definition=$(curl -s -f "$url"); then
+ die "Failed to fetch offliner definition for ${offliner_name} from ${url}"
+ fi
+
+ if ! echo "$definition" | jq . >/dev/null 2>&1; then
+ die "Invalid JSON received for ${offliner_name}"
+ fi
+
+ echo "$definition"
+}
+
+register_offliner_via_api() {
+ local offliner_id="$1"
+ local config="${offliner_configs[$offliner_id]}"
+ echo "Registering ${offliner} via API..."
+
+ if [ -z "$config" ]; then
+ die "No configuration found for offliner: ${offliner_id}"
+ fi
+ # Create the payload with offliner_id used as ci_secret_hash
+ local payload
+ payload=$(echo "$config" | jq --arg offliner_id "$offliner_id" '. + {"ci_secret_hash": $offliner_id, "offliner_id": $offliner_id}')
+
+ local response
+ local http_code
+ response=$(curl -s -w "\n%{http_code}" -X 'POST' \
+ "${ZIMFARM_BASE_API_URL}/offliners" \
+ -H 'accept: application/json' \
+ -H "Authorization: Bearer ${ZF_ADMIN_TOKEN}" \
+ -H 'Content-Type: application/json' \
+ -d "$payload")
+
+ # Extract HTTP status code (last line)
+ http_code=$(echo "$response" | tail -n1)
+ # Extract response body (all lines except last)
+ response=$(echo "$response" | head -n -1)
+
+ case "$http_code" in
+ 201)
+ echo "Successfully registered ${offliner_id}"
+ ;;
+ 409)
+ echo "WARNING: Offliner ${offliner_id} already exists, skipping..."
+ ;;
+ *)
+ local error_msg
+ error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code")
+ die "Registration failed for ${offliner_id}: ${error_msg}"
+ ;;
+ esac
+}
+
+# Function to create offliner definition schema
+create_offliner_definition() {
+ local offliner_id="$1"
+ local spec="${offliner_definitions[$offliner_id]}"
+
+ if [ -z "$spec" ]; then
+ die "No definition found for offliner: ${offliner_id}"
+ fi
+
+ echo "Creating definition for ${offliner_id}..."
+
+ # Create the payload for schema creation
+ local payload
+ payload=$(jq -n \
+ --arg version "dev" \
+ --arg ci_secret "$offliner_id" \
+ --argjson spec "$spec" \
+ '{version: $version, ci_secret: $ci_secret, spec: $spec}')
+
+
+ # Create the schema via API
+ local response
+ local http_code
+ response=$(curl -s -w "\n%{http_code}" -X 'POST' \
+ "${ZIMFARM_BASE_API_URL}/offliners/${offliner_id}/versions" \
+ -H 'accept: application/json' \
+ -H "Authorization: Bearer ${ZF_ADMIN_TOKEN}" \
+ -H 'Content-Type: application/json' \
+ -d "$payload")
+
+ # Extract HTTP status code (last line)
+ http_code=$(echo "$response" | tail -n1)
+ # Extract response body (all lines except last)
+ response=$(echo "$response" | head -n -1)
+
+ case "$http_code" in
+ 201)
+ echo "Successfully created schema for ${offliner_id}"
+ ;;
+ 409)
+ echo "WARNING: Offliner ${offliner_id} already exists, skipping..."
+ ;;
+ *)
+ local error_msg
+ error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code")
+ die "Definition creation failed for ${offliner_id}: ${error_msg}"
+ ;;
+ esac
+}
+
+echo "Retrieving admin access token"
+
+ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
+ "${ZIMFARM_BASE_API_URL}/auth/authorize" \
+ -H 'accept: application/json' \
+ -H 'Content-Type: application/json' \
+ -d '{"username": "admin", "password": "admin"}' \
+ | jq -r '.access_token')"
+
+if [ -z "$ZF_ADMIN_TOKEN" ] || [ "$ZF_ADMIN_TOKEN" = "null" ]; then
+ die "Failed to retrieve admin access token"
+fi
+
+echo "Admin token retrieved successfully"
+
+# Setup offliner configurations for API registration
+setup_offliner_configs
+
+# Declare associative array to store offliner definitions
+declare -A offliner_definitions
+
+echo "Fetching offliner definitions..."
+
+for offliner in "${OFFLINERS[@]}"; do
+ echo "Fetching ${offliner} definition..."
+ offliner_definitions["$offliner"]=$(fetch_offliner_definition "$offliner" | jq -c '.')
+done
+
+echo "All offliner definitions fetched successfully"
+
+for offliner in "${OFFLINERS[@]}"; do
+ register_offliner_via_api "$offliner"
+ create_offliner_definition "$offliner"
+done
+echo "All offliners registered via API successfully!"
diff --git a/dev/create_worker.sh b/dev/create_worker.sh
index 746ca9b..fe5c694 100755
--- a/dev/create_worker.sh
+++ b/dev/create_worker.sh
@@ -1,28 +1,110 @@
-echo "Retrieving access token"
+#!/bin/bash
+
+# Call it once to create a `test_worker`:
+# - retrieve an admin token
+# - create the `test_worker`` user
+# - create the associated worker object
+# - upload a test public key.
+#
+# To be used to have a "real" test worker for local development, typically to start
+# a worker manager or a task manager or simply assign tasks to a worker in the UI/API
+
+set -e
+
+ZIMFARM_BASE_API_URL="http://localhost:8004/v2"
+
+die() {
+ echo "ERROR: $1" >&2
+ exit 1
+}
+
+check_http_code() {
+ local http_code="$1"
+ local response="$2"
+
+ if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
+ :
+ else
+ error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code")
+ die "Could not checkin worker: ${error_msg}"
+ fi
+}
+
+echo "Retrieving admin access token"
ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
- 'http://localhost:8002/v2/auth/authorize' \
+ "${ZIMFARM_BASE_API_URL}/auth/authorize" \
-H 'accept: application/json' \
-H 'Content-Type: application/json' \
- -d '{"username":"admin","password":"admin"}' \
+ -d '{"username": "admin", "password": "admin"}' \
| jq -r '.access_token')"
+if [ -z "$ZF_ADMIN_TOKEN" ] || [ "$ZF_ADMIN_TOKEN" = "null" ]; then
+ die "Failed to retrieve admin access token"
+fi
-echo "Worker check-in (will create if missing)"
+echo "Create test_worker user"
-curl -s -X 'PUT' \
- 'http://localhost:8002/v2/workers/worker/check-in' \
+curl -s -X 'POST' \
+ "${ZIMFARM_BASE_API_URL}/users" \
-H 'accept: */*' \
-H 'Content-Type: application/json' \
-H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
-d '{
- "username": "admin",
+ "role":"worker",
+ "username": "test_worker",
+ "email":"test_worker@acme.com",
+ "password":"test_worker"
+}'
+
+echo "Retrieving test_worker access token"
+
+ZF_USER_TOKEN="$(curl -s -X 'POST' \
+ "${ZIMFARM_BASE_API_URL}/auth/authorize" \
+ -H 'accept: application/json' \
+ -H 'Content-Type: application/json' \
+ -d '{"username": "test_worker", "password": "test_worker"}' \
+ | jq -r '.access_token')"
+
+if [ -z "$ZF_USER_TOKEN" ] || [ "$ZF_USER_TOKEN" = "null" ]; then
+ die "Failed to retrieve worker access token"
+fi
+
+response=$(curl -s -w "\n%{http_code}" -X 'PUT' \
+ "${ZIMFARM_BASE_API_URL}/workers/test_worker/check-in" \
+ -H 'accept: */*' \
+ -H 'Content-Type: application/json' \
+ -H "Authorization: Bearer $ZF_USER_TOKEN" \
+ -d '{
+ "username": "test_worker",
"cpu": 3,
- "memory": 1024,
- "disk": 0,
+ "memory": 2147483648,
+ "disk": 4294967296,
"offliners": [
"zimit"
]
-}'
+}')
+
+http_code=$(echo "$response" | tail -n1)
+response=$(echo "$response" | head -n -1)
+
+check_http_code "$http_code" "$response"
+
+echo "Generating SSH key pair (Ed25519)"
+ssh-keygen -t ed25519 -f id_ed25519 -N ""
+payload="$(jq -n --arg key "$(< id_ed25519.pub)" '{key: $key}')"
+
+
+echo "Uploading worker keys to API"
+response=$(curl -s -w "\n%{http_code}" -X POST "${ZIMFARM_BASE_API_URL}/users/test_worker/keys" \
+ -H 'accept: */*' \
+ -H "Authorization: Bearer $ZF_USER_TOKEN" \
+ -H 'Content-Type: application/json; charset=utf-8' \
+ -d "$payload")
+
+http_code=$(echo "$response" | tail -n1)
+response=$(echo "$response" | head -n -1)
+
+check_http_code "$http_code" "$response"
echo "DONE"
diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml
index 9cd5502..b889e77 100644
--- a/dev/docker-compose.yml
+++ b/dev/docker-compose.yml
@@ -1,35 +1,98 @@
services:
zimfarm-db:
- image: postgres:15.2-bullseye
+ image: postgres:17.3-bookworm
+ container_name: zimarm-postgresdb
ports:
- 127.0.0.1:5432:5432
volumes:
- zimfarm_data:/var/lib/postgresql/data
+ - ./postgres-initdb:/docker-entrypoint-initdb.d
environment:
- POSTGRES_DB=zimfarm
- POSTGRES_USER=zimfarm
- POSTGRES_PASSWORD=zimpass
+ healthcheck:
+ test: ["CMD", "pg_isready", "-q", "-d", "dbname=zimfarm user=zimfarm"]
+ interval: 10s
+ timeout: 5s
+ retries: 3
zimfarm-api:
- image: ghcr.io/openzim/zimfarm-dispatcher:latest
+ image: ghcr.io/openzim/zimfarm-backend:latest
+ container_name: zimfarm-api
ports:
- 127.0.0.1:8004:80
environment:
BINDING_HOST: 0.0.0.0
JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw
POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@zimfarm-db:5432/zimfarm
- ALEMBIC_UPGRADE_HEAD_ON_START: "1"
- ZIMIT_USE_RELAXED_SCHEMA: "y"
+ INIT_USERNAME: admin
+ INIT_PASSWORD: admin
+ ZIMIT_USE_RELAXED_SCHEMA: true
+ ALLOWED_ORIGINS: http://localhost:8003
+ # upload artifacts, logs and zim to receiver for simplicity
+ ARTIFACTS_UPLOAD_URI: sftp://uploader@zimfarm-receiver:22/logs/ # reusing logs dir, kind of a hack
+ LOGS_UPLOAD_URI: sftp://uploader@zimfarm-receiver:22/logs/
+ ZIM_UPLOAD_URI: sftp://uploader@zimfarm-receiver:22/zim/
+ ZIMCHECK_OPTION: --all
depends_on:
- - zimfarm-db
+ zimfarm-db:
+ condition: service_healthy
+ command:
+ - uvicorn
+ - zimfarm_backend.main:app
+ - --host
+ - "0.0.0.0"
+ - --port
+ - "80"
+ - --proxy-headers
+ - --forwarded-allow-ips
+ - "*"
zimfarm-ui:
image: ghcr.io/openzim/zimfarm-ui:latest
+ container_name: zimfarm-ui
ports:
- 127.0.0.1:8003:80
+ volumes:
+ - ./zimfarm_ui_dev/config.json:/usr/share/nginx/html/config.json:ro
+ depends_on:
+ zimfarm-api:
+ condition: service_healthy
+ zimfarm-worker-manager:
+ image: ghcr.io/openzim/zimfarm-worker-manager:latest
+ container_name: zimfarm-worker-manager
+ depends_on:
+ zimfarm-api:
+ condition: service_healthy
+ command: worker-manager --webapi-uri 'http://zimfarm-api:80/v2' --username test_worker --name test_worker
+ environment:
+ - DEBUG=true
+ - TASK_WORKER_IMAGE=ghcr.io/openzim/zimfarm-task-worker:latest
+ - ENVIRONMENT=development
+ - ZIMFARM_DISK=4Gb
+ - ZIMFARM_MEMORY=2Gb
+ - ZIMFARM_CPU=1
+ - POLL_INTERVAL=10
+ volumes:
+ - /var/run/docker.sock:/var/run/docker.sock
+ - ./id_ed25519:/etc/ssh/keys/zimfarm
+ profiles:
+ - worker
+ zimfarm-receiver:
+ image: ghcr.io/openzim/zimfarm-receiver:latest
+ container_name: zimfarm-receiver
+ ports:
+ - 127.0.0.1:8222:22
+ volumes:
+ - ./create-warehouse-paths.sh:/contrib/create-warehouse-paths.sh
environment:
- ZIMFARM_WEBAPI: http://localhost:8004/v1
+ - ZIMFARM_WEBAPI=http://zimfarm-api:80/v2
depends_on:
- - zimfarm-api
+ zimfarm-api:
+ condition: service_healthy
+ profiles:
+ - worker
zimit-api:
+ container_name: zimit-api
build:
dockerfile: Dockerfile-api
context: ..
@@ -52,16 +115,19 @@ services:
ports:
- 127.0.0.1:8002:80
environment:
- INTERNAL_ZIMFARM_WEBAPI: http://zimfarm-api:80/v1
+ INTERNAL_ZIMFARM_WEBAPI: http://zimfarm-api:80/v2
_ZIMFARM_USERNAME: admin
_ZIMFARM_PASSWORD: admin
- TASK_WORKER: worker
+ TASK_WORKER: test_worker
HOOK_TOKEN: a_very_secret_token
CALLBACK_BASE_URL: http://zimit-api:80/api/v1/requests/hook
DIGEST_KEY: d1a2df7f0a229cc6
+ ZIMIT_IMAGE: openzim/zimit:3.0.5
+ ZIMIT_DEFINITION_VERSION: dev
depends_on:
- zimfarm-api
zimit-ui-dev:
+ container_name: zimit-ui-dev
build:
dockerfile: dev/zimit_ui_dev/Dockerfile
context: ..
@@ -74,6 +140,7 @@ services:
depends_on:
- zimit-api
zimit-ui-prod:
+ container_name: zimit-ui-prod
build:
dockerfile: Dockerfile-ui
context: ..
diff --git a/dev/postgres-initdb/init-extensions.sh b/dev/postgres-initdb/init-extensions.sh
new file mode 100755
index 0000000..ddeca2e
--- /dev/null
+++ b/dev/postgres-initdb/init-extensions.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+ CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+EOSQL
diff --git a/dev/start_first_req_task.sh b/dev/start_first_req_task.sh
index 1c3fc1f..c4bbf7a 100755
--- a/dev/start_first_req_task.sh
+++ b/dev/start_first_req_task.sh
@@ -1,3 +1,22 @@
+#!/bin/bash
+
+die() {
+ echo "ERROR: $1" >&2
+ exit 1
+}
+
+check_http_code() {
+ local http_code="$1"
+ local response="$2"
+
+ if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
+ :
+ else
+ error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code")
+ die "Could not checkin worker: ${error_msg}"
+ fi
+}
+
echo "Retrieving access token"
ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
@@ -7,25 +26,31 @@ ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \
-d '{"username":"admin","password":"admin"}' \
| jq -r '.access_token')"
+if [ -z "$ZF_ADMIN_TOKEN" ] || [ "$ZF_ADMIN_TOKEN" = "null" ]; then
+ die "Failed to retrieve admin access token"
+fi
+
echo "Get last requested task"
LAST_TASK_ID="$(curl -s -X 'GET' \
'http://localhost:8004/v2/requested-tasks' \
-H 'accept: application/json' \
-H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
- | jq -r '.items[0]._id')"
+ | jq -r '.items[0].id')"
if [ "$LAST_TASK_ID" = "null" ]; then
- echo "No pending requested task. Exiting script."
- exit 1
+ die "No pending requested task. Exiting script."
fi
echo "Start task"
-curl -s -X 'POST' \
- "http://localhost:8004/v2/tasks/$LAST_TASK_ID?worker_name=worker" \
+response=$(curl -s -w "\n%{http_code}" -X 'POST' \
+ "http://localhost:8004/v2/tasks/$LAST_TASK_ID?worker_name=test_worker" \
-H 'accept: application/json' \
-H "Authorization: Bearer $ZF_ADMIN_TOKEN" \
- -d ''
+)
+http_code=$(echo "$response" | tail -n1)
+
+check_http_code "$http_code"
echo "DONE"
diff --git a/dev/zimfarm_ui_dev/config.json b/dev/zimfarm_ui_dev/config.json
new file mode 100644
index 0000000..cbea836
--- /dev/null
+++ b/dev/zimfarm_ui_dev/config.json
@@ -0,0 +1,3 @@
+{
+ "ZIMFARM_WEBAPI": "http://localhost:8004/v2"
+}
diff --git a/dev/zimit_ui_dev/config.json b/dev/zimit_ui_dev/config.json
index 6c4bc3c..be99129 100644
--- a/dev/zimit_ui_dev/config.json
+++ b/dev/zimit_ui_dev/config.json
@@ -1,7 +1,6 @@
{
"stop_new_requests_on": false,
"zimit_ui_api": "http://localhost:8002/api/v1",
- "zimfarm_api": "http://localhost:8004/v1",
"wikipedia_offline_article": "https://en.wikipedia.org/wiki/Offline",
"kiwix_home_page": "https://kiwix.org",
"kiwix_download_page": "https://kiwix.org/en/applications/",
diff --git a/ui/public/config.json b/ui/public/config.json
index 207731a..86b8b54 100644
--- a/ui/public/config.json
+++ b/ui/public/config.json
@@ -1,7 +1,6 @@
{
"stop_new_requests_on": false,
"zimit_ui_api": "/api/v1",
- "zimfarm_api": "https://api.farm.zimit.kiwix.org/v1",
"wikipedia_offline_article": "https://en.wikipedia.org/wiki/Offline",
"kiwix_home_page": "https://kiwix.org",
"kiwix_download_page": "https://kiwix.org/en/applications/",
diff --git a/ui/src/components/NewRequestForm.vue b/ui/src/components/NewRequestForm.vue
index 65e468b..ad84252 100644
--- a/ui/src/components/NewRequestForm.vue
+++ b/ui/src/components/NewRequestForm.vue
@@ -1,6 +1,8 @@
@@ -48,7 +57,7 @@ const hasDefinitions = computed(() => mainStore.offlinerDefinition !== undefined
diff --git a/ui/src/config.ts b/ui/src/config.ts
index 4d2f9d1..a75d1f0 100644
--- a/ui/src/config.ts
+++ b/ui/src/config.ts
@@ -6,7 +6,6 @@ import constants from './constants'
export type Config = {
stop_new_requests_on: boolean
zimit_ui_api: string
- zimfarm_api: string
wikipedia_offline_article: string
kiwix_home_page: string
kiwix_download_page: string
diff --git a/ui/src/stores/main.ts b/ui/src/stores/main.ts
index 61168d2..e2ad864 100644
--- a/ui/src/stores/main.ts
+++ b/ui/src/stores/main.ts
@@ -1,5 +1,5 @@
-import { defineStore } from 'pinia'
import axios, { AxiosError } from 'axios'
+import { defineStore } from 'pinia'
import constants from '../constants'
import { getCurrentLocale, setCurrentLocale, type Language } from '../i18n'
@@ -71,6 +71,7 @@ export type TaskData = {
downloadLink: string
progress: number
rank: number | undefined
+ offlinerDefinitionVersion: string
}
export type BlacklistEntry = {
@@ -82,6 +83,19 @@ export type BlacklistEntry = {
wp1Hint: boolean | null
}
+export interface Paginator {
+ count: number
+ skip: number
+ limit: number
+ page_size: number
+ page: number
+}
+
+export interface ListResponse {
+ meta: Paginator
+ items: T[]
+}
+
export const useMainStore = defineStore('main', {
state: () =>
({
@@ -96,6 +110,7 @@ export const useMainStore = defineStore('main', {
taskNotFound: false,
snackbarDisplayed: false,
snackbarContent: '',
+ trackerStatus: undefined,
blacklistReason: undefined
}) as RootState,
actions: {
@@ -132,6 +147,7 @@ export const useMainStore = defineStore('main', {
saveOfflinersDefinitions(offlinersDefinitions: OfflinerDefinition) {
this.offlinerDefinition = offlinersDefinitions
},
+
increment() {
this.count++
},
@@ -153,11 +169,8 @@ export const useMainStore = defineStore('main', {
async loadOfflinerDefinition() {
try {
const offlinerDefinition = (
- await axios.get(this.config.zimfarm_api + '/offliners/zimit')
+ await axios.get(this.config.zimit_ui_api + '/offliner-definition')
).data
- offlinerDefinition.flags = offlinerDefinition.flags.filter(
- (flag) => this.config.new_request_advanced_flags.indexOf(flag.data_key) > -1
- )
this.offlinerDefinition = offlinerDefinition
this.offlinerNotFound = false
} catch (error) {
@@ -262,9 +275,6 @@ export const useMainStore = defineStore('main', {
loadingStatus(state) {
return { shoudDisplay: state.loading, text: state.loadingText }
},
- offlinerFlags(state) {
- return state.offlinerDefinition?.flags || []
- },
getFormValue: (state) => {
return (name: string) => {
const formValue = state.formValues.find((formValue) => formValue.name === name)
diff --git a/ui/src/views/RequestStatus.vue b/ui/src/views/RequestStatus.vue
index 1888975..4ae1665 100644
--- a/ui/src/views/RequestStatus.vue
+++ b/ui/src/views/RequestStatus.vue
@@ -1,7 +1,8 @@