diff --git a/.gitignore b/.gitignore index a76f45d..fd3b24b 100644 --- a/.gitignore +++ b/.gitignore @@ -130,5 +130,10 @@ dmypy.json .DS_Store -# ignore all vscode, this is not standard configuration in this place +# ignore all editor files, this is not standard configuration in this place .vscode +*.vim + +# ignore worker SSH keys +id_ed25519* +id_rsa* diff --git a/Dockerfile-api b/Dockerfile-api index 3fd46af..2786741 100644 --- a/Dockerfile-api +++ b/Dockerfile-api @@ -1,4 +1,3 @@ - FROM python:3.12-alpine LABEL org.opencontainers.image.source https://github.com/openzim/zimit-frontend diff --git a/Dockerfile-ui b/Dockerfile-ui index b56e33a..e27b3ba 100644 --- a/Dockerfile-ui +++ b/Dockerfile-ui @@ -1,4 +1,4 @@ -FROM node:20-alpine as ui_builder +FROM node:20-alpine AS ui_builder RUN apk --no-cache add yarn WORKDIR /src/ui diff --git a/api/src/zimitfrontend/constants.py b/api/src/zimitfrontend/constants.py index 2a49cc8..250f6ae 100644 --- a/api/src/zimitfrontend/constants.py +++ b/api/src/zimitfrontend/constants.py @@ -85,6 +85,9 @@ class ApiConfiguration: zimfarm_username = os.getenv("_ZIMFARM_USERNAME", "-") zimfarm_password = os.getenv("_ZIMFARM_PASSWORD", "-") zimit_image = os.getenv("ZIMIT_IMAGE", "openzim/zimit:1.2.0") + zimit_definition_version = os.getenv("ZIMIT_DEFINITION_VERSION", "") + if not zimit_definition_version: + _, zimit_definition_version = zimit_image.split(":") zimit_size_limit = _get_int_setting("ZIMIT_SIZE_LIMIT", 2**30 * 4) zimit_time_limit = _get_int_setting("ZIMIT_TIME_LIMIT", 3600 * 2) diff --git a/api/src/zimitfrontend/main.py b/api/src/zimitfrontend/main.py index 07a84ab..48016af 100644 --- a/api/src/zimitfrontend/main.py +++ b/api/src/zimitfrontend/main.py @@ -8,11 +8,10 @@ from zimitfrontend import __about__ from zimitfrontend.constants import ApiConfiguration, logger -from zimitfrontend.routes import hook, requests, tracker +from zimitfrontend.routes import hook, offliners, requests, tracker class Main: - def create_app(self) -> FastAPI: self.app = FastAPI( title=__about__.__api_title__, @@ -74,6 +73,7 @@ async def internal_exception_handler( # pyright: ignore[reportUnusedFunction] api.include_router(router=requests.router) api.include_router(router=hook.router) api.include_router(router=tracker.router) + api.include_router(router=offliners.router) self.app.mount(f"/api/{__about__.__api_version__}", api) diff --git a/api/src/zimitfrontend/routes/offliners.py b/api/src/zimitfrontend/routes/offliners.py new file mode 100644 index 0000000..f5f64b2 --- /dev/null +++ b/api/src/zimitfrontend/routes/offliners.py @@ -0,0 +1,39 @@ +from http import HTTPStatus + +from fastapi import APIRouter, HTTPException + +from zimitfrontend.constants import ApiConfiguration +from zimitfrontend.routes.schemas import OfflinerDefinitionSchema +from zimitfrontend.zimfarm import query_api + +router = APIRouter( + prefix="/offliner-definition", + tags=["all"], +) + + +@router.get( + "", + summary="Get the definition for the zimit offliner", + status_code=200, + responses={ + 200: { + "description": "Zimit offliner definition schema", + }, + }, +) +def get_offliner_definition() -> OfflinerDefinitionSchema: + _, status, schema = query_api( + "GET", f"/offliners/zimit/{ApiConfiguration.zimit_definition_version}" + ) + if status != HTTPStatus.OK: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail={ + "error": ( + f"Failed to get offliner defintion on Zimfarm with HTTP {status}" + ), + "zimfarm_message": schema, + }, + ) + return OfflinerDefinitionSchema.model_validate(schema) diff --git a/api/src/zimitfrontend/routes/requests.py b/api/src/zimitfrontend/routes/requests.py index 4f1839f..59d534c 100644 --- a/api/src/zimitfrontend/routes/requests.py +++ b/api/src/zimitfrontend/routes/requests.py @@ -167,6 +167,7 @@ def _cap_limit(user_limit: int, zimit_limit: int) -> int: "tags": [], "enabled": True, "config": config, + "version": ApiConfiguration.zimit_definition_version, } # add notification callback if email supplied diff --git a/api/src/zimitfrontend/routes/schemas.py b/api/src/zimitfrontend/routes/schemas.py index 83711fd..6a72ded 100644 --- a/api/src/zimitfrontend/routes/schemas.py +++ b/api/src/zimitfrontend/routes/schemas.py @@ -24,6 +24,7 @@ class TaskInfo(CamelModel): flags: list[TaskInfoFlag] progress: int | None rank: int | None + offliner_definition_version: str class TaskCreateRequest(CamelModel): @@ -90,6 +91,7 @@ class ZimfarmTask(BaseModel): # rank is populated only on GET /requested_tasks/{id}, and not on any of # other endpoint and not on the webhook calls rank: int | None = None + version: str class HookStatus(BaseModel): @@ -101,3 +103,29 @@ class HookProcessingResult(BaseModel): mail_target: str | None = None mail_subject: str | None = None mail_body: str | None = None + + +class Choice(BaseModel): + title: str + value: str + + +class FlagSchema(BaseModel): + data_key: str + key: str + type: str + choices: list[Choice] | None = None + label: str + description: str + required: bool = False + secret: bool = False + min: int | None = None + max: int | None = None + min_length: int | None = None + max_length: int | None = None + pattern: str | None = None + + +class OfflinerDefinitionSchema(BaseModel): + help: str + flags: list[FlagSchema] diff --git a/api/src/zimitfrontend/routes/utils.py b/api/src/zimitfrontend/routes/utils.py index 89da0ed..aabb5b8 100644 --- a/api/src/zimitfrontend/routes/utils.py +++ b/api/src/zimitfrontend/routes/utils.py @@ -79,6 +79,7 @@ def get_task_info(task: Any) -> TaskInfo: else 0 ), rank=zimfarm_task.rank, + offliner_definition_version=zimfarm_task.version, ) diff --git a/api/tests/unit/routes/test_utils.py b/api/tests/unit/routes/test_utils.py index 9ea768b..a993960 100644 --- a/api/tests/unit/routes/test_utils.py +++ b/api/tests/unit/routes/test_utils.py @@ -48,6 +48,8 @@ "notification": {"ended": {"webhook": ["bla"]}}, "container": {"progress": {"partialZim": True}}, "rank": 123, + "offliner": "zimit", + "version": "initial", }, TaskInfo( id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf", @@ -62,6 +64,7 @@ ], progress=0, rank=123, + offliner_definition_version="initial", ), id="full", ), @@ -71,6 +74,8 @@ "config": {"warehouse_path": "/other", "offliner": {}}, "status": "blu", "rank": 456, + "offliner": "zimit", + "version": "initial", }, TaskInfo( id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf", @@ -81,6 +86,7 @@ flags=[], progress=0, rank=456, + offliner_definition_version="initial", ), id="simple", ), @@ -91,6 +97,8 @@ "container": {"progress": {"partialZim": False}}, "status": "bla", "rank": 456, + "offliner": "zimit", + "version": "initial", }, TaskInfo( id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf", @@ -101,6 +109,7 @@ flags=[], progress=0, rank=456, + offliner_definition_version="initial", ), id="limit_not_hit", ), @@ -111,6 +120,8 @@ "container": {"progress": {"overall": 100}}, "status": "bla", "rank": 456, + "offliner": "zimit", + "version": "initial", }, TaskInfo( id="6341c25f-aac9-41aa-b9bb-3ddee058a0bf", @@ -121,6 +132,7 @@ flags=[], progress=100, rank=456, + offliner_definition_version="initial", ), id="no_limit_info", ), @@ -153,6 +165,8 @@ def test_convert_zimfarm_task_to_info(task: Any, expected: TaskInfo): "notification": {"ended": {"webhook": ["bla"]}}, "container": {"progress": {"partialZim": True}}, "rank": 123, + "offliner": "zimit", + "version": "initial", } ) diff --git a/dev/README.md b/dev/README.md index 02017b9..1004246 100644 --- a/dev/README.md +++ b/dev/README.md @@ -2,7 +2,7 @@ This is a docker-compose configuration to be used **only** for development purpo almost zero security in the stack configuration. It is composed of the Zimit frontend UI and API (of course), but also a local Zimfarm DB, -API and UI, so that you can test the whole integration locally. +API, worker and UI, so that you can test the whole integration locally. Zimit frontend UI has two containers, one identical to production and one allowing hot reload of local developments. @@ -10,48 +10,89 @@ of local developments. Zimit frontend API has one container, slightly modified to allow hot reload of most modifications done to the source code. -Zimfarm UI, API and DB are deployed with official production Docker images. +Zimfarm UI, API worker, and DB are deployed with official production Docker images. ## List of containers -### zimit_ui_prod +### zimit-ui-prod This container is Zimit frontend UI as served in production (already compiled as a static website, so not possible to live-edit) -### zimit_ui_dev +### zimit-ui-dev This container is Zimit frontend UI served in development mode (possible to live-edit) -### zimit_api +### zimit-api This container is Zimit frontend API server (only slightly modified to enable live reload of edits) -## zimfarm_db +## zimfarm-db This container is a local Zimfarm database -## zimfarm_api +## zimfarm-api This container is a local Zimfarm API -## zimfarm_ui +## zimfarm-ui This container is a local Zimfarm UI +## zimfarm-worker-manager + +This container is a local Zimfarm worker manager. It pulls the Zimfarm task worker image to +execute tasks + +## zimfarm-receiver + +This container stores the uploaded files/logs for each task. + ## Instructions -First start the Docker-Compose stack: +### Starting the Compose Stack -```sh -cd dev -docker compose -p zimit up -d -``` +- To start the compose services (without the worker): -If it is your first execution of the dev stack, you need to create a "virtual" worker in Zimfarm DB: + ```sh + cd dev + docker compose up --pull always --build + ``` -```sh -dev/create_worker.sh -``` +- To start the compose services (with a registered worker): + + ```sh + cd dev + docker compose --profile worker up --pull always --build + ``` + +- If you are running with worker profile, you will need to create warehouse paths to upload the logs and files for each task. + ```sh + docker exec -it zimfarm-receiver bash + /contrib/create-warehouse-paths.sh + ``` + +If it is your first execution of the dev stack, you need to create offliners and a "virtual" worker in Zimfarm DB. Thus, you need to start the services without the worker +profile till you register a worker. + +- To create offliners: + + ```sh + dev/create_offliners.sh + ``` + + This pulls the latest offliner definitions from the respective offliner repositories + and registers them with the Zimfarm API. The versions of the offliner definitions + are hardcoded to "dev". This is the same as the `ZIMIT_DEFINITION_VERSION` defined in `dev/docker-compose.yml` + +- To register a worker + ```sh + dev/create_worker.sh + ``` + +NOTE: These shell scripts have been configured withs some reasonable defaults like: + +- admin username is `admin` with password `admin`. This must be the same as teh `INIT_USERNAME` AND `INIT_PASSWORD` of the `zimfarm-api` service and `_ZIMFARM_USERNAME` and `_ZIMFARM_PASSWORD` of the `zimit-api` service. See the Compose file for the definition of these environment variables. +- a worker `test_worker` with 1Gb memory, 1Gb disk and 1 CPU. These are specified in the `environment` section of the `zimfarm-worker-manager` too. If you have requested a task via Zimit UI and want to simulate a worker starting this task to observe the consequence in Zimit UI, you might use the `dev/start_first_req_task.sh`. diff --git a/dev/create-warehouse-paths.sh b/dev/create-warehouse-paths.sh new file mode 100755 index 0000000..593c5a6 --- /dev/null +++ b/dev/create-warehouse-paths.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +set -e + +mkdir -p \ + /jail/zim/freecodecamp \ + /jail/zim/gutenberg \ + /jail/zim/ifixit \ + /jail/zim/devdocs \ + /jail/zim/libretexts \ + /jail/zim/mindtouch \ + /jail/zim/mooc \ + /jail/zim/other \ + /jail/zim/phet \ + /jail/zim/stack_exchange \ + /jail/zim/ted \ + /jail/zim/videos \ + /jail/zim/vikidia \ + /jail/zim/wikibooks \ + /jail/zim/wikihow \ + /jail/zim/wikinews \ + /jail/zim/wikipedia \ + /jail/zim/wikiquote \ + /jail/zim/wikisource \ + /jail/zim/wikiversity \ + /jail/zim/wikivoyage \ + /jail/zim/wiktionary \ + /jail/zim/zimit + +chmod 777 \ + /jail/zim/freecodecamp \ + /jail/zim/gutenberg \ + /jail/zim/ifixit \ + /jail/zim/devdocs \ + /jail/zim/libretexts \ + /jail/zim/mindtouch \ + /jail/zim/mooc \ + /jail/zim/other \ + /jail/zim/phet \ + /jail/zim/stack_exchange \ + /jail/zim/ted \ + /jail/zim/videos \ + /jail/zim/vikidia \ + /jail/zim/wikibooks \ + /jail/zim/wikihow \ + /jail/zim/wikinews \ + /jail/zim/wikipedia \ + /jail/zim/wikiquote \ + /jail/zim/wikisource \ + /jail/zim/wikiversity \ + /jail/zim/wikivoyage \ + /jail/zim/wiktionary \ + /jail/zim/zimit diff --git a/dev/create_offliners.sh b/dev/create_offliners.sh new file mode 100755 index 0000000..c70d927 --- /dev/null +++ b/dev/create_offliners.sh @@ -0,0 +1,174 @@ +#!/bin/bash + +# Script to register offliners in Zimfarm +# - retrieve an admin token +# - fetch offliner definitions from their respective GitHub repositories +# - register each offliner and its definition with the backend API + +set -e + +# List of offliners to register +OFFLINERS=("zimit") + +# Base URL for offliner definitions +OFFLINER_DEFINITIONS_BASE_URL="https://raw.githubusercontent.com/openzim" + +# zimfarm API URL +ZIMFARM_BASE_API_URL="http://localhost:8004/v2" + +# Offliner configuration data for API registration +declare -A offliner_configs + +# Initialize offliner configurations +setup_offliner_configs() { + # DashModel offliners + offliner_configs["zimit"]='{"base_model":"CamelModel","docker_image_name":"openzim/zimit","command_name":"zimit"}' +} + +die() { + echo "ERROR: $1" >&2 + exit 1 +} + +fetch_offliner_definition() { + local offliner_name="$1" + local url="${OFFLINER_DEFINITIONS_BASE_URL}/${offliner_name}/refs/heads/main/offliner-definition.json" + + local definition + if ! definition=$(curl -s -f "$url"); then + die "Failed to fetch offliner definition for ${offliner_name} from ${url}" + fi + + if ! echo "$definition" | jq . >/dev/null 2>&1; then + die "Invalid JSON received for ${offliner_name}" + fi + + echo "$definition" +} + +register_offliner_via_api() { + local offliner_id="$1" + local config="${offliner_configs[$offliner_id]}" + echo "Registering ${offliner} via API..." + + if [ -z "$config" ]; then + die "No configuration found for offliner: ${offliner_id}" + fi + # Create the payload with offliner_id used as ci_secret_hash + local payload + payload=$(echo "$config" | jq --arg offliner_id "$offliner_id" '. + {"ci_secret_hash": $offliner_id, "offliner_id": $offliner_id}') + + local response + local http_code + response=$(curl -s -w "\n%{http_code}" -X 'POST' \ + "${ZIMFARM_BASE_API_URL}/offliners" \ + -H 'accept: application/json' \ + -H "Authorization: Bearer ${ZF_ADMIN_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "$payload") + + # Extract HTTP status code (last line) + http_code=$(echo "$response" | tail -n1) + # Extract response body (all lines except last) + response=$(echo "$response" | head -n -1) + + case "$http_code" in + 201) + echo "Successfully registered ${offliner_id}" + ;; + 409) + echo "WARNING: Offliner ${offliner_id} already exists, skipping..." + ;; + *) + local error_msg + error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code") + die "Registration failed for ${offliner_id}: ${error_msg}" + ;; + esac +} + +# Function to create offliner definition schema +create_offliner_definition() { + local offliner_id="$1" + local spec="${offliner_definitions[$offliner_id]}" + + if [ -z "$spec" ]; then + die "No definition found for offliner: ${offliner_id}" + fi + + echo "Creating definition for ${offliner_id}..." + + # Create the payload for schema creation + local payload + payload=$(jq -n \ + --arg version "dev" \ + --arg ci_secret "$offliner_id" \ + --argjson spec "$spec" \ + '{version: $version, ci_secret: $ci_secret, spec: $spec}') + + + # Create the schema via API + local response + local http_code + response=$(curl -s -w "\n%{http_code}" -X 'POST' \ + "${ZIMFARM_BASE_API_URL}/offliners/${offliner_id}/versions" \ + -H 'accept: application/json' \ + -H "Authorization: Bearer ${ZF_ADMIN_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "$payload") + + # Extract HTTP status code (last line) + http_code=$(echo "$response" | tail -n1) + # Extract response body (all lines except last) + response=$(echo "$response" | head -n -1) + + case "$http_code" in + 201) + echo "Successfully created schema for ${offliner_id}" + ;; + 409) + echo "WARNING: Offliner ${offliner_id} already exists, skipping..." + ;; + *) + local error_msg + error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code") + die "Definition creation failed for ${offliner_id}: ${error_msg}" + ;; + esac +} + +echo "Retrieving admin access token" + +ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \ + "${ZIMFARM_BASE_API_URL}/auth/authorize" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{"username": "admin", "password": "admin"}' \ + | jq -r '.access_token')" + +if [ -z "$ZF_ADMIN_TOKEN" ] || [ "$ZF_ADMIN_TOKEN" = "null" ]; then + die "Failed to retrieve admin access token" +fi + +echo "Admin token retrieved successfully" + +# Setup offliner configurations for API registration +setup_offliner_configs + +# Declare associative array to store offliner definitions +declare -A offliner_definitions + +echo "Fetching offliner definitions..." + +for offliner in "${OFFLINERS[@]}"; do + echo "Fetching ${offliner} definition..." + offliner_definitions["$offliner"]=$(fetch_offliner_definition "$offliner" | jq -c '.') +done + +echo "All offliner definitions fetched successfully" + +for offliner in "${OFFLINERS[@]}"; do + register_offliner_via_api "$offliner" + create_offliner_definition "$offliner" +done +echo "All offliners registered via API successfully!" diff --git a/dev/create_worker.sh b/dev/create_worker.sh index 746ca9b..fe5c694 100755 --- a/dev/create_worker.sh +++ b/dev/create_worker.sh @@ -1,28 +1,110 @@ -echo "Retrieving access token" +#!/bin/bash + +# Call it once to create a `test_worker`: +# - retrieve an admin token +# - create the `test_worker`` user +# - create the associated worker object +# - upload a test public key. +# +# To be used to have a "real" test worker for local development, typically to start +# a worker manager or a task manager or simply assign tasks to a worker in the UI/API + +set -e + +ZIMFARM_BASE_API_URL="http://localhost:8004/v2" + +die() { + echo "ERROR: $1" >&2 + exit 1 +} + +check_http_code() { + local http_code="$1" + local response="$2" + + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + : + else + error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code") + die "Could not checkin worker: ${error_msg}" + fi +} + +echo "Retrieving admin access token" ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \ - 'http://localhost:8002/v2/auth/authorize' \ + "${ZIMFARM_BASE_API_URL}/auth/authorize" \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ - -d '{"username":"admin","password":"admin"}' \ + -d '{"username": "admin", "password": "admin"}' \ | jq -r '.access_token')" +if [ -z "$ZF_ADMIN_TOKEN" ] || [ "$ZF_ADMIN_TOKEN" = "null" ]; then + die "Failed to retrieve admin access token" +fi -echo "Worker check-in (will create if missing)" +echo "Create test_worker user" -curl -s -X 'PUT' \ - 'http://localhost:8002/v2/workers/worker/check-in' \ +curl -s -X 'POST' \ + "${ZIMFARM_BASE_API_URL}/users" \ -H 'accept: */*' \ -H 'Content-Type: application/json' \ -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \ -d '{ - "username": "admin", + "role":"worker", + "username": "test_worker", + "email":"test_worker@acme.com", + "password":"test_worker" +}' + +echo "Retrieving test_worker access token" + +ZF_USER_TOKEN="$(curl -s -X 'POST' \ + "${ZIMFARM_BASE_API_URL}/auth/authorize" \ + -H 'accept: application/json' \ + -H 'Content-Type: application/json' \ + -d '{"username": "test_worker", "password": "test_worker"}' \ + | jq -r '.access_token')" + +if [ -z "$ZF_USER_TOKEN" ] || [ "$ZF_USER_TOKEN" = "null" ]; then + die "Failed to retrieve worker access token" +fi + +response=$(curl -s -w "\n%{http_code}" -X 'PUT' \ + "${ZIMFARM_BASE_API_URL}/workers/test_worker/check-in" \ + -H 'accept: */*' \ + -H 'Content-Type: application/json' \ + -H "Authorization: Bearer $ZF_USER_TOKEN" \ + -d '{ + "username": "test_worker", "cpu": 3, - "memory": 1024, - "disk": 0, + "memory": 2147483648, + "disk": 4294967296, "offliners": [ "zimit" ] -}' +}') + +http_code=$(echo "$response" | tail -n1) +response=$(echo "$response" | head -n -1) + +check_http_code "$http_code" "$response" + +echo "Generating SSH key pair (Ed25519)" +ssh-keygen -t ed25519 -f id_ed25519 -N "" +payload="$(jq -n --arg key "$(< id_ed25519.pub)" '{key: $key}')" + + +echo "Uploading worker keys to API" +response=$(curl -s -w "\n%{http_code}" -X POST "${ZIMFARM_BASE_API_URL}/users/test_worker/keys" \ + -H 'accept: */*' \ + -H "Authorization: Bearer $ZF_USER_TOKEN" \ + -H 'Content-Type: application/json; charset=utf-8' \ + -d "$payload") + +http_code=$(echo "$response" | tail -n1) +response=$(echo "$response" | head -n -1) + +check_http_code "$http_code" "$response" echo "DONE" diff --git a/dev/docker-compose.yml b/dev/docker-compose.yml index 9cd5502..b889e77 100644 --- a/dev/docker-compose.yml +++ b/dev/docker-compose.yml @@ -1,35 +1,98 @@ services: zimfarm-db: - image: postgres:15.2-bullseye + image: postgres:17.3-bookworm + container_name: zimarm-postgresdb ports: - 127.0.0.1:5432:5432 volumes: - zimfarm_data:/var/lib/postgresql/data + - ./postgres-initdb:/docker-entrypoint-initdb.d environment: - POSTGRES_DB=zimfarm - POSTGRES_USER=zimfarm - POSTGRES_PASSWORD=zimpass + healthcheck: + test: ["CMD", "pg_isready", "-q", "-d", "dbname=zimfarm user=zimfarm"] + interval: 10s + timeout: 5s + retries: 3 zimfarm-api: - image: ghcr.io/openzim/zimfarm-dispatcher:latest + image: ghcr.io/openzim/zimfarm-backend:latest + container_name: zimfarm-api ports: - 127.0.0.1:8004:80 environment: BINDING_HOST: 0.0.0.0 JWT_SECRET: DH8kSxcflUVfNRdkEiJJCn2dOOKI3qfw POSTGRES_URI: postgresql+psycopg://zimfarm:zimpass@zimfarm-db:5432/zimfarm - ALEMBIC_UPGRADE_HEAD_ON_START: "1" - ZIMIT_USE_RELAXED_SCHEMA: "y" + INIT_USERNAME: admin + INIT_PASSWORD: admin + ZIMIT_USE_RELAXED_SCHEMA: true + ALLOWED_ORIGINS: http://localhost:8003 + # upload artifacts, logs and zim to receiver for simplicity + ARTIFACTS_UPLOAD_URI: sftp://uploader@zimfarm-receiver:22/logs/ # reusing logs dir, kind of a hack + LOGS_UPLOAD_URI: sftp://uploader@zimfarm-receiver:22/logs/ + ZIM_UPLOAD_URI: sftp://uploader@zimfarm-receiver:22/zim/ + ZIMCHECK_OPTION: --all depends_on: - - zimfarm-db + zimfarm-db: + condition: service_healthy + command: + - uvicorn + - zimfarm_backend.main:app + - --host + - "0.0.0.0" + - --port + - "80" + - --proxy-headers + - --forwarded-allow-ips + - "*" zimfarm-ui: image: ghcr.io/openzim/zimfarm-ui:latest + container_name: zimfarm-ui ports: - 127.0.0.1:8003:80 + volumes: + - ./zimfarm_ui_dev/config.json:/usr/share/nginx/html/config.json:ro + depends_on: + zimfarm-api: + condition: service_healthy + zimfarm-worker-manager: + image: ghcr.io/openzim/zimfarm-worker-manager:latest + container_name: zimfarm-worker-manager + depends_on: + zimfarm-api: + condition: service_healthy + command: worker-manager --webapi-uri 'http://zimfarm-api:80/v2' --username test_worker --name test_worker + environment: + - DEBUG=true + - TASK_WORKER_IMAGE=ghcr.io/openzim/zimfarm-task-worker:latest + - ENVIRONMENT=development + - ZIMFARM_DISK=4Gb + - ZIMFARM_MEMORY=2Gb + - ZIMFARM_CPU=1 + - POLL_INTERVAL=10 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./id_ed25519:/etc/ssh/keys/zimfarm + profiles: + - worker + zimfarm-receiver: + image: ghcr.io/openzim/zimfarm-receiver:latest + container_name: zimfarm-receiver + ports: + - 127.0.0.1:8222:22 + volumes: + - ./create-warehouse-paths.sh:/contrib/create-warehouse-paths.sh environment: - ZIMFARM_WEBAPI: http://localhost:8004/v1 + - ZIMFARM_WEBAPI=http://zimfarm-api:80/v2 depends_on: - - zimfarm-api + zimfarm-api: + condition: service_healthy + profiles: + - worker zimit-api: + container_name: zimit-api build: dockerfile: Dockerfile-api context: .. @@ -52,16 +115,19 @@ services: ports: - 127.0.0.1:8002:80 environment: - INTERNAL_ZIMFARM_WEBAPI: http://zimfarm-api:80/v1 + INTERNAL_ZIMFARM_WEBAPI: http://zimfarm-api:80/v2 _ZIMFARM_USERNAME: admin _ZIMFARM_PASSWORD: admin - TASK_WORKER: worker + TASK_WORKER: test_worker HOOK_TOKEN: a_very_secret_token CALLBACK_BASE_URL: http://zimit-api:80/api/v1/requests/hook DIGEST_KEY: d1a2df7f0a229cc6 + ZIMIT_IMAGE: openzim/zimit:3.0.5 + ZIMIT_DEFINITION_VERSION: dev depends_on: - zimfarm-api zimit-ui-dev: + container_name: zimit-ui-dev build: dockerfile: dev/zimit_ui_dev/Dockerfile context: .. @@ -74,6 +140,7 @@ services: depends_on: - zimit-api zimit-ui-prod: + container_name: zimit-ui-prod build: dockerfile: Dockerfile-ui context: .. diff --git a/dev/postgres-initdb/init-extensions.sh b/dev/postgres-initdb/init-extensions.sh new file mode 100755 index 0000000..ddeca2e --- /dev/null +++ b/dev/postgres-initdb/init-extensions.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; +EOSQL diff --git a/dev/start_first_req_task.sh b/dev/start_first_req_task.sh index 1c3fc1f..c4bbf7a 100755 --- a/dev/start_first_req_task.sh +++ b/dev/start_first_req_task.sh @@ -1,3 +1,22 @@ +#!/bin/bash + +die() { + echo "ERROR: $1" >&2 + exit 1 +} + +check_http_code() { + local http_code="$1" + local response="$2" + + if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + : + else + error_msg=$(echo "$response" | jq -r '.errors // .message // .detail // "Unknown error"' 2>/dev/null || echo "HTTP $http_code") + die "Could not checkin worker: ${error_msg}" + fi +} + echo "Retrieving access token" ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \ @@ -7,25 +26,31 @@ ZF_ADMIN_TOKEN="$(curl -s -X 'POST' \ -d '{"username":"admin","password":"admin"}' \ | jq -r '.access_token')" +if [ -z "$ZF_ADMIN_TOKEN" ] || [ "$ZF_ADMIN_TOKEN" = "null" ]; then + die "Failed to retrieve admin access token" +fi + echo "Get last requested task" LAST_TASK_ID="$(curl -s -X 'GET' \ 'http://localhost:8004/v2/requested-tasks' \ -H 'accept: application/json' \ -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \ - | jq -r '.items[0]._id')" + | jq -r '.items[0].id')" if [ "$LAST_TASK_ID" = "null" ]; then - echo "No pending requested task. Exiting script." - exit 1 + die "No pending requested task. Exiting script." fi echo "Start task" -curl -s -X 'POST' \ - "http://localhost:8004/v2/tasks/$LAST_TASK_ID?worker_name=worker" \ +response=$(curl -s -w "\n%{http_code}" -X 'POST' \ + "http://localhost:8004/v2/tasks/$LAST_TASK_ID?worker_name=test_worker" \ -H 'accept: application/json' \ -H "Authorization: Bearer $ZF_ADMIN_TOKEN" \ - -d '' +) +http_code=$(echo "$response" | tail -n1) + +check_http_code "$http_code" echo "DONE" diff --git a/dev/zimfarm_ui_dev/config.json b/dev/zimfarm_ui_dev/config.json new file mode 100644 index 0000000..cbea836 --- /dev/null +++ b/dev/zimfarm_ui_dev/config.json @@ -0,0 +1,3 @@ +{ + "ZIMFARM_WEBAPI": "http://localhost:8004/v2" +} diff --git a/dev/zimit_ui_dev/config.json b/dev/zimit_ui_dev/config.json index 6c4bc3c..be99129 100644 --- a/dev/zimit_ui_dev/config.json +++ b/dev/zimit_ui_dev/config.json @@ -1,7 +1,6 @@ { "stop_new_requests_on": false, "zimit_ui_api": "http://localhost:8002/api/v1", - "zimfarm_api": "http://localhost:8004/v1", "wikipedia_offline_article": "https://en.wikipedia.org/wiki/Offline", "kiwix_home_page": "https://kiwix.org", "kiwix_download_page": "https://kiwix.org/en/applications/", diff --git a/ui/public/config.json b/ui/public/config.json index 207731a..86b8b54 100644 --- a/ui/public/config.json +++ b/ui/public/config.json @@ -1,7 +1,6 @@ { "stop_new_requests_on": false, "zimit_ui_api": "/api/v1", - "zimfarm_api": "https://api.farm.zimit.kiwix.org/v1", "wikipedia_offline_article": "https://en.wikipedia.org/wiki/Offline", "kiwix_home_page": "https://kiwix.org", "kiwix_download_page": "https://kiwix.org/en/applications/", diff --git a/ui/src/components/NewRequestForm.vue b/ui/src/components/NewRequestForm.vue index 65e468b..ad84252 100644 --- a/ui/src/components/NewRequestForm.vue +++ b/ui/src/components/NewRequestForm.vue @@ -1,6 +1,8 @@ @@ -48,7 +57,7 @@ const hasDefinitions = computed(() => mainStore.offlinerDefinition !== undefined diff --git a/ui/src/config.ts b/ui/src/config.ts index 4d2f9d1..a75d1f0 100644 --- a/ui/src/config.ts +++ b/ui/src/config.ts @@ -6,7 +6,6 @@ import constants from './constants' export type Config = { stop_new_requests_on: boolean zimit_ui_api: string - zimfarm_api: string wikipedia_offline_article: string kiwix_home_page: string kiwix_download_page: string diff --git a/ui/src/stores/main.ts b/ui/src/stores/main.ts index 61168d2..e2ad864 100644 --- a/ui/src/stores/main.ts +++ b/ui/src/stores/main.ts @@ -1,5 +1,5 @@ -import { defineStore } from 'pinia' import axios, { AxiosError } from 'axios' +import { defineStore } from 'pinia' import constants from '../constants' import { getCurrentLocale, setCurrentLocale, type Language } from '../i18n' @@ -71,6 +71,7 @@ export type TaskData = { downloadLink: string progress: number rank: number | undefined + offlinerDefinitionVersion: string } export type BlacklistEntry = { @@ -82,6 +83,19 @@ export type BlacklistEntry = { wp1Hint: boolean | null } +export interface Paginator { + count: number + skip: number + limit: number + page_size: number + page: number +} + +export interface ListResponse { + meta: Paginator + items: T[] +} + export const useMainStore = defineStore('main', { state: () => ({ @@ -96,6 +110,7 @@ export const useMainStore = defineStore('main', { taskNotFound: false, snackbarDisplayed: false, snackbarContent: '', + trackerStatus: undefined, blacklistReason: undefined }) as RootState, actions: { @@ -132,6 +147,7 @@ export const useMainStore = defineStore('main', { saveOfflinersDefinitions(offlinersDefinitions: OfflinerDefinition) { this.offlinerDefinition = offlinersDefinitions }, + increment() { this.count++ }, @@ -153,11 +169,8 @@ export const useMainStore = defineStore('main', { async loadOfflinerDefinition() { try { const offlinerDefinition = ( - await axios.get(this.config.zimfarm_api + '/offliners/zimit') + await axios.get(this.config.zimit_ui_api + '/offliner-definition') ).data - offlinerDefinition.flags = offlinerDefinition.flags.filter( - (flag) => this.config.new_request_advanced_flags.indexOf(flag.data_key) > -1 - ) this.offlinerDefinition = offlinerDefinition this.offlinerNotFound = false } catch (error) { @@ -262,9 +275,6 @@ export const useMainStore = defineStore('main', { loadingStatus(state) { return { shoudDisplay: state.loading, text: state.loadingText } }, - offlinerFlags(state) { - return state.offlinerDefinition?.flags || [] - }, getFormValue: (state) => { return (name: string) => { const formValue = state.formValues.find((formValue) => formValue.name === name) diff --git a/ui/src/views/RequestStatus.vue b/ui/src/views/RequestStatus.vue index 1888975..4ae1665 100644 --- a/ui/src/views/RequestStatus.vue +++ b/ui/src/views/RequestStatus.vue @@ -1,7 +1,8 @@