Feat: video generation, user guides and semantic assertions

.gitignore

@@ -132,6 +132,10 @@ __pypackages__/
 celerybeat-schedule
 celerybeat.pid
 
+# Runtime artifacts
+artifacts/*
+!artifacts/.gitkeep
+
 # SageMath parsed files
 *.sage.py
 

Dockerfile

@@ -4,7 +4,7 @@ FROM python:3.12-slim AS builder
 WORKDIR /app
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends gcc && \
+    apt-get install -y --no-install-recommends gcc ffmpeg && \
     rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir uv
@@ -25,6 +25,10 @@ ENV ENVIRONMENT=production
 ENV PYTHONUNBUFFERED=1
 ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
 
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends ffmpeg && \
+    rm -rf /var/lib/apt/lists/*
+
 RUN playwright install --with-deps chromium && \
     mkdir -p /app/logs && \
     useradd -m docgen && \

Dockerfile.dev

@@ -3,7 +3,7 @@ FROM python:3.12-slim
 WORKDIR /app
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends gcc && \
+    apt-get install -y --no-install-recommends gcc ffmpeg && \
     rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir uv
@@ -19,4 +19,4 @@ RUN playwright install --with-deps chromium
 ENV ENVIRONMENT=development
 ENV PYTHONUNBUFFERED=1
 
-CMD ["watchfiles", "arq src.worker.WorkerSettings", "src"]
+CMD ["watchfiles", "arq src.worker.WorkerSettings", "src"]

README.md

@@ -25,7 +25,7 @@
 - `src/models/queries.py`: centralized Cypher statements.
 - `src/services/labeling/`: page analysis, element naming, action descriptions, and Playwright-based transition labeling.
 - `src/core/`: settings, logging, Neo4j, Redis, and Playwright lifecycle management.
-- `src/services/bdd`, `guides`, and `video` currently contain placeholders and no implemented business logic.
+- `src/services/video`: live-URL MP4 walkthrough generation using Playwright screenshots, composited cursor/zoom effects, optional audio, and ffmpeg encoding.
 
 ## Existing Data Models
 - Neo4j `State` node:
@@ -131,3 +131,71 @@
 - Preserve ARQ task names registered in `WorkerSettings`.
 - Preserve early logging initialization, Neo4j warning-level filtering, rotating file logging, and `/app/logs` persistence.
 - Production images must include Playwright Chromium and run as the non-root `docgen` user.
+
+## Video Generation Task
+
+`task_generate_video` creates an MP4 product walkthrough from the same flow input shape used by BDD:
+
+```json
+{
+  "session_id": "session-id",
+  "flows": [
+    {
+      "checkpoint_hash": "start-state-hash",
+      "transition_ids": ["transition-1"]
+    }
+  ]
+}
+```
+
+The task waits for labeling completion just like BDD, opens the checkpoint/start URL in Playwright, performs the recorded actions on the live page, and renders a reference-style walkthrough: the app appears as a smaller floating window with shadow on a neutral background, with smooth zoom, cursor movement, typing, and UI sounds.
+
+```json
+{
+  "status": "success",
+  "session_id": "session-id",
+  "artifact_path": "artifacts/videos/session-id-video.mp4",
+  "duration_seconds": 4.2,
+  "resolution": "1280x720",
+  "fps": 30,
+  "flow_count": 1
+}
+```
+
+By default, Docker mounts container output from `/app/artifacts` to the host project folder `artifacts/`, so generated videos are visible at `artifacts/videos/<session-id>-video.mp4`. Set `DOCGEN_ARTIFACTS_DIR` to mount a different host directory.
+
+Runtime requirements:
+- Playwright Chromium for live-page rendering. The checkpoint URL must be reachable from inside the DocGen container.
+- Pillow for frame compositing.
+- `ffmpeg` for MP4/H.264 encoding and optional audio muxing.
+
+Rendering notes:
+- The renderer does not use a spotlight/dim mask around target elements.
+- The click pulse animation is intentionally omitted.
+- Higher `VIDEO_ACTION_SPEED` values make transitions faster; lower values make them slower.
+- Audio uses click and keypress sounds only, then normalizes the WAV mix before ffmpeg muxes it into the MP4.
+
+Environment defaults:
+- `VIDEO_MAX_RETRIES`
+- `VIDEO_RETRY_DELAY_SECONDS`
+- `VIDEO_OUTPUT_DIR`
+- `VIDEO_DEFAULT_WIDTH`
+- `VIDEO_DEFAULT_HEIGHT`
+- `VIDEO_DEFAULT_FPS`
+- `VIDEO_DEFAULT_AUDIO_ENABLED`
+- `VIDEO_DEFAULT_AUDIO_VOLUME`
+- `VIDEO_AUDIO_GAIN`
+- `VIDEO_AUDIO_NORMALIZE_PEAK`
+- `VIDEO_ACTION_SPEED`
+- `VIDEO_WINDOW_SCALE`
+- `VIDEO_WINDOW_BACKGROUND_COLOR`
+- `VIDEO_WINDOW_SHADOW_STRENGTH`
+- `VIDEO_WINDOW_BORDER_RADIUS`
+- `VIDEO_PRE_ACTION_HOLD_SECONDS`
+- `VIDEO_ZOOM_SECONDS`
+- `VIDEO_CURSOR_TRAVEL_SECONDS`
+- `VIDEO_ACTION_HOLD_SECONDS`
+- `VIDEO_PAGE_SETTLE_SECONDS`
+- `VIDEO_RELEASE_SECONDS`
+- `VIDEO_TYPING_FRAME_SECONDS`
+- `VIDEO_RANDOM_SEED`

artifacts/.gitkeep

@@ -0,0 +1 @@
+

docker-compose.yml

@@ -23,6 +23,14 @@ services:
       - CONTEXT_DISTANCE_THRESHOLD=${CONTEXT_DISTANCE_THRESHOLD:-0.40}
       - BDD_RETRY_DELAY_SECONDS=${BDD_RETRY_DELAY_SECONDS:-30}
       - BDD_MAX_RETRIES=${BDD_MAX_RETRIES:-5}
+      - VIDEO_RETRY_DELAY_SECONDS=${VIDEO_RETRY_DELAY_SECONDS:-30}
+      - VIDEO_MAX_RETRIES=${VIDEO_MAX_RETRIES:-5}
+      - VIDEO_OUTPUT_DIR=${VIDEO_OUTPUT_DIR:-artifacts/videos}
+      - VIDEO_DEFAULT_WIDTH=${VIDEO_DEFAULT_WIDTH:-1280}
+      - VIDEO_DEFAULT_HEIGHT=${VIDEO_DEFAULT_HEIGHT:-720}
+      - VIDEO_DEFAULT_FPS=${VIDEO_DEFAULT_FPS:-30}
+      - VIDEO_ACTION_SPEED=${VIDEO_ACTION_SPEED:-1.0}
+      - VIDEO_RANDOM_SEED=${VIDEO_RANDOM_SEED:-42}
 
       # External Services
       - NEO4J_URL=${NEO4J_URL:-bolt://neo4j:7687}
@@ -41,7 +49,5 @@ services:
       - "host.docker.internal:host-gateway"
     restart: unless-stopped
     volumes:
-      - docgen_logs:/app/logs
-
-volumes:
-  docgen_logs:
+      - ${DOCGEN_LOGS_DIR:-${DOCGEN_DIR}/logs}:/app/logs
+      - ${DOCGEN_ARTIFACTS_DIR:-${DOCGEN_DIR}/artifacts}:/app/artifacts

overrides/api.dev.yml

@@ -10,4 +10,3 @@ services:
       - API_BASE_URL=${API_BASE_URL:-http://api:3000/api/v1}
     volumes:
       - ${DOCGEN_DIR}/src:/app/src
-      - ${DOCGEN_DIR}/logs:/app/logs

pyproject.toml

@@ -11,6 +11,7 @@ dependencies = [
     "neo4j>=6.2.0",
     "playwright>=1.60.0",
     "coverit-contracts",
+    "pillow>=11.0.0",
     "pydantic>=2.13.4",
     "pydantic-settings>=2.14.1",
     "redis>=5.3.1",

src/core/config.py

@@ -18,6 +18,23 @@ class Settings(BaseSettings):
     bdd_split_features: bool = False
     bdd_feature_similarity_threshold: float = 0.42
     bdd_singleton_merge_threshold: float = 0.25
+    semantic_assertions_enabled: bool = False
+    semantic_assertions_provider: str = "gemini"
+    semantic_assertions_model_base_url: str = "http://localhost:8000/v1"
+    semantic_assertions_model_name: str = "Qwen/Qwen2.5-Coder-1.5B-Instruct"
+    semantic_assertions_gemini_model: str = "gemini-2.5-flash-lite"
+    semantic_assertions_timeout_seconds: int = 20
+    semantic_assertions_max_assertions_per_scenario: int = 2
+    semantic_assertions_min_confidence: float = 0.65
+    semantic_assertions_html_summary_max_chars: int = 12000
+    video_retry_delay_seconds: int = 30
+    video_max_retries: int = 5
+    video_output_dir: str = "artifacts/videos"
+    video_default_width: int = 1280
+    video_default_height: int = 720
+    video_default_fps: int = 30
+    video_action_speed: float = 0.1
+    video_random_seed: int = 42
     jira_report_poll_batch_size: int = 3
 
     # Application
@@ -30,6 +47,7 @@ class Settings(BaseSettings):
     redis_url: str = "redis://redis:6379"
     api_base_url: str = "http://localhost:3000/api/v1"
     internal_service_token: str = ""
+    gemini_api_key: str = "AIzaSyD__8pzxbF02FoswxBcdDsBKOm-PHTUpAQ"
     neo4j_url: str = "bolt://localhost:7687"
     neo4j_password: str = "password"
     neo4j_username: str = "neo4j"

src/models/bdd.py

@@ -34,6 +34,7 @@ class ResolvedState(BaseModel):
     name: str
     description: str = ""
     url: str = ""
+    html: str = ""
     labeling_status: str
 
 
@@ -71,6 +72,18 @@ class FeaturePlan(BaseModel):
     scenarios: list[ScenarioPlan]
 
 
+class SemanticAssertion(BaseModel):
+    id: str
+    db_id: str
+    label: str
+    description: str = ""
+    target_state_db_id: str
+    context_id: str
+    severity: str = "blocking"
+    definition: dict[str, Any]
+    semantic: dict[str, Any] = Field(default_factory=dict)
+
+
 @dataclass(frozen=True)
 class CompiledFeature:
     id: str
@@ -84,5 +97,6 @@ class CompiledBdd:
     features: list[CompiledFeature]
     states: dict[str, dict]
     transitions: dict[str, dict]
+    assertions: dict[str, dict]
     feature_name: str | None = None
     feature_text: str | None = None

src/models/graph.py

@@ -20,7 +20,7 @@ class CrawlerTransition(BaseModel):
     from_state_id: str
     to_state_id: str
     locator: str
-    action_value: List[Dict[str, Any]]
+    action_value: List[Dict[str, Any]] = Field(default_factory=list)
 
 
 class LabeledTransition(BaseModel):

src/models/guides.py

@@ -0,0 +1,45 @@
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class UserGuideInput(BaseModel):
+    session_id: str = Field(min_length=1)
+    start_state_hash: str = Field(min_length=1)
+    end_state_hash: str = Field(min_length=1)
+
+
+class ResolvedGuideState(BaseModel):
+    db_id: str
+    state_hash: str
+    name: str
+    description: str = ""
+    url: str = ""
+    labeling_status: str = ""
+
+
+class ResolvedGuideTransition(BaseModel):
+    db_id: str
+    transition_id: str
+    name: str = ""
+    action: str
+    action_type: str = ""
+    locator_value: str = ""
+    labeling_status: str = ""
+    from_state: ResolvedGuideState
+    to_state: ResolvedGuideState
+
+
+class ResolvedGuidePath(BaseModel):
+    start_state: ResolvedGuideState
+    end_state: ResolvedGuideState
+    transitions: list[ResolvedGuideTransition] = Field(default_factory=list)
+
+
+class UserGuideResult(BaseModel):
+    status: Literal["success"]
+    session_id: str
+    start_state_hash: str
+    end_state_hash: str
+    guide: str
+    step_count: int