pip -> uv, CI and deploy updated accordingly (#7)

* pip -> uv, CI and deploy updated accordingly

* mcdocs CI fix

* tests CI fix - uv installs python 3.12 now

* tests CI fix - uv installs python 3.12 now

* ENVPATH fix

* uv in docker compose

* gh fix - remove backend/app volume mount

* more caching, less logs in backend tests ci job

* - pinned uv version to 0.9.17
- uv tool run for docs.yml

* - uv tool run for docs.yml
- uv for security ci

* warning fixes

* docker base img for all support containers, and updated all other stuff

* fixed bake-action stuff in ci

* fix dependencies

* docker scan fixes

* field serializer fix

* removed unused dependency

* added doc about ci/cd

Author: HardMax71

.github/workflows/docker.yml

@@ -13,17 +13,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+      - name: Build base image
+        run: |
+          docker build -f ./backend/Dockerfile.base -t integr8scode-base:latest ./backend
+
       - name: Build Docker image
-        uses: docker/build-push-action@v4
-        with:
-          context: ./backend
-          push: false
-          load: true
-          tags: integr8scode:test
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+        run: |
+          DOCKER_BUILDKIT=1 docker build \
+            --build-context base=docker-image://integr8scode-base:latest \
+            -t integr8scode:test \
+            ./backend
       - name: Run Trivy vulnerability scanner
         uses: aquasecurity/trivy-action@master
         with:

.github/workflows/docs.yml

@@ -31,20 +31,13 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v5
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: '3.12'
-
-      - name: Cache pip dependencies
-        uses: actions/cache@v4
-        with:
-          key: mkdocs-${{ hashFiles('mkdocs.yml') }}
-          path: ~/.cache/pip
-          restore-keys: mkdocs-
+          enable-cache: true
 
       - name: Install MkDocs
-        run: pip install mkdocs-material mkdocs-mermaid2-plugin mkdocs-swagger-ui-tag
+        run: uv tool install mkdocs --with mkdocs-material --with mkdocs-mermaid2-plugin --with mkdocs-swagger-ui-tag
 
       - name: Download OpenAPI spec
         run: |
@@ -53,7 +46,7 @@ jobs:
           > docs/reference/openapi.json
 
       - name: Build documentation
-        run: mkdocs build --strict
+        run: uv tool run mkdocs build --strict
 
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/mypy.yml

@@ -13,19 +13,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: '3.12'
-      - name: Install dependencies
+          enable-cache: true
+          cache-dependency-glob: "backend/uv.lock"
+
+      - name: Install Python and dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install mypy
-          pip install -r backend/requirements.txt
-          pip install -r backend/requirements-dev.txt
+          cd backend
+          uv python install 3.12
+          uv sync --frozen
+
       - name: Run mypy
         env:
           SECRET_KEY: ${{ secrets.TEST_SECRET_KEY }}
         run: |
           cd backend
-          mypy --config-file pyproject.toml .
+          uv run mypy --config-file pyproject.toml .

.github/workflows/ruff.yml

@@ -13,13 +13,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: '3.12'
-      - name: Install dependencies
+          enable-cache: true
+          cache-dependency-glob: "backend/uv.lock"
+
+      - name: Install Python and dependencies
         run: |
-          python -m pip install --upgrade pip
-          pip install ruff
+          cd backend
+          uv python install 3.12
+          uv sync --frozen
+
       - name: Run ruff
-        run: ruff check backend/ --config backend/pyproject.toml
+        run: |
+          cd backend
+          uv run ruff check . --config pyproject.toml

.github/workflows/security.yml

@@ -13,13 +13,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v4
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: '3.12'
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install bandit safety
+          enable-cache: true
+          cache-dependency-glob: "backend/uv.lock"
+
+      - name: Install bandit
+        run: uv tool install bandit
+
       - name: Run bandit
-        run: bandit -r backend/ -x backend/tests/ -ll
+        run: uv tool run bandit -r backend/ -x backend/tests/ -ll

.github/workflows/tests.yml

@@ -60,11 +60,19 @@ jobs:
         run: |
           echo "Pre-pulling base images to speed up builds..."
           docker pull python:3.12-slim &
+          docker pull ghcr.io/astral-sh/uv:0.9.17 &
           docker pull alpine:latest &
           docker pull confluentinc/cp-kafka:7.5.0 &
           docker pull confluentinc/cp-zookeeper:7.5.0 &
-          docker pull mongo:7 &
+          docker pull confluentinc/cp-schema-registry:7.5.0 &
+          docker pull mongo:8.0 &
           docker pull redis:7-alpine &
+          docker pull grafana/grafana:latest &
+          docker pull jaegertracing/all-in-one:1.52 &
+          docker pull victoriametrics/victoria-metrics:v1.96.0 &
+          docker pull otel/opentelemetry-collector-contrib:0.91.0 &
+          docker pull obsidiandynamics/kafdrop:3.31.0 &
+          docker pull danielqsj/kafka-exporter:latest &
           wait
           echo "Base images pulled successfully"
 
@@ -80,6 +88,8 @@ jobs:
           yq eval '.services.backend.environment += ["MONGO_ROOT_PASSWORD=rootpassword"]' -i docker-compose.ci.yaml
           # Disable OpenTelemetry SDK during tests to avoid exporter retries
           yq eval '.services.backend.environment += ["OTEL_SDK_DISABLED=true"]' -i docker-compose.ci.yaml
+          # Remove ./backend:/app volume mount - CI doesn't need hot-reload and it creates .venv permission conflicts
+          yq eval '.services.backend.volumes = [.services.backend.volumes[] | select(. != "./backend:/app")]' -i docker-compose.ci.yaml
 
           # MongoDB service already has defaults in docker-compose.yaml (root/rootpassword)
           # No need to override them
@@ -120,6 +130,8 @@ jobs:
             *.cache-from=type=gha,scope=buildkit-${{ github.repository }}-main
             *.cache-to=type=gha,mode=max,scope=buildkit-${{ github.repository }}-${{ github.ref_name }}
             *.pull=true
+        env:
+          BUILDKIT_PROGRESS: plain
 
       - name: Start services
         run: |
@@ -130,7 +142,7 @@ jobs:
               docker compose -f docker-compose.ci.yaml logs
               exit 1
             )
-            
+
             echo "Services started. Waiting for stabilization..."
             sleep 45
 
@@ -145,7 +157,7 @@ jobs:
 
             echo "Checking backend logs:"
             docker compose -f docker-compose.ci.yaml logs backend || echo "backend not found"
-            
+
             # Explicitly check for containers that have exited
             if docker compose -f docker-compose.ci.yaml ps | grep -q 'Exit'; then
               echo "::error::One or more containers have exited unexpectedly. See logs above."
@@ -171,18 +183,17 @@ jobs:
           kubectl get roles -n default
           kubectl get rolebindings -n default
 
-      - name: Set up Python for Tests
-        uses: actions/setup-python@v4
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: '3.12'
+          enable-cache: true
+          cache-dependency-glob: "backend/uv.lock"
 
-      - name: Install Python test dependencies
+      - name: Install Python and dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install -y python3 python3-pip
           cd backend
-          pip3 install -r requirements.txt
-          pip3 install -r requirements-dev.txt
+          uv python install 3.12
+          uv sync --frozen
 
       - name: Run backend tests with coverage
         timeout-minutes: 5
@@ -202,7 +213,7 @@ jobs:
           echo "Using BACKEND_BASE_URL=$BACKEND_BASE_URL"
           echo "Using SCHEMA_SUBJECT_PREFIX=$SCHEMA_SUBJECT_PREFIX"
           echo "MongoDB connection will use default CI credentials"
-          python -m pytest tests/integration tests/unit -v --cov=app --cov-branch --cov-report=xml --cov-report=term --cov-report=term-missing
+          uv run pytest tests/integration tests/unit -v --cov=app --cov-branch --cov-report=xml --cov-report=term --cov-report=term-missing
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5

backend/Dockerfile

@@ -1,17 +1,5 @@
-FROM python:3.12
-WORKDIR /app
-
-# Install required packages
-RUN apt-get update && apt-get upgrade -y liblzma-dev liblzma5 xz-utils && \
-    apt-get install -y libsnappy-dev && \
-    rm -rf /var/lib/apt/lists/*
-
-## Remove kubectl from runtime image: rely on Python client only
-
-# Install Python dependencies
-COPY requirements.txt .
-RUN pip install --no-cache-dir --upgrade pip setuptools>=70.0.0 wheel
-RUN pip install --no-cache-dir -r requirements.txt
+# Backend API service
+FROM base
 
 # Copy application files and configuration
 COPY ./app /app/app
@@ -33,7 +21,7 @@ CMD bash -c "\
   # Use kubeconfig if present, but do not block startup\
   if [ -f /app/kubeconfig.yaml ]; then export KUBECONFIG=/app/kubeconfig.yaml; fi && \
   WEB_CONCURRENCY=${WEB_CONCURRENCY:-4} WEB_THREADS=${WEB_THREADS:-1} WEB_TIMEOUT=${WEB_TIMEOUT:-60} \
-  gunicorn app.main:app \
+  uv run gunicorn app.main:app \
     -k uvicorn.workers.UvicornWorker \
     --bind 0.0.0.0:443 \
     --workers ${WEB_CONCURRENCY} \

backend/Dockerfile.base

@@ -0,0 +1,26 @@
+# Shared base image for all backend services
+# Contains: Python, system deps, uv, and all Python dependencies
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install all system dependencies needed by any service
+RUN apt-get update && apt-get install -y \
+    gcc \
+    curl \
+    libsnappy-dev \
+    liblzma-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.17 /uv /uvx /bin/
+
+# Copy dependency files
+COPY pyproject.toml uv.lock ./
+
+# Install Python dependencies (production only)
+# --no-install-project: don't install project itself, only dependencies
+RUN uv sync --frozen --no-dev --no-install-project
+
+# Set Python path so imports work
+ENV PYTHONPATH=/app

backend/Dockerfile.test

@@ -9,11 +9,14 @@ RUN apt-get update && apt-get install -y \
     curl \
     && rm -rf /var/lib/apt/lists/*
 
-# Copy requirements
-COPY requirements.txt requirements-dev.txt ./
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:0.9.17 /uv /uvx /bin/
 
-# Install Python dependencies
-RUN pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt
+# Copy dependency files
+COPY pyproject.toml uv.lock ./
+
+# Install Python dependencies (including dev deps for testing)
+RUN uv sync --frozen
 
 # Copy application code
 COPY . .
@@ -22,4 +25,4 @@ COPY . .
 ENV PYTHONPATH=/app
 
 # Default command runs all tests
-CMD ["pytest", "-v", "--tb=short"]
+CMD ["uv", "run", "pytest", "-v", "--tb=short"]

backend/app/infrastructure/kafka/events/base.py

@@ -2,7 +2,7 @@
 from typing import Any, ClassVar
 from uuid import uuid4
 
-from pydantic import ConfigDict, Field
+from pydantic import ConfigDict, Field, field_serializer
 from pydantic_avro import AvroBase
 
 from app.domain.enums.events import EventType
@@ -22,11 +22,11 @@ class BaseEvent(AvroBase):
     # Each subclass must define its topic
     topic: ClassVar[KafkaTopic]
 
-    model_config = ConfigDict(
-        json_encoders={
-            datetime: lambda v: v.isoformat()
-        }
-    )
+    model_config = ConfigDict()
+
+    @field_serializer('timestamp', when_used='json')
+    def serialize_timestamp(self, dt: datetime) -> str:
+        return dt.isoformat()
 
     def to_dict(self) -> dict[str, Any]:
         # Use mode='json' to properly serialize datetime objects to ISO strings