Migrate Quarry to use Poetry for dependency management.

.github/workflows/build-and-push.yaml

@@ -10,13 +10,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-
-      - name: git fetch
-        run: |
-          git fetch
-      - name: git checkout
-        run: |
-          git checkout ${{ github.head_ref }}
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.head_repo.full_name }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/tests.yaml

@@ -9,7 +9,10 @@ jobs:
   tox:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.head_repo.full_name }}
 
       - name: build tox container
         run: |
@@ -21,6 +24,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+          repository: ${{ github.head_repo.full_name }}
       - name: Install dependencies
         run: npm ci
       - name: Run tests

Dockerfile

@@ -8,13 +8,20 @@ RUN useradd -r -m quarry && \
 
 WORKDIR /app
 
-COPY requirements.txt /app
-# Install dependencies
-# TODO: Use a venv instead of --break-system-packages
-# TODO: Use newer pip. That requires newer celery, which in turn requires
-# newer versions of basically everything else.
+# 1. Update pip, install Poetry, and set venv path
 RUN pip install --break-system-packages --upgrade pip==24.0 wheel && \
-    pip install --break-system-packages -r requirements.txt
+    pip install --break-system-packages poetry
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+ENV PATH="/app/.venv/bin:$PATH"
+
+# 2. Copy dependency files
+COPY pyproject.toml poetry.lock /app/
+
+# 3. Install dependencies via Poetry (no --break-system-packages needed)
+RUN poetry install --no-root --only main --no-interaction
+
+# 4. Ensure quarry user can access the virtual environment
+RUN chown -R quarry:quarry /app/.venv
 
 # Copy app code
 USER quarry

README.md

@@ -60,6 +60,28 @@ If you had already run a dev environment (that is, ran `docker-compose up`) you
 the containers with the new dependencies by running `docker-compose build` before running
 `docker-compose up` again.
 
+### Direct Local Development (Non-Containerized) ###
+
+If you prefer to run the Flask application directly on your machine without Docker, you must first set up the Python environment using Poetry.
+
+**Install Poetry:**
+```bash
+pip install poetry
+```
+
+**Install Dependencies:** Run this command from the project root. It will create a local virtual environment and install all required packages.
+```bash
+poetry install
+```
+
+**Activate Shell and Run:** Run your application (e.g., for testing or running locally):
+```bash
+poetry shell
+flask run
+```
+
+**Note:** You may need to run local Redis and database services separately to avoid ConnectionErrors.
+
 ## Useful commands ##
 
 To pre-compile nunjucks templates:
@@ -168,4 +190,4 @@ If ansible doesn't detect a change for quarry helm the following can be run:
 `helm -n quarry upgrade --install quarry helm-quarry -f helm-quarry/prod-env.yaml`
 
 For shell access, a debug pod can be created on a running node with something lie
-$ kubectl debug node/quarry-127a-g4ndvpkr5sro-node-0 -it --image debian:stable  
+$ kubectl debug node/quarry-127a-g4ndvpkr5sro-node-0 -it --image debian:stable

docker-compose.yml

@@ -6,7 +6,7 @@ services:
     build: .
     # To mimic a production runtime, uncomment:
     # entrypoint: ["gunicorn", "-w", "2", "--bind", "0.0.0.0:5000", "wsgi:application"]
-    entrypoint: ["python3", "wsgi.py"]
+    entrypoint: ["poetry", "run", "python3", "wsgi.py"]
     volumes:
       - .:/app
       - results:/results
@@ -22,7 +22,7 @@ services:
     volumes:
       - .:/app
       - results:/results
-    entrypoint: ["celery", "--app", "quarry.web.worker", "worker", ]
+    entrypoint: ["poetry", "run", "celery", "--app", "quarry.web.worker", "worker", ]
     extra_hosts:
     - "host.docker.internal:host-gateway"
     depends_on: