diff --git a/.github/scripts/upgrade-lib.sh b/.github/scripts/upgrade-lib.sh new file mode 100644 index 0000000000..b7546afd57 --- /dev/null +++ b/.github/scripts/upgrade-lib.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Helpers for the upgrade-path job in build.yml. Sourced, not executed: the two +# steps that boot a container run in separate shells and both need `boot`. +# +# Expects VOLUME_DB, VOLUME_DATA and BOOT_TIMEOUT from the job env. + +# boot — start the standalone image on the shared +# volumes and block until /healthcheck answers true. Fails fast if the +# container exits instead of waiting out the timeout on a crash loop. +boot() { + image="$1" + name="$2" + + echo "Booting ${name} from ${image}" + docker run -d --name "$name" \ + -v "${VOLUME_DB}:/var/lib/postgresql" \ + -v "${VOLUME_DATA}:/var/www/euro-office/Data" \ + "$image" >/dev/null + + deadline=$(( $(date +%s) + BOOT_TIMEOUT )) + until [ "$(docker exec "$name" curl -sf http://localhost/healthcheck 2>/dev/null)" = "true" ]; do + if [ "$(docker inspect -f '{{.State.Running}}' "$name")" != "true" ]; then + echo "::error::${name} exited with code $(docker inspect -f '{{.State.ExitCode}}' "$name")" + docker logs "$name" 2>&1 | tail -40 + return 1 + fi + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "::error::${name} did not report healthy within ${BOOT_TIMEOUT}s" + docker logs "$name" 2>&1 | tail -40 + return 1 + fi + sleep 5 + done + + echo "${name} is healthy" +} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 18c75474d9..d0625dc6ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -510,3 +510,111 @@ jobs: name: playwright-report-${{ github.sha }} path: e2e/playwright-report/ retention-days: 14 + + # Upgrade path: boot the last released image against persistent volumes, then + # boot the image built here on those same volumes. The e2e job only ever sees + # empty volumes, which is why #314 shipped — a shifted `postgres` UID left + # pre-existing Postgres data owned by an account that no longer matched, and + # every fresh-volume boot stayed green. + # + # Skipped on tag pushes: the manifest job has already moved :latest to the + # image under test by then, so there would be nothing to upgrade from. PR and + # main builds are where :latest is genuinely the previous release. + upgrade: + runs-on: self-hosted + needs: [build, manifest] + if: | + !cancelled() && + needs.build.result == 'success' && + !startsWith(github.ref, 'refs/tags/') && + (github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository) && + (github.event_name == 'pull_request' || needs.manifest.result == 'success') + permissions: + contents: read + packages: read + + env: + FROM_IMAGE: ghcr.io/euro-office/documentserver:latest + CONTAINER: eo-upgrade-${{ github.run_id }} + VOLUME_DB: eo-upgrade-db-${{ github.run_id }} + VOLUME_DATA: eo-upgrade-data-${{ github.run_id }} + BOOT_TIMEOUT: 300 + + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Determine image + id: image + run: | + if [[ "${{ github.event_name }}" == "pull_request" ]]; then + echo "ref=${{ env.REGISTRY }}/${{ env.CACHE_IMAGE_NAME }}:pr-${{ github.event.pull_request.number }}" >> $GITHUB_OUTPUT + else + echo "ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:nightly" >> $GITHUB_OUTPUT + fi + + - name: Log in to GitHub Container Registry + uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 # v4.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Pull images + id: pull + env: + IMAGE_REF: ${{ steps.image.outputs.ref }} + run: | + docker pull "$IMAGE_REF" + # No published release to upgrade from (new brand, pruned tag): report + # it and skip rather than failing a build this cannot say anything about. + if docker pull "$FROM_IMAGE"; then + echo "have_from=true" >> $GITHUB_OUTPUT + else + echo "have_from=false" >> $GITHUB_OUTPUT + echo "::notice::$FROM_IMAGE not available - skipping the upgrade check" + fi + + - name: Boot previous release and seed the database + if: steps.pull.outputs.have_from == 'true' + run: | + set -euo pipefail + . .github/scripts/upgrade-lib.sh + + docker volume create "$VOLUME_DB" >/dev/null + docker volume create "$VOLUME_DATA" >/dev/null + boot "$FROM_IMAGE" "$CONTAINER-old" + + # A row of our own, so the check after the upgrade proves the same + # datadir came back rather than a silently reinitialised empty one. + docker exec "$CONTAINER-old" sudo -u postgres psql -q -d eurooffice \ + -c 'CREATE TABLE upgrade_probe (id integer)' \ + -c 'INSERT INTO upgrade_probe VALUES (314)' + docker rm -f "$CONTAINER-old" >/dev/null + + - name: Boot the built image on the same volumes + if: steps.pull.outputs.have_from == 'true' + env: + IMAGE_REF: ${{ steps.image.outputs.ref }} + run: | + set -euo pipefail + . .github/scripts/upgrade-lib.sh + + boot "$IMAGE_REF" "$CONTAINER-new" + + probe=$(docker exec "$CONTAINER-new" sudo -u postgres psql -tAq -d eurooffice \ + -c 'SELECT id FROM upgrade_probe' | tr -d '[:space:]') + if [ "$probe" != "314" ]; then + echo "::error::Postgres data from the previous release did not survive the upgrade (probe='$probe')" + exit 1 + fi + echo "Upgrade check passed: pre-existing Postgres volume in use after the upgrade." + + - name: Clean up + if: always() + run: | + docker rm -f "$CONTAINER-old" "$CONTAINER-new" >/dev/null 2>&1 || true + docker volume rm -f "$VOLUME_DB" "$VOLUME_DATA" >/dev/null 2>&1 || true diff --git a/build/.docker/standalone.bake.Dockerfile b/build/.docker/standalone.bake.Dockerfile index 91404235fe..6b91ac173a 100644 --- a/build/.docker/standalone.bake.Dockerfile +++ b/build/.docker/standalone.bake.Dockerfile @@ -29,6 +29,24 @@ ENV EO_CONF=${EO_CONF} ENV COMPANY_NAME_LOW=${COMPANY_NAME_LOW} ENV PRODUCT_NAME_LOW=${PRODUCT_NAME_LOW} +# Pin the UIDs/GIDs of every account that ends up owning persisted state. +# +# Debian allocates system UIDs in package-configuration order, which makes them +# a side effect of the package set: adding --no-install-recommends below for +# v9.3.3 dropped dbus's `messagebus` account from this layer and slid postgres +# from 103 to 102, so Postgres volumes written by v9.3.2 suddenly had a datadir +# owned by what is now `rabbitmq` and the server refused to start (#314). +# +# These are the values v9.3.3 shipped, so pinning them changes nothing for +# existing volumes; entrypoint.sh repairs the ones written before that. The +# adduser/useradd calls in the package postinst scripts (and in the .deb's own +# postinst, for `ds`) are no-ops when the account already exists, so the +# packages adopt these accounts as they are. +RUN groupadd -r -g 103 redis && useradd -r -u 101 -g redis -d /var/lib/redis -s /usr/sbin/nologin redis && \ + groupadd -r -g 104 postgres && useradd -r -u 102 -g postgres -d /var/lib/postgresql -s /bin/bash postgres && \ + groupadd -r -g 105 rabbitmq && useradd -r -u 103 -g rabbitmq -d /var/lib/rabbitmq -s /usr/sbin/nologin rabbitmq && \ + groupadd -r -g 107 ds && useradd -r -u 105 -g ds -d ${EO_ROOT} -s /usr/sbin/nologin ds + RUN apt-get update && \ ACCEPT_EULA=Y apt-get install -yq --no-install-recommends \ postgresql postgresql-client redis-server rabbitmq-server \ diff --git a/build/scripts/standalone/entrypoint.sh b/build/scripts/standalone/entrypoint.sh index 03314ed633..15bf04695a 100644 --- a/build/scripts/standalone/entrypoint.sh +++ b/build/scripts/standalone/entrypoint.sh @@ -452,6 +452,49 @@ if [ ! -f "$API_TPL" ] && [ -f "${EO_ROOT}/web-apps/apps/api/documents/api.js" ] cp "${EO_ROOT}/web-apps/apps/api/documents/api.js" "$API_TPL" fi +# -------------------------------------------------------------------- +# Repair ownership of persisted state before the services touch it. +# +# The service accounts used to get their UIDs allocated dynamically by the +# package installs, which made them a function of package order: dropping +# Recommends in v9.3.3 removed dbus's `messagebus` account from that layer +# and slid postgres from 103 to 102. A Postgres volume written by v9.3.2 +# then had its datadir owned by a UID that resolves to `rabbitmq` in the +# new image, postgresql-common refused to start on the mismatch, the +# entrypoint died under `set -e` before nginx, and a restart policy turned +# that into a loop reporting nothing but the ownership line (#314). +# +# The UIDs are pinned in the Dockerfile now, so this cannot recur, but +# volumes written before that still carry the old ones and a bind mount can +# arrive owned by anything. Chown only what is actually mismatched, since +# the data directories can be large. +# -------------------------------------------------------------------- +ensure_owner() { + owner="$1" + shift + uid="$(id -u "$owner" 2>/dev/null)" || return 0 + gid="$(id -g "$owner" 2>/dev/null)" || return 0 + + for dir in "$@"; do + [ -d "$dir" ] || continue + current="$(stat -c '%u:%g' "$dir" 2>/dev/null)" || continue + [ "$current" = "${uid}:${gid}" ] && continue + + echo "Repairing ownership of ${dir}: ${current} -> ${uid}:${gid} (${owner})" + chown -R "${uid}:${gid}" "$dir" \ + || echo "WARNING: could not chown ${dir}; ${owner} may fail to start." >&2 + done +} + +[ "$DB_HOST" = "localhost" ] && ensure_owner postgres /var/lib/postgresql +[ "$AMQP_HOST" = "localhost" ] && [ -z "${AMQP_URI:-}" ] && ensure_owner rabbitmq /var/lib/rabbitmq +ensure_owner ds "$DATA_DIR" "$EO_LOG" + +# $DATA_DIR holds the persisted secrets and the repair above is recursive, so +# put the owner back: on a fresh install $PRIVATE_DIR is root-owned (mode 700, +# ds group inherited from $DATA_DIR) and only this script reads it. +[ -d "$PRIVATE_DIR" ] && chown -R root "$PRIVATE_DIR" + # -------------------------------------------------------------------- # Start bundled services only when the corresponding host points at # localhost. When an external host is configured, we already waited