From 702c9a9ba9d91a1761ede702bd5c78ad9b7f064f Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 19 Nov 2025 16:57:32 -0500 Subject: [PATCH 1/5] Reorganize CI jobs so they appear sensibly in the GitHub UI Signed-off-by: Adrian Edwards --- .github/workflows/checks.yml | 13 +-- .github/workflows/docs.yml | 16 ++++ .github/workflows/macos.yml | 51 ++++++++++++ .github/workflows/publish_images.yml | 79 +++++++++++++++++++ .../{build_docker.yml => smoke_test.yml} | 39 +-------- 5 files changed, 148 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 .github/workflows/macos.yml create mode 100644 .github/workflows/publish_images.yml rename .github/workflows/{build_docker.yml => smoke_test.yml} (92%) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8b2606fbf0..9fa5cec1b1 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,20 +1,9 @@ -name: "run-linting-checks" +name: "Lint and Style Checks" on: pull_request: branches: [main, release] jobs: - check-docs: - name: runner / check docs - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v6 - - name: Ensure docs build cleanly - # Setting `O` to pass extra options to the sphinx-build command. - run: O="-a -E -n -W --keep-going" make docs - run-pylint: name: runner / pylint permissions: write-all diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000000..7eef9ef68b --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,16 @@ +name: "Docs build test" +on: + pull_request: + branches: [main, release] + +jobs: + check-docs: + name: runner / check docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install uv + uses: astral-sh/setup-uv@v6 + - name: Ensure docs build cleanly + # Setting `O` to pass extra options to the sphinx-build command. + run: O="-a -E -n -W --keep-going" make docs diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 0000000000..eb079c64c7 --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,51 @@ +name: Test MacOS +on: + push: + branches: + - main + - release + pull_request: + release: + types: + - published + workflow_dispatch: + +permissions: {} + +jobs: + test-macos: + name: Test on macOS + runs-on: macos-latest + env: + UV_LOCKED: true # Assert that uv.lock is up-to-date + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: ".python-version" + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: "stable" + + # We don't use `make install` because it requires user input + # Instead, we manually sync and run a subset of commands + - name: Install dependencies + run: uv sync --all-groups + + - name: Install workers + run: uv run scripts/install/workers.sh dev + + - name: Install nltk + run: | + uv run python -m nltk.downloader stopwords + uv run python -m nltk.downloader punkt + uv run python -m nltk.downloader popular + uv run python -m nltk.downloader universal_tagset diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml new file mode 100644 index 0000000000..fce4eaeae0 --- /dev/null +++ b/.github/workflows/publish_images.yml @@ -0,0 +1,79 @@ +name: Publish Docker Images +on: + push: + branches: + - main + - release + pull_request: + release: + types: + - published + workflow_dispatch: + +permissions: {} + +jobs: + push-image: + name: Push image + needs: test-e2e + # We don't push images on pull requests + if: github.event_name != 'pull_request' + permissions: + contents: read # to fetch code (actions/checkout) + packages: write # to push docker image + strategy: + matrix: + image: + - backend + - database + - keyman + - rabbitmq + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + id: setup-buildx + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set container metadata + uses: docker/metadata-action@v5 + id: meta + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest + with: + annotations: | + org.opencontainers.image.title=augur_${{ matrix.image}} + labels: | + org.opencontainers.image.title=augur_${{ matrix.image}} + images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }} + # Pushes to the main branch update the *:devel-latest tag + # Releases update the *:latest tag and the *: tag + tags: | + type=raw,value=devel-latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=latest,enable=${{ github.event_name == 'release' }} + type=raw,value=${{ github.event.release.tag_name }},enable=${{ github.event_name == 'release' }} + + - name: Build and push + id: push + uses: docker/build-push-action@v6 + with: + annotations: ${{ steps.meta.outputs.annotations }} + context: . + file: ./docker/${{ matrix.image }}/Dockerfile + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64 + # Only push if we've tagged the image in the metadata step + push: ${{ steps.meta.outputs.tags != '' }} + tags: ${{ steps.meta.outputs.tags }} + # Use the same cache as the build step + cache-from: type=gha,scope=container-${{ matrix.image }} + cache-to: type=gha,scope=container-${{ matrix.image }},mode=min diff --git a/.github/workflows/build_docker.yml b/.github/workflows/smoke_test.yml similarity index 92% rename from .github/workflows/build_docker.yml rename to .github/workflows/smoke_test.yml index 3a0e3f953a..e6b6f18787 100644 --- a/.github/workflows/build_docker.yml +++ b/.github/workflows/smoke_test.yml @@ -1,4 +1,4 @@ -name: Build Docker images +name: End to end Smoke Tests on: push: branches: @@ -13,43 +13,6 @@ on: permissions: {} jobs: - test-macos: - name: Test on macOS - runs-on: macos-latest - env: - UV_LOCKED: true # Assert that uv.lock is up-to-date - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install uv - uses: astral-sh/setup-uv@v6 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version-file: ".python-version" - - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version: "stable" - - # We don't use `make install` because it requires user input - # Instead, we manually sync and run a subset of commands - - name: Install dependencies - run: uv sync --all-groups - - - name: Install workers - run: uv run scripts/install/workers.sh dev - - - name: Install nltk - run: | - uv run python -m nltk.downloader stopwords - uv run python -m nltk.downloader punkt - uv run python -m nltk.downloader popular - uv run python -m nltk.downloader universal_tagset - test-e2e: name: End-to-end test (Docker) From 51db5ae20533af828b9b846a8b3e6c6aee7ddef4 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 19 Nov 2025 17:08:43 -0500 Subject: [PATCH 2/5] docs and comments Signed-off-by: Adrian Edwards --- .github/workflows/checks.yml | 1 + .github/workflows/docs.yml | 1 + .github/workflows/macos.yml | 3 ++- .github/workflows/publish_images.yml | 1 + .github/workflows/smoke_test.yml | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 9fa5cec1b1..2b400fdc9a 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,4 +1,5 @@ name: "Lint and Style Checks" +# Checks that automate detection of basic errors (type declaration, spelling, linting, forgetting to update the lockfile, etc) on: pull_request: branches: [main, release] diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7eef9ef68b..7ec64a8da4 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,4 +1,5 @@ name: "Docs build test" +# ensure the docs build still works on: pull_request: branches: [main, release] diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index eb079c64c7..3cf88119d8 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,4 +1,5 @@ name: Test MacOS +# Unknown. MacOS functionality test i guess, probably incomplete on: push: branches: @@ -17,7 +18,7 @@ jobs: name: Test on macOS runs-on: macos-latest env: - UV_LOCKED: true # Assert that uv.lock is up-to-date + UV_LOCKED: true # Assert that uv.lock is up-to-date steps: - name: Checkout repository uses: actions/checkout@v4 diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index fce4eaeae0..f96b96247f 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -1,4 +1,5 @@ name: Publish Docker Images +# Publish docker images to our registry on: push: branches: diff --git a/.github/workflows/smoke_test.yml b/.github/workflows/smoke_test.yml index e6b6f18787..7b9c8c858c 100644 --- a/.github/workflows/smoke_test.yml +++ b/.github/workflows/smoke_test.yml @@ -1,4 +1,5 @@ name: End to end Smoke Tests +# perform an end to end smoke test on docker and podman to make sure things work as expected on: push: branches: From cd55b78bb5ac72dfd4fd03b74d8a3a5c8d3d402a Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 19 Nov 2025 17:08:54 -0500 Subject: [PATCH 3/5] add mypy check for type checking Signed-off-by: Adrian Edwards --- .github/workflows/checks.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2b400fdc9a..bcfdbe974c 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -41,3 +41,15 @@ jobs: uses: astral-sh/setup-uv@v6 - name: Ensure uv lockfile is up to date run: uv lock --check + + run-mypy: + name: runner / mypy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: tsuyoshicho/action-mypy@v4 + with: + github_token: ${{ secrets.github_token }} + reporter: github-pr-review + level: warning + execute_command: "uv run mypy" From fc1a835263aaaea9712a328b48a8e4811c2adfb6 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Wed, 19 Nov 2025 17:17:27 -0500 Subject: [PATCH 4/5] restrict github token permissions per CI suggestion Signed-off-by: Adrian Edwards --- .github/workflows/checks.yml | 3 +++ .github/workflows/docs.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bcfdbe974c..823160b877 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -45,6 +45,9 @@ jobs: run-mypy: name: runner / mypy runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write steps: - uses: actions/checkout@v4 - uses: tsuyoshicho/action-mypy@v4 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 7ec64a8da4..fd12582032 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,5 +1,8 @@ name: "Docs build test" # ensure the docs build still works +permissions: + contents: read + on: pull_request: branches: [main, release] From 346a46d7f26aab05645880740076252c3e7c81bb Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Thu, 20 Nov 2025 13:30:52 -0500 Subject: [PATCH 5/5] Copilot-suggested fixes Signed-off-by: Adrian Edwards --- .github/workflows/docs.yml | 2 +- .github/workflows/publish_images.yml | 4 ++-- .github/workflows/smoke_test.yml | 9 +++------ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index fd12582032..efc5919faf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -12,7 +12,7 @@ jobs: name: runner / check docs runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install uv uses: astral-sh/setup-uv@v6 - name: Ensure docs build cleanly diff --git a/.github/workflows/publish_images.yml b/.github/workflows/publish_images.yml index f96b96247f..86a3a00133 100644 --- a/.github/workflows/publish_images.yml +++ b/.github/workflows/publish_images.yml @@ -52,9 +52,9 @@ jobs: DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest with: annotations: | - org.opencontainers.image.title=augur_${{ matrix.image}} + org.opencontainers.image.title=augur_${{ matrix.image }} labels: | - org.opencontainers.image.title=augur_${{ matrix.image}} + org.opencontainers.image.title=augur_${{ matrix.image }} images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }} # Pushes to the main branch update the *:devel-latest tag # Releases update the *:latest tag and the *: tag diff --git a/.github/workflows/smoke_test.yml b/.github/workflows/smoke_test.yml index 7b9c8c858c..4d81a4fdce 100644 --- a/.github/workflows/smoke_test.yml +++ b/.github/workflows/smoke_test.yml @@ -14,7 +14,6 @@ on: permissions: {} jobs: - test-e2e: name: End-to-end test (Docker) runs-on: ubuntu-latest @@ -23,7 +22,7 @@ jobs: run: | sudo rm -rf /usr/share/dotnet sudo rm -rf "$AGENT_TOOLSDIRECTORY" - + - name: Checkout repository uses: actions/checkout@v4 @@ -263,8 +262,6 @@ jobs: # We use tail so that we can see the name of each file as it's printed run: "podman run -t --rm -v augur_logs:/logs bash -c 'find /logs -type f | xargs tail -n +0'" - - push-image: name: Push image needs: test-e2e @@ -303,9 +300,9 @@ jobs: DOCKER_METADATA_ANNOTATIONS_LEVELS: index,manifest with: annotations: | - org.opencontainers.image.title=augur_${{ matrix.image}} + org.opencontainers.image.title=augur_${{ matrix.image }} labels: | - org.opencontainers.image.title=augur_${{ matrix.image}} + org.opencontainers.image.title=augur_${{ matrix.image }} images: ghcr.io/${{ github.repository_owner }}/augur_${{ matrix.image }} # Pushes to the main branch update the *:devel-latest tag # Releases update the *:latest tag and the *: tag