ci_integration.yml

---
# Full CI for commits to the integration branches (main, develop, next)
name: "CI - Integration Branch"
on:  # yamllint disable
  push:
    branches:
      - "main"
      - "develop"
      - "next"
  workflow_call:
  workflow_dispatch:

jobs:
  security-tooling:
    name: "Security Tooling"
    runs-on: "ubuntu-24.04"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Security checks"
        uses: "./.github/actions/security-checks"
  check-schema:
    name: "Run REST API schema checks"
    runs-on: "ubuntu-24.04"
    env:
      INVOKE_NAUTOBOT_LOCAL: "True"
      NAUTOBOT_DB_USER: "nautobot"
      NAUTOBOT_DB_PASSWORD: "decinablesprewad"
      NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
      NAUTOBOT_CONFIG: "nautobot/core/tests/nautobot_config.py"
    services:
      postgres:
        image: "postgres:14-alpine"
        env:
          POSTGRES_USER: "nautobot"
          POSTGRES_PASSWORD: "decinablesprewad"
          POSTGRES_DB: "nautobot"
        ports:
          - "5432:5432"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Setup environment"
        uses: "networktocode/gh-action-setup-poetry-environment@v6"
        with:
          poetry-install-options: ""  # override default "--only dev"
          poetry-version: "2.1.4"
          python-version: "3.14"
      - name: "Run check-schema"
        # REST API schema generation currently includes GraphQL schema generation, which is dependent on the DB :-(
        run: "poetry run invoke migrate check-schema"
  ui-build-check:
    name: "Check UI src and dist files validity and integrity"
    runs-on: "ubuntu-24.04"
    env:
      INVOKE_NAUTOBOT_LOCAL: "True"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Setup environment"
        uses: "networktocode/gh-action-setup-poetry-environment@v6"
        with:
          poetry-install-options: "--only dev --only linting"
          poetry-version: "2.1.4"
          python-version: "3.14"
      - name: "Setup Node.js"
        uses: "actions/setup-node@v4"
        with:
          node-version: "24"
      - name: "Install npm dependencies"
        run: "poetry run invoke npm --command ci"
      - name: "Calculate checksum of originally committed UI dist files"
        shell: bash
        run: |
          original_checksum=$(find nautobot/project-static/dist -type f -exec md5sum {} + | sort -k 2 | md5sum)
          echo "original_checksum=$original_checksum" >> "$GITHUB_ENV"
      - name: "Build UI from source"
        run: "poetry run invoke ui-build"
      - name: "Re-calculate checksum of UI dist files after the build"
        shell: bash
        run: |
          updated_checksum=$(find nautobot/project-static/dist -type f -exec md5sum {} + | sort -k 2 | md5sum)
          echo "updated_checksum=$updated_checksum" >> "$GITHUB_ENV"
      - name: "Compare calculated checksums"
        run: |
          echo "Comparing UI dist files checksums: \"$original_checksum\" and \"$updated_checksum\"."
          if [ "$original_checksum" = "$updated_checksum" ]; then
            echo "Checksums match."
          else
            >&2 echo "ERROR: Checksums do not match. Please verify build."
            exit 1
          fi
  tests-postgres:
    # Sticking with Python 3.13 rather than 3.14 here for now, because Django 5.2's test runner doesn't
    # support parallel execution under 3.14's default multiprocessing start method. Relevant references:
    # https://docs.python.org/3/library/multiprocessing.html#multiprocessing-start-methods
    # https://code.djangoproject.com/ticket/36531
    name: "Unit tests (PostgreSQL and Python 3.13)"
    runs-on: "ubuntu-24.04"
    env:
      INVOKE_NAUTOBOT_LOCAL: "True"
      NAUTOBOT_DB_USER: "nautobot"
      NAUTOBOT_DB_PASSWORD: "decinablesprewad"
      NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
    services:
      postgres:
        image: "postgres:14-alpine"
        env:
          POSTGRES_USER: "nautobot"
          POSTGRES_PASSWORD: "decinablesprewad"
          POSTGRES_DB: "nautobot"
        ports:
          - "5432:5432"
      redis:
        image: "redis:6-alpine"
        ports:
          - "6379:6379"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Setup environment"
        uses: "networktocode/gh-action-setup-poetry-environment@v6"
        with:
          poetry-install-options: "--extras=remote_storage"
          poetry-version: "2.1.4"
          # Sticking with Python 3.13 rather than 3.14 here for now, because Django 5.2's test runner doesn't
          # support parallel execution under 3.14's default multiprocessing start method. Relevant references:
          # https://docs.python.org/3/library/multiprocessing.html#multiprocessing-start-methods
          # https://code.djangoproject.com/ticket/36531
          python-version: "3.13"
      - name: "Run unittest"
        run: "poetry run invoke tests --failfast --no-keepdb --no-cache-test-fixtures --parallel"
  tests-mysql:
    name: "Unit tests (MySQL and Python 3.10)"
    runs-on: "ubuntu-24.04"
    env:
      INVOKE_NAUTOBOT_LOCAL: "True"
      NAUTOBOT_DB_USER: "root"
      NAUTOBOT_DB_PASSWORD: "decinablesprewad"
      NAUTOBOT_DB_HOST: "127.0.0.1"
      NAUTOBOT_DB_PORT: "3306"
      NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.mysql"
    services:
      mysql:
        image: "mysql:8"
        env:
          MYSQL_ROOT_PASSWORD: "decinablesprewad"
          MYSQL_PASSWORD: "decinablesprewad"
          MYSQL_DATABASE: "nautobot"
        ports:
          - "3306:3306"
      redis:
        image: "redis:6-alpine"
        ports:
          - "6379:6379"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Install Required Dependencies for MySQL Client"
        run: "sudo apt-get update && sudo apt-get install --no-install-recommends -y default-libmysqlclient-dev gcc"
      - name: "Setup environment"
        uses: "networktocode/gh-action-setup-poetry-environment@v6"
        with:
          poetry-install-options: "--extras=mysql --extras=remote_storage"
          poetry-version: "2.1.4"
          python-version: "3.10"
      - name: "Run unittest"
        run: "poetry run invoke tests --failfast --no-keepdb --no-cache-test-fixtures --parallel"
  migration-tests-mysql:
    name: "Run migrations against test dataset (MySQL and Python 3.10)"
    runs-on: "ubuntu-24.04"
    env:
      MIGRATION_TEST_DATASET: "development/datasets/nautobot_1.5.16_mysql.tar.gz"
      INVOKE_NAUTOBOT_LOCAL: "True"
      NAUTOBOT_CONFIG: "nautobot/core/tests/nautobot_config.py"
      NAUTOBOT_DB_USER: "root"
      NAUTOBOT_DB_PASSWORD: "decinablesprewad"
      NAUTOBOT_DB_HOST: "127.0.0.1"
      NAUTOBOT_DB_PORT: "3306"
      NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.mysql"
    services:
      mysql:
        image: "mysql:8"
        env:
          MYSQL_ROOT_PASSWORD: "decinablesprewad"
          MYSQL_PASSWORD: "decinablesprewad"
          MYSQL_DATABASE: "nautobot"
        ports:
          - "3306:3306"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Install Required Dependencies for MySQL Client"
        run: "sudo apt-get update && sudo apt-get install --no-install-recommends -y default-libmysqlclient-dev gcc"
      - name: "Setup environment"
        uses: "networktocode/gh-action-setup-poetry-environment@v6"
        with:
          poetry-install-options: "--extras mysql"
          poetry-version: "2.1.4"
          python-version: "3.10"
      - name: "Run migration test"
        run: "poetry run invoke migration-test --db-engine mysql --dataset $MIGRATION_TEST_DATASET"
  migration-tests-postgres:
    name: "Run migrations against test dataset (PostgreSQL and Python 3.14)"
    runs-on: "ubuntu-24.04"
    env:
      MIGRATION_TEST_DATASET: "development/datasets/nautobot_1.5.16_postgres.tar.gz"
      INVOKE_NAUTOBOT_LOCAL: "True"
      NAUTOBOT_CONFIG: "nautobot/core/tests/nautobot_config.py"
      NAUTOBOT_DB_USER: "nautobot"
      NAUTOBOT_DB_PASSWORD: "decinablesprewad"
      NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
      PGPASSWORD: "decinablesprewad"
    services:
      postgres:
        image: "postgres:14-alpine"
        env:
          POSTGRES_USER: "nautobot"
          POSTGRES_PASSWORD: "decinablesprewad"
          POSTGRES_DB: "nautobot"
        ports:
          - "5432:5432"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Setup environment"
        uses: "networktocode/gh-action-setup-poetry-environment@v6"
        with:
          poetry-install-options: ""  # override default "--only dev"
          poetry-version: "2.1.4"
          python-version: "3.14"
      - name: "Run migration test"
        run: "poetry run invoke migration-test --db-engine postgres --dataset $MIGRATION_TEST_DATASET"
  integration-test:
    runs-on: "ubuntu-24.04"
    env:
      INVOKE_NAUTOBOT_LOCAL: "True"
      NAUTOBOT_DB_USER: "nautobot"
      NAUTOBOT_DB_PASSWORD: "decinablesprewad"
      NAUTOBOT_DB_ENGINE: "django_prometheus.db.backends.postgresql"
      NAUTOBOT_SELENIUM_URL: "http://localhost:4444/wd/hub"
    services:
      postgres:
        image: "postgres:14-alpine"
        env:
          POSTGRES_USER: "nautobot"
          POSTGRES_PASSWORD: "decinablesprewad"
          POSTGRES_DB: "nautobot"
        ports:
          - "5432:5432"
      redis:
        image: "redis:6-alpine"
        ports:
          - "6379:6379"
      selenium:
        image: "selenium/standalone-firefox:4.27"
        ports:
          - "4444:4444"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Setup environment"
        uses: "networktocode/gh-action-setup-poetry-environment@v6"
        with:
          poetry-install-options: ""  # override default "--only dev"
          poetry-version: "2.1.4"
          # Sticking with Python 3.13 rather than 3.14 here for now, because Django 5.2's test runner doesn't
          # support parallel execution under 3.14's default multiprocessing start method. Relevant references:
          # https://docs.python.org/3/library/multiprocessing.html#multiprocessing-start-methods
          # https://code.djangoproject.com/ticket/36531
          python-version: "3.13"
      - name: "Run Integration Tests"
        # If NAUTOBOT_SELENIUM_HOST is set to 'localhost' or '127.0.0.1' the connection does not work
        run: "NAUTOBOT_SELENIUM_HOST=`hostname -f` poetry run invoke tests --tag integration --no-keepdb"
  container-build:
    name: "Build Container Images (GHCR Only)"
    if: |
      github.event_name == 'push' &&
      (github.ref_name == 'develop' || github.ref_name == 'next' || github.ref_name == 'ltm-1.6')
    needs:
      - "check-schema"
      - "integration-test"
      - "migration-tests-mysql"
      - "migration-tests-postgres"
      - "tests-mysql"
      - "tests-postgres"
      - "ui-build-check"
    strategy:
      fail-fast: true
      matrix:
        python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ]
        arch: [ "amd64", "arm64" ]
        os: [ "ubuntu-24.04", "ubuntu-24.04-arm" ]
        exclude:
          - os: "ubuntu-24.04"
            arch: "arm64"
          - os: "ubuntu-24.04-arm"
            arch: "amd64"
    runs-on: "${{ matrix.os }}"
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Set up Docker Buildx"
        uses: "docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd" # v4
      - name: "Login to GitHub Container Registry"
        uses: "docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc" # v2
        with:
          registry: "ghcr.io"
          username: "${{ github.actor }}"
          password: "${{ secrets.GITHUB_TOKEN }}"

      - name: "Determine if nautobot-version is branch name or from tag name"
        id: "versionname"
        run: |
          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
            echo "nautobot-version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
          else
            echo "nautobot-version=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
          fi

      - name: "Build Only Test (prod - final target)"
        uses: "./.github/actions/build-nautobot-image"
        with:
          images: "ghcr.io/nautobot/nautobot"
          python-version: "${{ matrix.python-version }}"
          nautobot-version: "${{ steps.versionname.outputs.nautobot-version }}"
          arch: "${{ matrix.arch }}"
          push: false
          target: "final"
          poetry-parallel: true
          cache-scope: "nautobot-${{ steps.versionname.outputs.nautobot-version }}"

      - name: "Build and Push (dev - final-dev target)"
        uses: "./.github/actions/build-nautobot-image"
        with:
          images: "ghcr.io/nautobot/nautobot-dev"
          python-version: "${{ matrix.python-version }}"
          nautobot-version: "${{ steps.versionname.outputs.nautobot-version }}"
          arch: "${{ matrix.arch }}"
          push: true
          target: "final-dev"
          poetry-parallel: true
          cache-scope: "nautobot-dev-${{ steps.versionname.outputs.nautobot-version }}"

  container-merge:
    name: "Merge Manifests (publish GHCR, final-dev only)"
    if: |
      github.event_name == 'push' &&
      (github.ref_name == 'develop' || github.ref_name == 'next' || github.ref_name == 'ltm-1.6')
    needs:
      - "container-build"
    runs-on: "ubuntu-24.04"
    strategy:
      fail-fast: true
      matrix:
        python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ]
    env:
      BUILD_PUSH_CONTAINER_ACTION_DEFAULT_PYTHON_VERSION: "3.14"
      BUILD_PUSH_CONTAINER_ACTION_SET_LATEST: "false" # Change this to false for LTM, Pre-release versions force this to false.
    steps:
      - name: "Check out repository code"
        uses: "actions/checkout@v4"
      - name: "Set up Docker Buildx"
        uses: "docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd" # v4
      - name: "Login to GitHub Container Registry"
        uses: "docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc" # v2
        with:
          registry: "ghcr.io"
          username: "${{ github.actor }}"
          password: "${{ secrets.GITHUB_TOKEN }}"

      - name: "Determine if nautobot-version is branch name or from tag name"
        id: "versionname"
        run: |
          if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
            echo "nautobot-version=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
          else
            echo "nautobot-version=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT
          fi

      - name: "Merge (dev - final-dev target)"
        uses: "./.github/actions/merge-image-digests"
        with:
          image: "ghcr.io/nautobot/nautobot-dev"
          python-version: "${{ matrix.python-version }}"
          nautobot-version: "${{ steps.versionname.outputs.nautobot-version }}"
          default-python-version: "${{ env.BUILD_PUSH_CONTAINER_ACTION_DEFAULT_PYTHON_VERSION }}"
          set-latest: "${{ env.BUILD_PUSH_CONTAINER_ACTION_SET_LATEST }}"
          cache-scope: "nautobot-dev-${{ steps.versionname.outputs.nautobot-version }}"