End to End integration tests #191
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End to End integration tests | |
| on: | |
| pull_request: | |
| push: | |
| schedule: | |
| - cron: '0 3 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| runtime_images: | |
| name: Build Python 3.9 Runtime Image | |
| uses: ./.github/workflows/runtime_images.yml | |
| permissions: | |
| packages: write | |
| setup: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| - name: Update pip | |
| run: python3 -m pip install -U pip setuptools | |
| - name: Install python requirements for matrix_parser.py | |
| run: python3 -m pip install -U PyYAML | |
| - name: Identify branch | |
| id: branch | |
| run: python3 tools/github/workflow/set_branch_output_env.py | |
| - name: Identify Matrix | |
| id: matrix | |
| run: echo "matrix=$(./tools/test/matrix_parser.py < ./etc/docker/test/matrix_integration_tests.yml)" >> $GITHUB_OUTPUT | |
| outputs: | |
| branch: ${{ steps.branch.outputs.branch }} | |
| matrix: ${{ steps.matrix.outputs.matrix }} | |
| integration-tests: | |
| needs: [runtime_images, setup] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| cfg: ${{ fromJson(needs.setup.outputs.matrix) }} | |
| steps: | |
| - name: Checkout rucio containers repository | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| with: | |
| repository: rucio/containers | |
| fetch-depth: 0 | |
| - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 | |
| name: Checkout rucio source | |
| with: | |
| path: dev/rucio | |
| fetch-depth: 0 | |
| - name: Select tag for rucio containers | |
| shell: bash | |
| run: | | |
| # Change to cloned rucio/rucio repo | |
| cd $GITHUB_WORKSPACE/dev/rucio | |
| # Get current branch and corresponding latest tag in rucio/rucio repo | |
| BRANCH=$(git rev-parse --abbrev-ref HEAD) | |
| OWNER="${{ github.repository_owner }}" | |
| if [ $OWNER != 'rucio' ]; then | |
| echo "The workflow is running in user ${OWNER}'s fork. Fetching branches and tags from rucio/rucio instead." | |
| git remote add rucio https://github.com/rucio/rucio | |
| git fetch rucio --tags -f | |
| fi | |
| echo "On branch ${BRANCH}" | |
| if [ $BRANCH == 'master' ]; then | |
| GIT_REF="master" | |
| else | |
| GIT_REF=$(git describe --tags --abbrev=0) | |
| IFS=. read major minor micro build <<<"${GIT_REF}" | |
| RELEASE_FAMILY=$major | |
| LATEST_RELEASE_IN_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags/$RELEASE_FAMILY.* | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}') | |
| LATEST_RUCIO_RELEASE_FAMILY=$(git for-each-ref --format '%(refname)' refs/tags | sort -k 1.11V | tail -1 | awk -F'/' '{print $3}' | awk -F'.' '{print $1}') | |
| echo "Release line for ${BRANCH} is ${RELEASE_FAMILY}" | |
| echo "The latest release line for rucio is ${LATEST_RUCIO_RELEASE_FAMILY}" | |
| echo "The latest release in ${RELEASE_FAMILY} is ${LATEST_RELEASE_IN_RELEASE_FAMILY}" | |
| if [ $LATEST_RUCIO_RELEASE_FAMILY = $RELEASE_FAMILY ]; then | |
| GIT_REF='master' # always use containers/master when working on latest rucio/rucio release line | |
| else | |
| GIT_REF=$LATEST_RELEASE_IN_RELEASE_FAMILY # for non-master release line, use the latest rucio/containers tag for the given release family | |
| fi | |
| fi | |
| cd $GITHUB_WORKSPACE | |
| # Check if rucio/containers has a corresponding $GIT_REF | |
| if [ $(git tag -l "$GIT_REF") ]; then | |
| git checkout tags/$GIT_REF | |
| else | |
| echo "Tag $GIT_REF not found in rucio/containers. Integration test containers will be built against the master branch instead." | |
| git checkout master | |
| fi | |
| - name: Set up Docker Buildx | |
| if: needs.runtime_images.outputs.build_locally == 'true' | |
| uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| continue-on-error: true | |
| - name: Build Python 3.9 image locally | |
| if: needs.runtime_images.outputs.build_locally == 'true' | |
| uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 | |
| with: | |
| context: dev/rucio | |
| file: dev/rucio/etc/docker/test/runtime.Dockerfile | |
| target: final | |
| build-args: | | |
| PYTHON=3.9 | |
| push: false | |
| tags: ${{ needs.runtime_images.outputs.py39_image }} | |
| outputs: type=docker | |
| continue-on-error: false | |
| - name: Pull all required images | |
| id: images | |
| shell: bash | |
| run: | | |
| docker compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam pull | |
| - name: Set Runtime Image in Docker Compose | |
| shell: bash | |
| run: | | |
| docker image ls | |
| sed -i 's;image: .*rucio-dev.*;image: ${{ needs.runtime_images.outputs.py39_image }};' \ | |
| $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml | |
| - name: Start containers | |
| run: | | |
| export DEV_PROFILES=storage,externalmetadata,iam | |
| docker compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam up -d | |
| - name: Initialize tests | |
| shell: bash | |
| run: | | |
| docker exec -t dev-rucio-1 bash -c " | |
| set -e | |
| cd /rucio_source | |
| cp etc/rse-accounts.cfg.template /opt/rucio/etc/rse-accounts.cfg | |
| cp etc/rse-accounts.cfg.template /opt/rucio/etc/rse-accounts.cfg.template | |
| cp etc/rse_repository.json /opt/rucio/etc/rse_repository.json | |
| cp etc/rclone-init.cfg /opt/rucio/etc/rclone-init.cfg | |
| # Create symlinks to the mounted SSH keys in standard locations | |
| ln -sf /root/.ssh/ruciouser_sshkey /root/.ssh/id_rsa 2>/dev/null || true | |
| ln -sf /root/.ssh/ruciouser_sshkey.pub /root/.ssh/id_rsa.pub 2>/dev/null || true | |
| pip install --no-cache-dir -e /rucio_source | |
| tools/run_tests.sh -ir | |
| " | |
| - name: File Upload/Download Test | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_rucio_server.py | |
| - name: UploadClient Test | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_upload.py | |
| - name: File Upload/Download Test using 'impl' parameter | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_impl_upload_download.py | |
| - name: Test gfal2 implementation on xrootd protocol | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_gfal2_impl.py | |
| - name: Test Protocol XrootD | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_xrootd.py | |
| - name: Test Protocol SSH (scp) | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_ssh.py | |
| - name: Test Protocol Rsync | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rsync.py | |
| - name: Test Protocol Rclone | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_rse_protocol_rclone.py | |
| - name: Test Conveyor | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_conveyor.py | |
| - name: Execute transfer and export FTS transfer details | |
| id: tpc | |
| shell: bash | |
| run: | | |
| docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short --export-artifacts-from="test_tpc" tests/test_tpc.py | |
| FTS_LOG_FILE=$(docker exec -t dev-rucio-1 cat /tmp/test_tpc.artifact) | |
| echo "fts3log=$FTS_LOG_FILE" >> $GITHUB_OUTPUT | |
| - name: Verify TPC transfers from FTS logs | |
| shell: bash | |
| run: | | |
| FTS_LOG_FILE=${{ steps.tpc.outputs.fts3log }} | |
| if docker exec -t dev-fts-1 /bin/bash -c "grep -Fq \"3rd pull\" $FTS_LOG_FILE" | |
| then | |
| echo "TPC 3rd party pull verified" | |
| else | |
| echo "TPC failed" | |
| exit 1 | |
| fi | |
| - name: Test Token Deletion | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_reaper.py::test_deletion_with_tokens | |
| - name: Print server error logs | |
| if: ${{ success() || failure() }} | |
| shell: bash | |
| run: | | |
| docker exec -t dev-rucio-1 cat /var/log/rucio/httpd_error_log | |
| - name: Archive Upload/Download Test | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_download.py::test_download_from_archive_on_xrd | |
| - name: Test external metadata plugin (mongo) | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaMongo | |
| - name: Test external metadata plugin (postgres) | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaExternalPostgresJSON | |
| - name: Test external metadata plugin (elasticsearch) | |
| run: docker exec -t dev-rucio-1 tools/pytest.sh -v --tb=short tests/test_did_meta_plugins.py::TestDidMetaElastic | |
| - name: Stop containers | |
| run: docker compose -f $GITHUB_WORKSPACE/dev/rucio/etc/docker/dev/docker-compose.yml --profile storage --profile externalmetadata --profile iam down |