[#697] Github actions to run all PRC tests

.github/workflows/README.md

@@ -0,0 +1,18 @@
+Tests of Python iRODS Client
+============================
+
+Test PRC (Python iRODS Client) with an instance of the iRODS server.
+
+The tests we will run are
+
+-    `main.yml`: static type tests using `mypy`.
+-    `run-the-tests.yml`: set up separate nodes (via Docker compose) to run the client suite.  Nodes include a provider, catalog, and client.
+     (See the `docker-testing` subdirectory for the README documentation and implementation of this test mechanism.)
+-    `run-local-suite.yml`: test suite run  within an OS-level virtualized container, also hosting iRODS server and managed by Docker or equivalent.
+-    `run-bats-tests.yml`: run a set of tests, each in its own container under similar conditions to (c).  Each test may have its own unique setup.
+     (See the `harness` subdirectory for the README documentation and implementation of this test mechanism.)
+
+Running
+-------
+
+The tests are run each time a commit is pushed to this repository or a fork of it.  They are also run when a pull request is made.

.github/workflows/run-bats-tests.yml

@@ -0,0 +1,30 @@
+name: run-bats-tests
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    timeout-minutes: 20
+
+    name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+          working-directory: ./irods/test/harness
+    strategy:
+      matrix:
+        python: ['3.9','3.13']
+        irods_server: ['4.3.4','5.0.2']
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build images
+        run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"
+
+      - name: run tests
+        run: |
+          for script in ../scripts/test[0-9]*; do
+              ./docker_container_driver.sh -V $script
+          done

.github/workflows/run-local-suite.yml

@@ -0,0 +1,28 @@
+name: run-local-suite
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    timeout-minutes: 20
+
+    name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+          working-directory: ./irods/test/harness
+    strategy:
+      matrix:
+        python: ['3.9','3.13']
+        irods_server: ['4.3.4','5.0.2']
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Build images
+        run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"
+
+      - name: run tests
+        run: |
+          ./docker_container_driver.sh -V ../scripts/run_suite_locally.sh

.github/workflows/run-the-tests.yml

@@ -0,0 +1,38 @@
+name: run-the-tests
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    timeout-minutes: 20
+
+    name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }}
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+          working-directory: ./docker-testing
+    strategy:
+      matrix:
+        python: ['3.9','3.13']
+        irods_server: ['4.3.4','5.0.2']
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Start containers
+        run: ./start_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"
+
+      - name: run test
+        run: |
+          while :; do
+            client_container=$(docker ps --format "{{.Names}}"|grep python.client)
+            [ -n "$client_container" ] && break
+            sleep 1
+          done
+          echo "client_container = [$client_container]"
+          docker exec "${client_container}" /repo_root/docker-testing/run_tests.sh
+
+      - name: Stop containers
+        if: always()
+        run: ./stop_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}"

Dockerfile.prc_test.centos

@@ -24,6 +24,5 @@ RUN  python${py_N} repo/docker_build/iinit.py \
         password rods
 SHELL ["/bin/bash","-c"]
 CMD  echo "Waiting on iRODS server... " ; \
-     python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \
      sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \
      newgrp irods < repo/run_python_tests.sh

Dockerfile.prc_test.ubuntu

@@ -31,6 +31,5 @@ SHELL ["/bin/bash","-c"]
 #  3. run python tests as the new group
 
 CMD  echo "Waiting on iRODS server... " ; \
-     python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \
      sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \
      newgrp irods < repo/run_python_tests.sh

docker-testing/README.md

@@ -0,0 +1,66 @@
+# A Topological Setup for Testing the Python Client
+
+The `docker-testing` directory contains the necessary files for building and
+running tests from the perspective of a specific client node in a larger network.
+
+We currently allow a choice of Python interpreter and iRODS server to be installed
+on the client and provider nodes of a simulated network topology.
+
+The choice of versions are dictated when running the test:
+
+|:------------------:|:---------------:|
+|Environment Variable| Valid Range     |
+|:-------------------|-----------------|
+IRODS_PACKAGE_VERSION|4.3.1 to 5.0.2   |
+PYTHON_VERSION       |3.9 to 3.13      |
+|:-------------------|-----------------|
+
+Currently the database server is fixed as Postgres.
+
+## Details of usage
+
+The file `$REPO/.github/workflows/run-the-tests.yml`
+(where `$REPO` is the /path/to/local/python-irodsclient repository)
+contains commands for starting the server and client containers and running the PRC
+suite in response to a push or pull-request.
+
+The tests can also be run on any workstation with "docker compose" installed.
+What follows is a short summary of how to run the test configuration "at the bench".
+It is this procedure which is run within the Github workflows.
+
+   1. cd into top level of $REPO
+
+   2. run:
+      ```
+      ./docker-testing/start_containers.sh 4.3.4 3.11
+      ```
+      This builds and runs the docker images for the project,  with "4.3.4" being the iRODS
+      version installed on the provider and "3.11" is the version of python run on the client side.
+
+   3. run:
+      ```
+      docker exec <name-of-python-client-container> /repo_root/docker-testing/run_tests.sh
+      ```
+      (Note:  `/repo_root` is an actual literal path, internal to the container.)
+      You'll see the test output displayed on the console.  At completion, xmlrunner outputs are in /tmp.
+
+   4. use `docker logs -f` with the provider instance name to tail the irods server log output
+
+DEBUGGING
+---------
+We can also to run a specific test that we specify by name:
+
+```
+$ docker exec -it <name-of-python-client-container> /repo_root/docker_testing/run_tests.sh irods.test.<module>.<class>.<method>
+```
+
+Optionally we can also enter the PDB command-line debugger at a place of our choosing in the source code, by stopping on a breakpoint,
+and then stepping through code.
+
+The breakpoint can be placed by adding the line
+
+```
+import pdb;pdb.set_trace()
+```
+
+immediately before the source line in the test code at which we wish to enter the debugger.

docker-testing/harness-docker-compose-irods-4.yml

@@ -0,0 +1 @@
+harness-docker-compose.yml

docker-testing/harness-docker-compose-irods-5.yml

@@ -0,0 +1 @@
+harness-docker-compose.yml

docker-testing/harness-docker-compose.yml

@@ -0,0 +1,47 @@
+version: '3'
+
+services:
+    irods-catalog:
+        build:
+            context: irods_catalog_${irods_major}
+        # 5432 is exposed by default and can conflict with other postgres containers.
+        # When the metalnx-db service is no longer needed, this stanza can be removed.
+        ports:
+            - "5430:5432"
+        environment:
+            - POSTGRES_PASSWORD=testpassword
+
+    python-client:
+        build:
+            context: python_client
+            args:
+                python_version: ${python_version}
+        command:
+            tail -f /dev/null
+        volumes:
+            - ${repo_external}:/repo_root:ro
+            - /tmp/irods-client-share.py-${python_version}:/irods_shared
+        depends_on:
+            irods-catalog-provider:
+                condition: service_healthy
+
+    irods-catalog-provider:
+        volumes:
+            - /tmp/irods-client-share.py-${python_version}:/irods_shared
+        build:
+            context: irods_catalog_provider_${irods_major}
+            args:
+                irods_version: ${irods_version}
+        shm_size: 500mb
+        healthcheck:
+            test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"]
+            interval: 10s
+            timeout: 10s
+            retries: 3
+            #start_period: 20s
+            #start_interval: 2s
+        ports:
+            - "1247:1247"
+        depends_on:
+            - irods-catalog
+

docker-testing/iinit.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+from irods.password_obfuscation import encode
+import json
+import os
+import sys
+from os import chmod
+from os.path import expanduser,exists,join
+from getopt import getopt
+
+
+home_env_path = expanduser('~/.irods')
+env_file_path = join(home_env_path,'irods_environment.json')
+auth_file_path = join(home_env_path,'.irodsA')
+
+
+def do_iinit(host, port, user, zone, password):
+    if not exists(home_env_path):
+        os.makedirs(home_env_path)
+    else:
+        raise RuntimeError('~/.irods already exists')
+
+    with open(env_file_path,'w') as env_file:
+        json.dump ( { "irods_host": host,
+                      "irods_port": int(port),
+                      "irods_user_name": user,
+                      "irods_zone_name": zone  }, env_file, indent=4)
+    with open(auth_file_path,'w') as auth_file:
+        auth_file.write(encode(password))
+    chmod (auth_file_path,0o600)
+
+
+def get_kv_pairs_from_cmdline(*args):
+    arglist = list(args)
+    while arglist:
+        k = arglist.pop(0)
+        v = arglist.pop(0)
+        yield k,v
+
+
+if __name__ == '__main__':
+    args = sys.argv[1:]
+    dct = {k:v for k,v in get_kv_pairs_from_cmdline(*args)}
+    do_iinit(**dct)

docker-testing/irods_catalog_4/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:12
+
+COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh

docker-testing/irods_catalog_4/init-user-db.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Adapted from "Initialization script" in documentation for official Postgres dockerhub:
+#   https://hub.docker.com/_/postgres/
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE DATABASE "ICAT";
+    CREATE USER irods WITH PASSWORD 'testpassword';
+    GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods;
+EOSQL

docker-testing/irods_catalog_5/Dockerfile

@@ -0,0 +1,3 @@
+FROM postgres:16
+
+COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh

docker-testing/irods_catalog_5/init-user-db.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Adapted from "Initialization script" in documentation for official Postgres dockerhub:
+#   https://hub.docker.com/_/postgres/
+set -e
+
+psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
+    CREATE DATABASE "ICAT";
+    CREATE USER irods WITH PASSWORD 'testpassword';
+    GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods;
+    ALTER DATABASE "ICAT" OWNER TO irods
+EOSQL

docker-testing/irods_catalog_provider_4/Dockerfile

@@ -0,0 +1,55 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && \
+    apt-get install -y \
+        apt-transport-https \
+        gnupg \
+        wget \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \
+    echo "deb [arch=amd64] https://packages.irods.org/apt/ focal main" | tee /etc/apt/sources.list.d/renci-irods.list
+
+RUN apt-get update && \
+    apt-get install -y \
+        libcurl4-gnutls-dev \
+        jq \
+        python3 \
+        python3-distro \
+        python3-jsonschema \
+        python3-pip \
+        python3-psutil \
+        python3-requests \
+        rsyslog \
+        unixodbc \
+        gawk \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+ARG irods_version=4.3.1
+ARG irods_package_version_suffix=-0~focal
+ARG irods_package_version=${irods_version}${irods_package_version_suffix}
+ARG irods_resource_plugin_version=${irods_version}.0${irods_package_version_suffix}
+
+RUN apt-get update && \
+    apt-get install -y \
+         irods-database-plugin-postgres=${irods_package_version} \
+         irods-runtime=${irods_package_version} \
+         irods-server=${irods_package_version} \
+         irods-icommands=${irods_package_version} \
+    && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/*
+
+COPY setup-${irods_version}.input /
+RUN mv /setup-${irods_version}.input /irods_setup.input
+
+WORKDIR /
+COPY entrypoint.sh .
+RUN chmod u+x ./entrypoint.sh
+ENTRYPOINT ["./entrypoint.sh"]