diff --git a/.github/workflows/run-bats-tests.yml b/.github/workflows/run-bats-tests.yml new file mode 100644 index 000000000..0475d1a54 --- /dev/null +++ b/.github/workflows/run-bats-tests.yml @@ -0,0 +1,30 @@ +name: run-bats-tests + +on: [push, pull_request] + +jobs: + tests: + timeout-minutes: 20 + + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./irods/test/harness + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + for script in ../scripts/test00[1-9]*; do + ./docker_container_driver.sh -V $script + done diff --git a/.github/workflows/run-local-suite.yml b/.github/workflows/run-local-suite.yml new file mode 100644 index 000000000..e3d48ea95 --- /dev/null +++ b/.github/workflows/run-local-suite.yml @@ -0,0 +1,28 @@ +name: run-local-suite + +on: [push, pull_request] + +jobs: + tests: + timeout-minutes: 20 + + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./irods/test/harness + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Build images + run: ./create_docker_images.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run tests + run: | + ./docker_container_driver.sh -V ../scripts/run_suite_locally.sh diff --git a/.github/workflows/run-the-tests.yml b/.github/workflows/run-the-tests.yml new file mode 100644 index 000000000..a7367dced --- /dev/null +++ b/.github/workflows/run-the-tests.yml @@ -0,0 +1,38 @@ +name: run-the-tests + +on: [push, pull_request] + +jobs: + tests: + timeout-minutes: 20 + + name: Python ${{ matrix.python }}, iRODS ${{ matrix.irods_server }} + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./docker-testing + strategy: + matrix: + python: ['3.9','3.13'] + irods_server: ['4.3.4','5.0.2'] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start containers + run: ./start_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" + + - name: run test + run: | + while :; do + client_container=$(docker ps --format "{{.Names}}"|grep python.client) + [ -n "$client_container" ] && break + sleep 1 + done + echo "client_container = [$client_container]" + docker exec "${client_container}" /repo_root/docker-testing/run_tests.sh + + - name: Stop containers + if: always() + run: ./stop_containers.sh "${{ matrix.irods_server }}" "${{ matrix.python }}" diff --git a/Dockerfile.prc_test.centos b/Dockerfile.prc_test.centos index debed6d6d..c171fa62e 100644 --- a/Dockerfile.prc_test.centos +++ b/Dockerfile.prc_test.centos @@ -24,6 +24,5 @@ RUN python${py_N} repo/docker_build/iinit.py \ password rods SHELL ["/bin/bash","-c"] CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/Dockerfile.prc_test.ubuntu b/Dockerfile.prc_test.ubuntu index e8c958a85..79ed07e12 100644 --- a/Dockerfile.prc_test.ubuntu +++ b/Dockerfile.prc_test.ubuntu @@ -31,6 +31,5 @@ SHELL ["/bin/bash","-c"] # 3. run python tests as the new group CMD echo "Waiting on iRODS server... " ; \ - python${PY_N} repo/docker_build/recv_oneshot -h irods-provider -p 8888 -t 360 && \ sudo groupadd -o -g $(stat -c%g /irods_shared) irods && sudo usermod -aG irods user && \ newgrp irods < repo/run_python_tests.sh diff --git a/docker-testing/README.md b/docker-testing/README.md new file mode 100644 index 000000000..e1e566c8a --- /dev/null +++ b/docker-testing/README.md @@ -0,0 +1,42 @@ +The file `$REPO/.github/workflows/run-the-tests.yml` +(where `$REPO` is the /path/to/local/python-irodsclient repository) +contains commands for starting the server and client containers and running the PRC +suite in response to a push or pull-request. + +The tests suite can also be run on any workstation with "docker compose" installed: + + 1. cd into top level of $REPO + + 2. run: + ``` + ./docker-testing/start_containers.sh 3.6 + ``` + This builds and runs the docker images. "3.6" is the version of python desired. + + 3. run: + ``` + docker exec /repo_root/docker-testing/run_tests.sh + ``` + (Note: `/repo_root` is an actual literal path, internal to the container.) + You'll see the test output displayed on the console. At completion, xmlrunner outputs are in /tmp. + + 4. use `docker logs -f` with the provider instance name to tail the irods server log output + +DEBUGGING +--------- +We can also to run a specific test that we specify by name: + +``` +$ docker exec -it /repo_root/docker_testing/run_tests.sh irods.test... +``` + +Optionally we can also enter the PDB command-line debugger at a place of our choosing in the source code, by stopping on a breakpoint, +and then stepping through code. + +The breakpoint can be placed by adding the line + +``` +import pdb;pdb.set_trace() +``` + +immediately before the source line in the test code at which we wish to enter the debugger. diff --git a/docker-testing/harness-docker-compose-irods-4.yml b/docker-testing/harness-docker-compose-irods-4.yml new file mode 120000 index 000000000..db32ad86f --- /dev/null +++ b/docker-testing/harness-docker-compose-irods-4.yml @@ -0,0 +1 @@ +harness-docker-compose.yml \ No newline at end of file diff --git a/docker-testing/harness-docker-compose-irods-5.yml b/docker-testing/harness-docker-compose-irods-5.yml new file mode 120000 index 000000000..db32ad86f --- /dev/null +++ b/docker-testing/harness-docker-compose-irods-5.yml @@ -0,0 +1 @@ +harness-docker-compose.yml \ No newline at end of file diff --git a/docker-testing/harness-docker-compose.yml b/docker-testing/harness-docker-compose.yml new file mode 100644 index 000000000..931be2166 --- /dev/null +++ b/docker-testing/harness-docker-compose.yml @@ -0,0 +1,47 @@ +version: '3' + +services: + irods-catalog: + build: + context: irods_catalog_${irods_major} + # 5432 is exposed by default and can conflict with other postgres containers. + # When the metalnx-db service is no longer needed, this stanza can be removed. + ports: + - "5430:5432" + environment: + - POSTGRES_PASSWORD=testpassword + + python-client: + build: + context: python_client + args: + python_version: ${python_version} + command: + tail -f /dev/null + volumes: + - ${repo_external}:/repo_root:ro + - /tmp/irods-client-share.py-${python_version}:/irods_shared + depends_on: + irods-catalog-provider: + condition: service_healthy + + irods-catalog-provider: + volumes: + - /tmp/irods-client-share.py-${python_version}:/irods_shared + build: + context: irods_catalog_provider_${irods_major} + args: + irods_version: ${irods_version} + shm_size: 500mb + healthcheck: + test: ["CMD", "su", "-", "irods", "-c", "ils || exit 1"] + interval: 10s + timeout: 10s + retries: 3 + #start_period: 20s + #start_interval: 2s + ports: + - "1247:1247" + depends_on: + - irods-catalog + diff --git a/docker-testing/iinit.py b/docker-testing/iinit.py new file mode 100755 index 000000000..d60bca8fd --- /dev/null +++ b/docker-testing/iinit.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python + +from getpass import getpass +from irods.password_obfuscation import encode +import json +import os +import sys +from os import chmod +from os.path import expanduser,exists,join +from getopt import getopt + + +home_env_path = expanduser('~/.irods') +env_file_path = join(home_env_path,'irods_environment.json') +auth_file_path = join(home_env_path,'.irodsA') + + +def do_iinit(host, port, user, zone, password): + if not exists(home_env_path): + os.makedirs(home_env_path) + else: + raise RuntimeError('~/.irods already exists') + + with open(env_file_path,'w') as env_file: + json.dump ( { "irods_host": host, + "irods_port": int(port), + "irods_user_name": user, + "irods_zone_name": zone }, env_file, indent=4) + with open(auth_file_path,'w') as auth_file: + auth_file.write(encode(password)) + chmod (auth_file_path,0o600) + + +def get_kv_pairs_from_cmdline(*args): + arglist = list(args) + while arglist: + k = arglist.pop(0) + v = arglist.pop(0) + yield k,v + + +if __name__ == '__main__': + import sys + args = sys.argv[1:] + dct = {k:v for k,v in get_kv_pairs_from_cmdline(*args)} + do_iinit(**dct) diff --git a/docker-testing/irods_catalog_4/Dockerfile b/docker-testing/irods_catalog_4/Dockerfile new file mode 100644 index 000000000..f02c4520c --- /dev/null +++ b/docker-testing/irods_catalog_4/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:12 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/docker-testing/irods_catalog_4/init-user-db.sh b/docker-testing/irods_catalog_4/init-user-db.sh new file mode 100644 index 000000000..5ff6b0375 --- /dev/null +++ b/docker-testing/irods_catalog_4/init-user-db.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; +EOSQL diff --git a/docker-testing/irods_catalog_5/Dockerfile b/docker-testing/irods_catalog_5/Dockerfile new file mode 100644 index 000000000..112ffbaaa --- /dev/null +++ b/docker-testing/irods_catalog_5/Dockerfile @@ -0,0 +1,3 @@ +FROM postgres:16 + +COPY init-user-db.sh /docker-entrypoint-initdb.d/init-user-db.sh diff --git a/docker-testing/irods_catalog_5/init-user-db.sh b/docker-testing/irods_catalog_5/init-user-db.sh new file mode 100644 index 000000000..f3c724e2f --- /dev/null +++ b/docker-testing/irods_catalog_5/init-user-db.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Adapted from "Initialization script" in documentation for official Postgres dockerhub: +# https://hub.docker.com/_/postgres/ +set -e + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + CREATE DATABASE "ICAT"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE "ICAT" to irods; + ALTER DATABASE "ICAT" OWNER TO irods +EOSQL diff --git a/docker-testing/irods_catalog_provider_4/Dockerfile b/docker-testing/irods_catalog_provider_4/Dockerfile new file mode 100644 index 000000000..75d3bd17a --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/Dockerfile @@ -0,0 +1,57 @@ +FROM ubuntu:20.04 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ focal main" | tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +ARG irods_version=4.3.1 +ARG irods_package_version_suffix=-0~focal +ARG irods_package_version=${irods_version}${irods_package_version_suffix} +ARG irods_resource_plugin_version=${irods_version}.0${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +COPY send_oneshot . +RUN chmod u+x ./entrypoint.sh +RUN chmod u+x ./send_oneshot +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker-testing/irods_catalog_provider_4/entrypoint.sh b/docker-testing/irods_catalog_provider_4/entrypoint.sh new file mode 100644 index 000000000..3a321db2f --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/entrypoint.sh @@ -0,0 +1,49 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q +do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py < "${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared + +#TODO ensure this is done for 4.3+ only. 4.2 doesn't have this server config key +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && \ + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && \ + touch ~/provider-address.do_not_remove + } +} || { echo >&2 "Error modifying $ORIG_SERVER_CONFIG"; exit 1; } + +echo "Starting server" + +cd /usr/sbin +su irods -c 'bash -c "./irodsServer -u"' + diff --git a/docker-testing/irods_catalog_provider_4/send_oneshot b/docker-testing/irods_catalog_provider_4/send_oneshot new file mode 100755 index 000000000..b265af15d --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/send_oneshot @@ -0,0 +1,6 @@ +#!/usr/bin/gawk -f +BEGIN { + SERVER = "/inet/tcp/"ENVIRON["PORT"]"/0/0" + print ARGV[1] " - " strftime() |& SERVER + close(SERVER) +} diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.1.input b/docker-testing/irods_catalog_provider_4/setup-4.3.1.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.1.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.2.input b/docker-testing/irods_catalog_provider_4/setup-4.3.2.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.2.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.3.input b/docker-testing/irods_catalog_provider_4/setup-4.3.3.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.3.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_4/setup-4.3.4.input b/docker-testing/irods_catalog_provider_4/setup-4.3.4.input new file mode 100644 index 000000000..d8c10deca --- /dev/null +++ b/docker-testing/irods_catalog_provider_4/setup-4.3.4.input @@ -0,0 +1,28 @@ + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +1248 + +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +32_byte_server_control_plane_key +rods + + diff --git a/docker-testing/irods_catalog_provider_5/Dockerfile b/docker-testing/irods_catalog_provider_5/Dockerfile new file mode 100644 index 000000000..d808ed458 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/Dockerfile @@ -0,0 +1,62 @@ +FROM ubuntu:24.04 +#dwm ^ + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && \ + apt-get install -y \ + apt-transport-https \ + gnupg \ + wget \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +RUN wget -qO - https://packages.irods.org/irods-signing-key.asc | apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ noble main" | tee /etc/apt/sources.list.d/renci-irods.list + +RUN apt-get update && \ + apt-get install -y \ + libcurl4-gnutls-dev \ + jq \ + python3 \ + python3-distro \ + python3-jsonschema \ + python3-pip \ + python3-psutil \ + python3-requests \ + rsyslog \ + unixodbc \ + gawk \ + postgresql-client-16 \ + vim-tiny \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +# - postgres client 16 is for pg_isready +# TODO delete vim-tiny + +ARG irods_version=5.0.1 +ARG irods_package_version_suffix=-0~noble +ARG irods_package_version=${irods_version}${irods_package_version_suffix} + +RUN apt-get update && \ + apt-get install -y \ + irods-database-plugin-postgres=${irods_package_version} \ + irods-runtime=${irods_package_version} \ + irods-server=${irods_package_version} \ + irods-icommands=${irods_package_version} \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* + +COPY setup-${irods_version}.input / +RUN mv /setup-${irods_version}.input /irods_setup.input + +WORKDIR / +COPY entrypoint.sh . +COPY send_oneshot . +RUN chmod u+x ./entrypoint.sh +RUN chmod u+x ./send_oneshot +ENTRYPOINT ["./entrypoint.sh"] diff --git a/docker-testing/irods_catalog_provider_5/entrypoint.sh b/docker-testing/irods_catalog_provider_5/entrypoint.sh new file mode 100644 index 000000000..b3aec63cf --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/entrypoint.sh @@ -0,0 +1,50 @@ +#! /bin/bash -e + +catalog_db_hostname=irods-catalog + +echo "Waiting for iRODS catalog database to be ready" + +until pg_isready -h ${catalog_db_hostname} -d ICAT -U irods -q +do + sleep 1 +done + +echo "iRODS catalog database is ready" + +setup_input_file=/irods_setup.input + +if [ -e "${setup_input_file}" ]; then + echo "Running iRODS setup" + python3 /var/lib/irods/scripts/setup_irods.py < "${setup_input_file}" + rm /irods_setup.input +fi + +ORIG_SERVER_CONFIG=/etc/irods/server_config.json +MOD_SERVER_CONFIG=/tmp/server_config.json.$$ + +chown -R irods:irods /irods_shared +chmod 0777 /irods_shared + +#TODO ensure this is done for 4.3+ only. 4.2 doesn't have this server config key +{ + [ -f ~/provider-address.do_not_remove ] || { + jq <$ORIG_SERVER_CONFIG >$MOD_SERVER_CONFIG \ + '.host_resolution.host_entries += [ + { + "address_type": "local", + "addresses": [ + "irods-catalog-provider", + "'$(hostname)'" + ] + } + ]' && \ + cat <$MOD_SERVER_CONFIG >$ORIG_SERVER_CONFIG && \ + touch ~/provider-address.do_not_remove + } +} || { echo >&2 "Error modifying $ORIG_SERVER_CONFIG"; exit 1; } + +echo "Starting server" + +cd /usr/sbin +su irods -c 'bash -c "./irodsServer -p /tmp/irods.pid"' + diff --git a/docker-testing/irods_catalog_provider_5/send_oneshot b/docker-testing/irods_catalog_provider_5/send_oneshot new file mode 100755 index 000000000..b265af15d --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/send_oneshot @@ -0,0 +1,6 @@ +#!/usr/bin/gawk -f +BEGIN { + SERVER = "/inet/tcp/"ENVIRON["PORT"]"/0/0" + print ARGV[1] " - " strftime() |& SERVER + close(SERVER) +} diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.0.input b/docker-testing/irods_catalog_provider_5/setup-5.0.0.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.0.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.1.input b/docker-testing/irods_catalog_provider_5/setup-5.0.1.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.1.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/irods_catalog_provider_5/setup-5.0.2.input b/docker-testing/irods_catalog_provider_5/setup-5.0.2.input new file mode 100644 index 000000000..9bcaf0852 --- /dev/null +++ b/docker-testing/irods_catalog_provider_5/setup-5.0.2.input @@ -0,0 +1,26 @@ + + + + + +irods-catalog +5432 +ICAT +irods +y +testpassword + +y +demoResc + +tempZone +1247 +20000 +20199 +rods +y +TEMPORARY_ZONE_KEY +32_byte_server_negotiation_key__ +rods + + diff --git a/docker-testing/print_repo_root_location b/docker-testing/print_repo_root_location new file mode 100755 index 000000000..79d91af7e --- /dev/null +++ b/docker-testing/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/docker-testing/python_client/Dockerfile b/docker-testing/python_client/Dockerfile new file mode 100644 index 000000000..83c7ff885 --- /dev/null +++ b/docker-testing/python_client/Dockerfile @@ -0,0 +1,3 @@ +ARG python_version +FROM python:${python_version} +RUN pip install remote-pdb diff --git a/docker-testing/recv_oneshot b/docker-testing/recv_oneshot new file mode 100755 index 000000000..47e2bdd09 --- /dev/null +++ b/docker-testing/recv_oneshot @@ -0,0 +1,35 @@ +#!/usr/bin/env python +from __future__ import print_function +import sys, os, time +from socket import * +import getopt + +def try_connect(host,port): + try: + s=socket(AF_INET,SOCK_STREAM) + s.connect((host,port)) + return s + except: + s.close() + return None + +# Options: +# +# -t timeout +# -h host +# -p port + +t = now = time.time() +opts = dict(getopt.getopt(sys.argv[1:],'t:h:p:')[0]) + +host = opts['-h'] +port = int(opts['-p']) +timeout = float(opts['-t']) + +while time.time() < now + timeout: + time.sleep(1) + s = try_connect(host, port) + if s: + print(s.recv(32767).decode('utf-8'),end='') + exit(0) +exit(1) diff --git a/docker-testing/run_tests.sh b/docker-testing/run_tests.sh new file mode 100755 index 000000000..9cbd59a01 --- /dev/null +++ b/docker-testing/run_tests.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e -x +PYTHON=$(which python3) +if [ -z "$PYTHON" ]; then + PYTHON=$(which python) +fi +DIR=$(dirname "$0") +cd "$DIR" + +## Replacing server ready notify mechanism with Docker healthcheck +#$PYTHON ./recv_oneshot -h irods-catalog-provider -p 8888 -t 360 + +REPO="$(./print_repo_root_location)" +#$PYTHON -m pip install "$REPO[tests]" + +if [ -d /irods_shared ]; then + groupadd -o -g $(stat -c%g /irods_shared) irods # Appropriate the integer codes for irods group ... + useradd -g irods -u $(stat -c%u /irods_shared) irods # ... and user. + mkdir /irods_shared/{tmp,reg_resc} + chown irods:irods /irods_shared/{tmp,reg_resc} + chmod 777 /irods_shared/reg_resc + chmod g+ws /irods_shared/tmp + useradd -G irods -m -s/bin/bash user + cp -r /"$REPO"{,.copy} + REPO+=.copy + chown -R user "$REPO" + chmod u+w "$REPO"/irods/test/test-data + $PYTHON -m pip install "$REPO[tests]" +fi + +su - user -c "\ +$PYTHON '$DIR'/iinit.py \ + host irods-catalog-provider \ + port 1247 \ + user rods \ + password rods \ + zone tempZone +$PYTHON '$REPO'/irods/test/runner.py $*" diff --git a/docker-testing/send_oneshot b/docker-testing/send_oneshot new file mode 100755 index 000000000..b265af15d --- /dev/null +++ b/docker-testing/send_oneshot @@ -0,0 +1,6 @@ +#!/usr/bin/gawk -f +BEGIN { + SERVER = "/inet/tcp/"ENVIRON["PORT"]"/0/0" + print ARGV[1] " - " strftime() |& SERVER + close(SERVER) +} diff --git a/docker-testing/start_containers.sh b/docker-testing/start_containers.sh new file mode 100755 index 000000000..fd110154e --- /dev/null +++ b/docker-testing/start_containers.sh @@ -0,0 +1,56 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 [-n] [-b ""] [irods_version] python_version"; exit 2; +} + +SHELL_DOCKER_COMPOSE_BUILD_ARGS="" +DO_NOT_RUN="" + +while [[ $1 = -* ]]; do + if [ $1 = "-b" ]; then + SHELL_DOCKER_COMPOSE_BUILD_ARGS=$2 + shift 2 + fi + if [ $1 = "-n" ]; then + DO_NOT_RUN=1 + shift + fi +done + +if [ $# -eq 2 ]; then + IRODS_VERSION=$1 + PYTHON_VERSION=$2 +elif [ $# -eq 1 ]; then + IRODS_VERSION=4.3.4 + PYTHON_VERSION=$1 +else + usage +fi + +shift $# + +[ -n "$PYTHON_VERSION" -a -n "$IRODS_VERSION" ] || { + usage +} + +IRODS_MAJOR=$(sed -e 's/\..*//' <<<"$IRODS_VERSION") + +DIR=$(dirname "$0") +cd "${DIR}" +REPO_ROOT=$(realpath ..) + +echo "\ +repo_external=\"${REPO_ROOT}\" +python_version=\"${PYTHON_VERSION}\" +irods_version=\"${IRODS_VERSION}\" +irods_major=\"${IRODS_MAJOR}\"" >.env + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml build $SHELL_DOCKER_COMPOSE_BUILD_ARGS + +if [ -z "$DO_NOT_RUN" ]; then + docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml up -d +fi diff --git a/docker-testing/stop_containers.sh b/docker-testing/stop_containers.sh new file mode 100755 index 000000000..7f8ff8df2 --- /dev/null +++ b/docker-testing/stop_containers.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -e + +# This script is launched on the docker host. + +usage() { + echo >&2 "usage: $0 [irods_version] python_version"; exit 1; +} + +if [ $# -eq 2 ]; then + IRODS_VERSION=$1 + PYTHON_VERSION=$2 +elif [ $# -eq 1 ]; then + IRODS_VERSION=4.3.4 + PYTHON_VERSION=$1 +else + usage +fi + +shift $# + +[ -n "$PYTHON_VERSION" -a -n "$IRODS_VERSION" ] || { + usage +} + + +IRODS_MAJOR=$(sed -e 's/\..*//' <<<"$IRODS_VERSION") + +docker compose -f harness-docker-compose-irods-${IRODS_MAJOR}.yml down diff --git a/irods/message/__init__.py b/irods/message/__init__.py index 818ae3677..9b8e0ec80 100644 --- a/irods/message/__init__.py +++ b/irods/message/__init__.py @@ -181,7 +181,7 @@ def ET(xml_type=(), server_version=None): logger = logging.getLogger(__name__) -IRODS_VERSION = (5, 0, 1, "d") +IRODS_VERSION = (5, 0, 2, "d") UNICODE = str diff --git a/irods/test/access_test.py b/irods/test/access_test.py index 7c496c685..73c122b76 100644 --- a/irods/test/access_test.py +++ b/irods/test/access_test.py @@ -533,11 +533,7 @@ def test_iRODSAccess_cannot_be_constructed_using_unsupported_type__issue_558(sel # Before the fix in #558, this would have been allowed and only later would the type discrepancy be revealed, # leading to opaque error messages. Now, the types are checked on the way in to ensure clarity and correctness. # TODO(#480): We cannot use the unittest.assertRaises context manager as this was introduced in python 3.1. - assertCall = getattr(self, "assertRaisesRegex", None) - if assertCall is None: - assertCall = self.assertRaisesRegexp - - assertCall( + self.assertRaisesRegex( TypeError, "'path' parameter must be of type 'str', 'irods.collection.iRODSCollection', " "'irods.data_object.iRODSDataObject', or 'irods.path.iRODSPath'.", diff --git a/irods/test/data_obj_test.py b/irods/test/data_obj_test.py index b6ed95886..3013aab7d 100644 --- a/irods/test/data_obj_test.py +++ b/irods/test/data_obj_test.py @@ -1,6 +1,6 @@ #! /usr/bin/env python -from datetime import datetime, timezone +from datetime import datetime, timezone, timedelta import base64 import collections import concurrent.futures @@ -3305,16 +3305,13 @@ def test_access_time__issue_700(self): if self.sess.server_version < (5,): self.skipTest("iRODS servers < 5.0.0 do not provide an access_time attribute for data objects.") - data_path= iRODSPath(self.coll.path, - unique_name(my_function_name(), datetime.now()) - ) - with self.sess.data_objects.open(data_path,"w") as f: - f.write(b'_') - with self.sess.data_objects.open(data_path,"r") as f: - f.read() + # Create a new, uniquely named test data object. + data = self.sess.data_objects.create( + f'{helpers.home_collection(self.sess)}/{unique_name(my_function_name(), datetime.now())}' + ) - data = self.sess.data_objects.get(data_path) - self.assertGreaterEqual(data.access_time, data.modify_time) + # Test that access_time is there, and of the right type. + self.assertIs(type(data.access_time), datetime) if __name__ == "__main__": # let the tests find the parent irods lib diff --git a/irods/test/demo.sh b/irods/test/demo.sh new file mode 100755 index 000000000..7ef5b7c2e --- /dev/null +++ b/irods/test/demo.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +echo "$0 running" +echo args: +for arg in $*; do + echo $((++x)): "[$arg]" +done + +exit 118 diff --git a/irods/test/demo_A.sh b/irods/test/demo_A.sh new file mode 120000 index 000000000..52239f28f --- /dev/null +++ b/irods/test/demo_A.sh @@ -0,0 +1 @@ +demo.sh \ No newline at end of file diff --git a/irods/test/demo_B.sh b/irods/test/demo_B.sh new file mode 120000 index 000000000..52239f28f --- /dev/null +++ b/irods/test/demo_B.sh @@ -0,0 +1 @@ +demo.sh \ No newline at end of file diff --git a/irods/test/demo_hook.sh b/irods/test/demo_hook.sh new file mode 100755 index 000000000..8a9f855fe --- /dev/null +++ b/irods/test/demo_hook.sh @@ -0,0 +1,3 @@ +#!/bin/bash +echo "-- HOOK RUNNING --" +command "/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" $* diff --git a/irods/test/exception_test.py b/irods/test/exception_test.py index 16a95c989..b400cf2aa 100644 --- a/irods/test/exception_test.py +++ b/irods/test/exception_test.py @@ -41,8 +41,8 @@ def test_400(self): excep_repr = repr(exc) errno_object = irods.exception.Errno(errno.EACCES) errno_repr = repr(errno_object) - self.assertRegexpMatches(errno_repr, r"\bErrno\b") - self.assertRegexpMatches( + self.assertRegex(errno_repr, r"\bErrno\b") + self.assertRegex( errno_repr, """['"]{msg}['"]""".format(msg=os.strerror(errno.EACCES)) ) self.assertIn(errno_repr, excep_repr) diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile new file mode 100644 index 000000000..b147aab4e --- /dev/null +++ b/irods/test/harness/000_install-irods.Dockerfile @@ -0,0 +1,14 @@ +FROM ubuntu:22.04 +COPY install.sh / +ARG irods_package_version +ENV IRODS_PACKAGE_VERSION "$irods_package_version" +RUN for phase in initialize install-essential-packages add-package-repo; do \ + bash /install.sh --w=$phase 0; \ + done +RUN /install.sh 4 +COPY start_postgresql_and_irods.sh manage_irods5_procs / +RUN apt install -y sudo +RUN useradd -ms/bin/bash testuser +RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers +RUN apt install -y faketime +CMD bash /start_postgresql_and_irods.sh diff --git a/irods/test/harness/001_bats-python3.Dockerfile b/irods/test/harness/001_bats-python3.Dockerfile new file mode 100644 index 000000000..b78179856 --- /dev/null +++ b/irods/test/harness/001_bats-python3.Dockerfile @@ -0,0 +1,5 @@ +FROM install-irods +RUN apt update; apt install -y python3-pip bats +RUN python3 -m pip install --upgrade pip +RUN python3 -m pip install virtualenv +RUN python3 -m virtualenv /py3 diff --git a/irods/test/harness/002_ssl-and-pam.Dockerfile b/irods/test/harness/002_ssl-and-pam.Dockerfile new file mode 100644 index 000000000..810d10e86 --- /dev/null +++ b/irods/test/harness/002_ssl-and-pam.Dockerfile @@ -0,0 +1 @@ +FROM bats-python3 diff --git a/irods/test/harness/003_compile-specific-python.Dockerfile b/irods/test/harness/003_compile-specific-python.Dockerfile new file mode 100644 index 000000000..72d2a323f --- /dev/null +++ b/irods/test/harness/003_compile-specific-python.Dockerfile @@ -0,0 +1,17 @@ +FROM ssl-and-pam +RUN apt update +RUN apt install -y wget build-essential +RUN apt install -y libssl-dev zlib1g-dev libffi-dev libncurses-dev wget build-essential +ARG python_version +RUN wget https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tar.xz +RUN touch /tmp/aaaaaaaaaa # dummmy change +RUN tar xf Python-${python_version}.tar.xz +WORKDIR /Python-${python_version} +RUN ./configure --prefix /root/python --with-ensurepip=install +RUN make -j +RUN mkdir /root/python +RUN make install +WORKDIR / +RUN /root/python/bin/python3 -m pip install virtualenv +RUN chmod a+rx /root +ENV PYTHON_VERSION=${python_version} diff --git a/irods/test/harness/README.txt b/irods/test/harness/README.txt new file mode 100644 index 000000000..7883bfdf3 --- /dev/null +++ b/irods/test/harness/README.txt @@ -0,0 +1,47 @@ +SAMPLE RUNS + +To build required images +------------------------ +Examples + + 1) ./build-docker.sh + DEFAULT: build single-node system based on latest iRODS release + + 2) IRODS_PACKAGE_VERSION=4.2.12 NO_CACHE='1' ./build-docker.sh [ ... optional in-directory dockerfiles in sequence ... ] + Build (ignoring docker cache) single-node system based on specified package version string. + +simple examples +--------------- +./docker_container_driver.sh tests/test_1.sh +./docker_container_driver.sh tests/test_2.sh + +Any script in a subdirectory of the repo (mounted at /prc within the container) can be +executed and will be able to find other scripts and source include files within the tree. +[See "experiment.sh" example below.] + +Examples of options in driver script +------------------------------------ + + 1. To start container and run test script: + C=$( ./docker_container_driver.sh -c -L -u testuser ../scripts/experiment.sh ) + + 2. To manually examine results afterward: + docker exec -it $C bash + +For both scripts, the environment variable DOCKER may be set to "podman" to run the alternative virtualizer. + +Demo / Demo hook / args +------------------------ + +$ ~/python-irodsclient/irods/test/harness$ ./docker_container_driver.sh ../demo.sh +ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[irods/test/demo.sh] +image=[ssl-and-pam] +.......-- HOOK RUNNING -- +/prc/irods/test/demo.sh running +args: +1: [arg1] +2: [arg2] +Killed: 1358fbff6eadac24f0915ffb414f0367deedc84b0c3e4de69a23bd3a8726298f +daniel@prec3431:~/python-irodsclient/irods/test/harness$ echo $? +118 + diff --git a/irods/test/harness/build-docker.sh b/irods/test/harness/build-docker.sh new file mode 100755 index 000000000..1ae6a24c0 --- /dev/null +++ b/irods/test/harness/build-docker.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# environment variables for build +# IRODS_PACKAGE_VERSION if defined is like "4.3.4" or "5.0.1". +# (but contains no '~' suffix for irods versions <= 4.2.10) +# PYTHON_VERSION is usually two dot-separated numbers: example "3.13", but could also have zero, one or three version numbers. +# (Do not specify the triple form, X.Y.Z, if that release is not known to exist - not counting alphas and release candidates) + +BASE=$(basename "$0") +DIR=$(realpath "$(dirname "$0")") +: ${DOCKER:=docker} + +if [ $# -gt 0 ]; then + IFS=$'\n' read -ra ARGS -d '' < <(realpath --relative-to "$DIR" "$@") + cd "$DIR" +else + cd "$DIR" + ARGS=([0-9]*.Dockerfile) +fi + +: ${PYTHON_VERSION:=3.13} export PYTHON_VERSION + +for dockerfile in "${ARGS[@]}"; do + image_name=${dockerfile#[0-9]*_} + image_name=${image_name%.Dockerfile} + irods_package_version_option="" + python_version_option="" + if [ "$image_name" = "install-irods" ]; then + irods_package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"} + elif [ "$image_name" = "compile-specific-python" ]; then + temp=$(./most_recent_python.sh $PYTHON_VERSION) + if [ -n "$temp" ]; then + PYTHON_VERSION="$temp" + fi + python_version_option=${PYTHON_VERSION:+"--build-arg=python_version=$PYTHON_VERSION"} + echo "************* DWM: python_version_option=[$python_version_option]" + else + package_version_option="" + fi + $DOCKER build -f $dockerfile -t $image_name . $irods_package_version_option $python_version_option \ + ${NO_CACHE+"--no-cache"} || + { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; } +done diff --git a/irods/test/harness/create_docker_images.sh b/irods/test/harness/create_docker_images.sh new file mode 100755 index 000000000..2f831a078 --- /dev/null +++ b/irods/test/harness/create_docker_images.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export IRODS_PACKAGE_VERSION=$1 +export PYTHON_VERSION=$2 + +[ -z "$1" -o -z "$2" ] && { + echo >&2 "usage $0 irods-vsn py-vsn"; exit 2; +} +shift 2 + +DIR=$(dirname "$0") + +"$DIR"/build-docker.sh $* diff --git a/irods/test/harness/docker_container_driver.sh b/irods/test/harness/docker_container_driver.sh new file mode 100755 index 000000000..749d5baa1 --- /dev/null +++ b/irods/test/harness/docker_container_driver.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash + +KILL_TEST_CONTAINER=1 +RUN_AS_USER="" +ECHO_CONTAINER="" +REMOVE_OPTION="--rm" +EXPLICIT_WORKDIR="" +VERBOSITY=0 +while [[ $1 = -* ]]; do + if [ "$1" = -V ]; then + VERBOSITY=1 + shift + fi + if [ "$1" = -c ]; then + ECHO_CONTAINER=1 + shift + fi + if [ "$1" = -L ]; then + KILL_TEST_CONTAINER=0 + shift + fi + if [ "$1" = -u ]; then + RUN_AS_USER="$2" + shift 2 + fi + if [ "$1" = -r ]; then + REMOVE_OPTION="$2" + shift 2 + fi + if [ "$1" = -w ]; then + EXPLICIT_WORKDIR="$2" + shift 2 + fi +done + +if [ "$1" = "" ]; then + echo >&2 "Usage: $0 [options] /path/to/script" + echo >&2 "With options: [-L] to leak, [-u username] to run as non-root user" + exit 1 +fi + +DIR=$(dirname $0) +. "$DIR"/test_script_parameters + +testscript=${1} +shift + +testscript_basename=$(basename "$testscript") +arglist=${wrapper_arglist[$testscript_basename]:-$*} # arglist dominated by symbolic link name if any + +if [ -L "$testscript" ]; then + testscript=$(realpath "$testscript") + testscript_basename=$(basename "$testscript") +fi + +original_testscript_abspath=$(realpath "$testscript") + +wrapped=${wrappers["$testscript_basename"]} + +if [ -n "$wrapped" ]; then + # wrapped is assumed to contain a leading path element relative to the referencing script's containing directory + testscript="$(dirname "$testscript")/$wrapped" + testscript_basename=$(basename "$testscript") +fi + +testscript_abspath=$(realpath "$testscript") + +cd "$DIR" + +image=${images[$testscript_basename]} + +if [ -z "$RUN_AS_USER" ]; then + RUN_AS_USER=${user[$testscript_basename]} +fi + +# Tests are run as testuser by default +: ${RUN_AS_USER:='testuser'} + +WORKDIR="" +if [ -n "$EXPLICIT_WORKDIR" ]; then + WORKDIR="$EXPLICIT_WORKDIR" +else + WORKDIR=${workdirs[$RUN_AS_USER]} +fi + +reporoot=$(./print_repo_root_location) +ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to $reporoot "$original_testscript_abspath") + +echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" +INNER_MOUNT=/prc + +: ${DOCKER:=docker} + +# Start the container. +echo image="[$image]" +CONTAINER=$($DOCKER run -d -v $reporoot:$INNER_MOUNT:ro $REMOVE_OPTION $image) + +# Wait for iRODS and database to start up. +TIME0=$(date +%s) +while :; do + [ `date +%s` -gt $((TIME0 + 30)) ] && { echo >&2 "Waited too long for DB and iRODS to start"; exit 124; } + sleep 1 + $DOCKER exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null + [ $? -ne 0 ] && { echo -n . >&2; continue; } + break +done + +if [ $VERBOSITY -gt 0 ]; then + echo $'\n'"==> Running script [$testscript_abspath]" + echo "in container [$CONTAINER]" + echo "with these *_VERSION variables in environment: " + $DOCKER exec $CONTAINER bash -c 'env|grep _VERSION' | sed $'s/^/\t/' +fi + +$DOCKER exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \ + ${WORKDIR:+"-w$WORKDIR"} \ + -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \ + $CONTAINER \ + $INNER_MOUNT/$(realpath --relative-to $reporoot "$testscript_abspath") \ + $arglist +STATUS=$? + +if [ $((0+KILL_TEST_CONTAINER)) -ne 0 ]; then + echo >&2 'Killed:' $($DOCKER stop --timeout=0 $CONTAINER) +fi + +[ -n "$ECHO_CONTAINER" ] && echo $CONTAINER +exit $STATUS diff --git a/irods/test/harness/install.sh b/irods/test/harness/install.sh new file mode 100755 index 000000000..cc8c23ca9 --- /dev/null +++ b/irods/test/harness/install.sh @@ -0,0 +1,155 @@ +#!/bin/bash + +IRODS_HOME=/var/lib/irods +DEV_HOME="$HOME" +: ${DEV_REPOS:="$DEV_HOME/github"} + +add_package_repo() +{ + local R="/etc/apt/sources.list.d/renci-irods.list" + echo >&2 "... installing package repo" + sudo apt update + sudo apt install -y lsb-release apt-transport-https + wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add - && \ + echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\ + sudo tee "$R" + sudo apt update +} + +DIST_NAME=$(grep '^VERSION_CODENAME=' /etc/os-release|sed 's/.*=//') + +# Expand a spec of the leading version tuple eg. 4.3.4 out to the full name of +# the most recent matching version of the package + +# Report the latest version spec (including OS) that matches the env var IRODS_PACKAGE_VERSION (eg. "5.0.2" -> "5.0.2-0~jammy) + +irods_package_vsn() { + apt list -a irods-server 2>/dev/null|awk '{print $2}'|grep '\w'|sort|\ + grep "$(perl -e 'print quotemeta($ARGV[0])' "$IRODS_PACKAGE_VERSION")"|tail -1 +} + +# Report the version number of the installed iRODS server if any. + +irods_vsn() { + local V=$(dpkg -l irods-server 2>/dev/null|grep '^ii\s'|awk '{print $3}') + echo "${V}" +} + +while [[ "$1" = -* ]]; do + ARG="$1" + shift + case $ARG in + --i=* | --irods=* |\ + --irods-version=*) IRODS_PACKAGE_VERSION=${ARG#*=};; + --w=* | --with=* | --with-options=* ) withopts=${ARG#*=} ;; + -v) VERBOSE=1;; + esac +done + + +run_phase() { + + local PHASE=$1 + local with_opts=" $2 " + + case "$PHASE" in + + 0) + + if [[ $with_opts = *\ initialize\ * ]]; then + apt-get -y update + apt-get install -y apt-transport-https wget lsb-release sudo jq + fi + + if [[ $with_opts = *\ sudo-without-pw\ * ]]; then + if [ `id -u` = 0 -a "${USER:-root}" = root ] ; then + echo >&2 "root authorization for 'sudo' is automatic - no /etc/sudoers modification needed" + else + if [ -f "/etc/sudoers" ]; then + if [ -n "$USER" ] ; then + # add a line with our USER name to /etc/sudoers if not already there + sudo su -c "sed -n '/^\s*[^#]/p' /etc/sudoers | grep '^$USER\s*ALL=(ALL)\s*NOPASSWD:\s*ALL\s*$' >/dev/null" || \ + sudo su -c "echo '$USER ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers" + else + echo >&2 "user login is '$USER' - can this be right?" + fi + else + echo >&2 "WARNING - Could not modify sudoers files" + echo -n >&2 " (hit 'Enter' to continue)" + read key + fi + fi # not root + fi # with-opts + + #------ (needed for both package install and build from source) + + if [[ $with_opts = *\ install-essential-packages\ * ]]; then + + if ! dpkg -l tzdata >/dev/null 2>&1 ; then + sudo su - root -c \ + "env DEBIAN_FRONTEND=noninteractive bash -c 'apt-get install -y tzdata'" + fi + sudo apt-get update + sudo apt-get install -y software-properties-common postgresql + sudo apt-get update && \ + sudo apt-get install -y libfuse2 unixodbc rsyslog + fi + + + if [[ $with_opts = *\ add-package-repo\ * ]]; then + add_package_repo -f + fi + + + if [[ $with_opts = *\ create-db\ * ]]; then + sudo su - postgres -c " + { dropdb --if-exists ICAT + dropuser --if-exists irods ; } >/dev/null 2>&1" + sudo su - postgres -c "psql <<\\ +________ + CREATE DATABASE \"ICAT\"; + CREATE USER irods WITH PASSWORD 'testpassword'; + GRANT ALL PRIVILEGES ON DATABASE \"ICAT\" to irods; +________" + echo >&2 "-- status of create-db = $? -- " + fi + ;; + + 4) + IRODS_TO_INSTALL=`irods_package_vsn` + sudo apt install -y irods-{dev,runtime}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + if [[ $with_opts != *\ basic\ * ]]; then + sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"} + fi + ;; + + 5) + if [ ! `irods_vsn` '<' "4.3" ]; then + PYTHON=python3 + else + PYTHON=python2 + fi + sudo $PYTHON /var/lib/irods/scripts/setup_irods.py < /var/lib/irods/packaging/localhost_setup_postgres.input + ;; + + *) echo >&2 "unrecognized phase: '$PHASE'." ; QUIT=1 ;; + esac + return $? +} + +#-------------------------- main + +QUIT=0 +while [ $# -gt 0 ] ; do + ARG=$1 ; shift + NOP="" ; run_phase $ARG " $withopts "; sts=$? + [ $QUIT != 0 ] && break + [ -n "$NOP" ] && continue + echo -n "== $ARG == " + if [ $sts -eq 0 ]; then + echo Y >&2 + else + [ $quit_on_phase_err ] && { echo >&2 "N - quitting"; exit 1; } + echo N >&2 + fi +done diff --git a/irods/test/harness/install_python_rule_engine b/irods/test/harness/install_python_rule_engine new file mode 100755 index 000000000..036273fa1 --- /dev/null +++ b/irods/test/harness/install_python_rule_engine @@ -0,0 +1,23 @@ +#!/bin/bash + +python_rule_plugin_package_spec() { + local PKG=irods-rule-engine-plugin-python + local search_str="+$IRODS_PACKAGE_VERSION" + if [ "$1" = "all" ]; then + search_str="" + fi + local VERSIONS=$(apt list -a $PKG 2>/dev/null|\ + awk '{print $2}'|\ + grep '\w'|\ + grep "$( + perl -e 'print quotemeta($ARGV[0])' "$search_str")" + ) + local LATEST_VERSION=$(sort -V <<<"$VERSIONS" | tail -1) + if [ "$1" = "latest" ]; then + echo "$PKG=$LATEST_VERSION" + else + echo "$VERSIONS" + fi +} + +apt install -y "$(python_rule_plugin_package_spec latest)" diff --git a/irods/test/harness/irods_version_greater_or_equal_to b/irods/test/harness/irods_version_greater_or_equal_to new file mode 100755 index 000000000..a5375b144 --- /dev/null +++ b/irods/test/harness/irods_version_greater_or_equal_to @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +import ast +import getopt +import glob +import json +import os +import sys + +def version_to_tuple(dot_str): + return tuple(int(_) for _ in dot_str.split('.')) + +# Fail unless iRODS version is greater than or equal to a version string given on the command line. +# With -e, we determine the installed iRODS version from the specified environment variable. +# Else use ~irods/version.json + +if __name__ == '__main__': + opts,arg = getopt.getopt (sys.argv[1:],'e:',['use_env_var=']) + + env_var = '' + + for opt,val in opts: + if opt in {'-e','--use_env_var'}: + env_var = val + + pattern = os.path.join(os.path.expanduser('~irods'), "*.json") + + # get full path of version.json + + if env_var: + version_to_test = os.environ[env_var] + else: + version_files = list( + filter( + (lambda name: name.lower().endswith('version.json')), + glob.glob(pattern) + ) + ) + # Load JSON struct containing iRODS version. + j = json.load(open(version_files[0])) + version_to_test = j["irods_version"] + + if version_to_tuple(version_to_test) < version_to_tuple(arg[0]): + exit(1) diff --git a/irods/test/harness/manage_irods5_procs b/irods/test/harness/manage_irods5_procs new file mode 100755 index 000000000..3327cb9be --- /dev/null +++ b/irods/test/harness/manage_irods5_procs @@ -0,0 +1,57 @@ +if [ `id -un` = irods ]; then + LAUNCH='bash -c' +else + LAUNCH='sudo su - irods -c' +fi + +STDOUT="" +PID="" + +start() { + if [ -z "$STDOUT" ] ; then + $LAUNCH 'irodsServer -d -p /tmp/irods.pid' + else + $LAUNCH 'irodsServer --stdout -p /tmp/irods.pid >/tmp/irods.log &' + fi +} + +rm_pid_file() { + if [ -z "$PID" ]; then + PID=$($LAUNCH 'cat /tmp/irods.pid') + fi + $LAUNCH 'rm -f /tmp/irods.pid >/dev/null 2>&1' +} + +stop() { + $LAUNCH 'kill -QUIT $(cat /tmp/irods.pid)' + rm_pid_file +} + +wait() { + $LAUNCH " + [ -z '$PID' ] && { echo >&2 'nothing to wait for.' ; exit 2; } + while ps -eo pid |grep $PID >/dev/null 2>&1; do sleep 1; done;" +} + +# ----------------------------------- +while [ -n "$1" ]; do + if [ "$1" = "stdout" ]; then + STDOUT=1 + elif [ "$1" = "start" ]; then + start + elif [ "$1" = "rescan-config" ]; then + $LAUNCH 'pkill -HUP irodsServer' + elif [ "$1" = "status" ]; then + pgrep -afl "irods(Delay|Agent|Server)" + elif [ "$1" = "stop" ]; then + stop + elif [ "$1" = "restart" ]; then + stop && start + elif [ "$1" = "wait" ]; then + wait + else + echo >&2 "usage: $0 [start|status|stop]" + exit 2 + fi + shift +done diff --git a/irods/test/harness/most_recent_python.sh b/irods/test/harness/most_recent_python.sh new file mode 100755 index 000000000..278f3ce7c --- /dev/null +++ b/irods/test/harness/most_recent_python.sh @@ -0,0 +1,25 @@ +#!/bin/bash +usage() { + echo >&2 "Usage: + $0 major.minor" + echo >&2 "Output: + prints full latest python version inclusive of the patch level." + exit 2 +} +MAJOR_MINOR=$1 +if [ -z "${MAJOR_MINOR}" ]; then # allow blank specification: most recent overall + MAJOR_MINOR='[0-9]\+\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ ^[0-9]+$ ]]; then # allow single integer, eg. 3 for most recent 3.y.z + MAJOR_MINOR+='\.[0-9]\+' +elif [[ $MAJOR_MINOR =~ [0-9]+\.[0-9]+ ]]; then # allow x.y form, will yield output of most recent x.y.z + MAJOR_MINOR=$(sed 's/\./\\./'<<<"${MAJOR_MINOR}") # insert backslash in front of "." +elif ! [[ $MAJOR_MINOR =~ [0-9]+\\?.[0-9]+ ]]; then + usage +fi + +url='https://www.python.org/ftp/python/' + +# Fetch the directory listing, extract version numbers, sort them to find the largest numerically. +curl --silent "$url"|\ +sed -n 's!.*href="\('"${MAJOR_MINOR}"'\.[0-9]\+\)/".*!\1!p'|sort -rV|\ +head -n 1 diff --git a/irods/test/harness/print_repo_root_location b/irods/test/harness/print_repo_root_location new file mode 100755 index 000000000..adcf3fc73 --- /dev/null +++ b/irods/test/harness/print_repo_root_location @@ -0,0 +1,5 @@ +#!/bin/bash +# The following line needs be kept updated to reflect true position relative to repository root, +# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved. +REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../../.. +realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT" diff --git a/irods/test/harness/setup_python_rule_engine b/irods/test/harness/setup_python_rule_engine new file mode 100755 index 000000000..6795f04e5 --- /dev/null +++ b/irods/test/harness/setup_python_rule_engine @@ -0,0 +1,92 @@ +#!/bin/bash + +# This script should be run as the service account user. + +wait="" +if [ $1 = --wait ]; then + wait=1 + shift +fi + +DIR=$(dirname "$0") + +server_ctl() { + # This script takes one argument. + # Valid arguments: start, stop, or restart. The appropriate action is then taken for the resident iRODS server. + if "$DIR"/irods_version_greater_or_equal_to --use_env_var=IRODS_PACKAGE_VERSION 5.0; then + # Make our ps-based script wait for process shutdown like 'irodsctl stop' does. + W="" + if [ "$1" = stop ]; then + W=wait + fi + /manage_irods5_procs $1 $W + else + ~/irodsctl $1 + fi +} + +jq_process_in_place() { + local filename=$1 + shift + local basenm=$(basename "$filename") + local tempname=/tmp/.$$.$basenm + + jq "$@" <"$filename" >"$tempname" && \ + cp "$tempname" "$filename" + STATUS=$? + rm -f "$tempname" + [ $STATUS = 0 ] || echo "**** jq process error" >&2 +} + +# -- Main part of script -- + +server_ctl stop + +jq_process_in_place /etc/irods/server_config.json \ + '.plugin_configuration.rule_engines[1:1]=[ { "instance_name": "irods_rule_engine_plugin-python-instance", + "plugin_name": "irods_rule_engine_plugin-python", + "plugin_specific_configuration": {} + } + ]' +echo ' +defined_in_both { + writeLine("stdout", "native rule") +} + +generic_failing_rule { + fail +} + +failing_with_message { + failmsg(-2, "error with code of minus 2") +} + +' >> /etc/irods/core.re + +echo ' +def defined_in_both(rule_args,callback,rei): + callback.writeLine("stdout", "python rule") + +def generic_failing_rule(*_): + raise RuntimeError + +def failing_with_message_py(rule_args,callback,rei): + callback.failing_with_message() + +' > /etc/irods/core.py + +server_ctl start + +# Wait until 'irule -a' shows Python Rule Engine Plugin among the choices +if [ -n "$wait" ]; then + times=0 + OUTFILE=/tmp/irule_output.stderr + while :; do + irule -a 2>/dev/null| grep irods_rule_engine_plugin-python-instance >/dev/null + [ ${PIPESTATUS[1]} -eq 0 ] && break + sleep 1 + if [ $((++times)) -ge 10 ]; then + echo >&2 "Failed to configure Python rule engine."; exit 2; + fi + done +fi diff --git a/irods/test/harness/start_postgresql_and_irods.sh b/irods/test/harness/start_postgresql_and_irods.sh new file mode 100755 index 000000000..0138aad47 --- /dev/null +++ b/irods/test/harness/start_postgresql_and_irods.sh @@ -0,0 +1,34 @@ +#!/bin/bash +service postgresql start +x=${DB_WAIT_SEC:-20} +while [ $x -ge 0 ] && { ! $SUDO su - postgres -c "psql -c '\l' >/dev/null 2>&1" || x=""; } +do + [ -z "$x" ] && break + echo >&2 "$((x--)) secs til database timeout"; sleep 1 +done +[ -z "$x" ] || { echo >&2 "Error -- database didn't start" ; exit 1; } +VERSION_file=$(ls /var/lib/irods/{VERSION,version}.json.dist 2>/dev/null) +if ! id -u irods >/dev/null 2>&1 ; then + /install.sh --w=create-db 0 + /install.sh 5 +fi +IRODS_VSN=$(jq -r '.irods_version' $VERSION_file) +IRODS_VSN_MAJOR=$(sed 's!\..*!!'<<<$IRODS_VSN) +if [ $IRODS_VSN_MAJOR -lt 5 ]; then + su - irods -c '~/irodsctl restart' +else + /manage_irods5_procs start +fi +IRODS_WAIT_SEC=20 +x=$IRODS_WAIT_SEC +SLEEPTIME="" +while [ $((x--)) -gt 0 ]; do + sleep $((SLEEPTIME+0)) + pgrep irodsServer + STATUS=$? + [ $STATUS -eq 0 ] && break + SLEEPTIME=1 +done +echo "($STATUS)" >/tmp/irods_status +[ $STATUS -eq 0 ] || exit 125 +tail -f /dev/null diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters new file mode 100644 index 000000000..2ee86f10e --- /dev/null +++ b/irods/test/harness/test_script_parameters @@ -0,0 +1,53 @@ +# keys for Arglist refer to argument given, which could be a symlink. + +declare -A wrapper_arglist=( + [demo.sh]="arg1 arg2" + [demo_A.sh]="arg1-a arg2-a" + [login_auth_test_must_run_manually.py]="-v TestLogins" + [login_auth_test_1.py]="-v TestAnonymousUser TestMiscellaneous" + [login_auth_test_2.py]="-v TestWithSSL" +) + +# keys for Wrapper refer to argument after resolution of any symlinks + +declare -A wrappers=( + [login_auth_test_must_run_manually.py]=./login_auth_test.sh + [PRC_issue_362.bats]=./login_auth_test.sh + [demo.sh]=./demo_hook.sh + [test001_pam_password_expiration.bats]=../login_auth_test.sh + [test002_write_native_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test003_write_pam_credentials_to_secrets_file.bats]=../login_auth_test.sh + [test004_prc_pam_password_internal_secrets_file_generation.bats]=../login_auth_test.sh + [test005_test_special_characters_in_pam_passwords.bats]=../login_auth_test.sh + [test006_connection_timeout_on_ssl_socket.bats]=../login_auth_test.sh + [test007_pam_features_in_new_auth_framework.bats]=../login_auth_test.sh + [test008_prc_write_irodsA_utility_in_native_mode.bats]=../login_auth_test.sh + [test009_test_special_characters_in_pam_passwords_auth_framework.bats]=../login_auth_test.sh +) + +# keys for Image and User refer to the basename after resolution to a wrapper if one is used + +declare -A images=( + [login_auth_test.sh]=compile-specific-python + [test_1.sh]=install-irods + [test_2.sh]=bats-python3 + [test_3.bats]=bats-python3 + [experiment.sh]=ssl-and-pam + [fail.sh]=ssl-and-pam + [login_auth_test_must_run_manually.py]=ssl-and-pam + [demo_hook.sh]=ssl-and-pam + [run_suite_locally.sh]=compile-specific-python + [test000.sh]=compile-specific-python +) + +declare -A user=( + [run_suite_locally.sh]=root +) + +# keys for WorkDir refer to user + +declare -A workdirs=( + [testuser]=/home/testuser + [irods]=/var/lib/irods + [root]=/ +) diff --git a/irods/test/harness/tests/test_1.sh b/irods/test/harness/tests/test_1.sh new file mode 100755 index 000000000..798302f15 --- /dev/null +++ b/irods/test/harness/tests/test_1.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +run() { + echo dir of this = $(realpath "$(dirname "${BASH_SOURCE[0]}")/repo") +} + +echo hello_there diff --git a/irods/test/harness/tests/test_2.sh b/irods/test/harness/tests/test_2.sh new file mode 100755 index 000000000..f5b9279f4 --- /dev/null +++ b/irods/test/harness/tests/test_2.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +run() { + echo dir of this = $(realpath "$(dirname "${BASH_SOURCE[0]}")/repo") +} + +echo later_alligator diff --git a/irods/test/harness/tests/test_3.bats b/irods/test/harness/tests/test_3.bats new file mode 100755 index 000000000..18be2ad07 --- /dev/null +++ b/irods/test/harness/tests/test_3.bats @@ -0,0 +1,21 @@ +#!/usr/bin/env bats + +. "$BATS_TEST_DIRNAME"/../setup_pam_and_ssl.funcs + +setup() { + echo setup >>/tmp/log + setup_preconnect_preference DONT_CARE + python3 "$BATS_TEST_DIRNAME"/repo/irods/test/setupssl.py +: +} + +teardown() { + echo teardown >>/tmp/log +: +} + +@test mytest { + grep acPreConn /etc/irods/core.re >>/tmp/log + echo test proper >>/tmp/log +: +} diff --git a/irods/test/login_auth_test.sh b/irods/test/login_auth_test.sh new file mode 100755 index 000000000..b6bec0161 --- /dev/null +++ b/irods/test/login_auth_test.sh @@ -0,0 +1,74 @@ +#!/bin/bash +. $(dirname $0)/scripts/test_support_functions +. $(dirname $0)/scripts/update_json_for_test + +IRODS_SERVER_CONFIG=/etc/irods/server_config.json +IRODS_SERVICE_ACCOUNT_ENV_FILE=~irods/.irods/irods_environment.json +LOCAL_ACCOUNT_ENV_FILE=~/.irods/irods_environment.json + +cannot_iinit='' +tries=8 +while true; do + iinit_as_rods >/dev/null 2>&1 && break + [ $((--tries)) -le 0 ] && { cannot_iinit=1; break; } + sleep 5 +done +[ -n "$cannot_iinit" ] && { echo >&2 "Could not iinit as rods."; exit 2; } + +setup_preconnect_preference DONT_CARE + +add_irods_to_system_pam_configuration + + +# set up /etc/irods/ssl directory and files +set_up_ssl sudo + +sudo useradd -ms/bin/bash alissa +sudo chpasswd <<<"alissa:test123" + +update_json_file $IRODS_SERVICE_ACCOUNT_ENV_FILE \ + "$(newcontent $IRODS_SERVICE_ACCOUNT_ENV_FILE ssl_keys)" + +# This is mostly so we can call python3 as just "python" +activate_virtual_env_with_prc_installed >/dev/null 2>&1 || { echo >&2 "couldn't set up virtual environment"; exit 1; } + +server_hup= +if irods_server_version ge 5.0.0; then + server_hup="y" + update_json_file $IRODS_SERVER_CONFIG \ + "$(newcontent $IRODS_SERVER_CONFIG tls_server_items tls_client_items)" + + sudo su - irods -c "/manage_irods5_procs rescan-config" +fi + +# Configure clients with admin user + TLS + +update_json_file $LOCAL_ACCOUNT_ENV_FILE \ + "$(newcontent $LOCAL_ACCOUNT_ENV_FILE ssl_keys encrypt_keys)" + +if [ $server_hup = y ]; then + # wait for server to be ready after configuration reload + while true; do + sleep 2 + if ils >/dev/null 2>&1; then + break + else + # Allow ~16 secs of total wait time. + [ $((++server_check)) -gt 8 ] && { + echo >&2 "Timed out on server reload"; exit 3; } + fi + done +fi + +original_script=/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT + +# Run tests. +if [ -x "$original_script" ]; then + command "$original_script" $* +elif [[ $original_script =~ \.py$ ]]; then + python "$original_script" $* +elif [[ $original_script =~ \.bats$ ]]; then + bats "$original_script" +else + echo >&2 "I don't know how to run this: original_script=[$original_script]" +fi diff --git a/irods/test/login_auth_test_1.py b/irods/test/login_auth_test_1.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_1.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_2.py b/irods/test/login_auth_test_2.py new file mode 120000 index 000000000..23402ed84 --- /dev/null +++ b/irods/test/login_auth_test_2.py @@ -0,0 +1 @@ +login_auth_test_must_run_manually.py \ No newline at end of file diff --git a/irods/test/login_auth_test_must_run_manually.py b/irods/test/login_auth_test_must_run_manually.py index 4303e47c1..745eaf53e 100644 --- a/irods/test/login_auth_test_must_run_manually.py +++ b/irods/test/login_auth_test_must_run_manually.py @@ -534,7 +534,7 @@ def test_nonanonymous_login_without_auth_file_fails__290(self): s.users.get("bob") os.unlink(bob_auth) # -- Check that we raise an appropriate exception pointing to the missing auth file path -- - with self.assertRaisesRegexp(NonAnonymousLoginWithoutPassword, bob_auth): + with self.assertRaisesRegex(NonAnonymousLoginWithoutPassword, bob_auth): with helpers.make_session(**login_options) as s: s.users.get("bob") finally: diff --git a/irods/test/meta_test.py b/irods/test/meta_test.py index aa33dae26..1a0d01bf4 100644 --- a/irods/test/meta_test.py +++ b/irods/test/meta_test.py @@ -683,16 +683,16 @@ def test_AVUs_populated_improperly_with_empties_or_nonstrings_fail_identically__ def test_nonstring_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", 0) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "incorrect type"): + with self.assertRaisesRegex(Bad_AVU_Field, "incorrect type"): self.coll.metadata.add(*args) def test_empty_string_as_AVU_value_raises_an_error__issue_434(self): args = ("an_attribute", "") - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.set(*args) - with self.assertRaisesRegexp(Bad_AVU_Field, "zero-length"): + with self.assertRaisesRegex(Bad_AVU_Field, "zero-length"): self.coll.metadata.add(*args) @unittest.skipUnless( @@ -724,10 +724,11 @@ def test_that_all_column_mappings_are_uniquely_and_properly_defined__issue_643( prepend_col_prefix_if_needed = lambda s: ( "COL_" + s if not s.startswith("COL_") else s ) + current_server_version = self.sess.server_version prc_column_defs = sorted( [ (prepend_col_prefix_if_needed(i[1].icat_key), i[1].icat_id) - for i in ModelBase.column_items + for i in ModelBase.column_items if current_server_version >= i[1].min_version ] ) diff --git a/irods/test/pam.bats/funcs b/irods/test/pam.bats/funcs deleted file mode 100644 index 30539a03a..000000000 --- a/irods/test/pam.bats/funcs +++ /dev/null @@ -1,108 +0,0 @@ -dot_to_space() { - sed 's/\./ /g'<<<"$1" -} - -CLEANUP=$':\n' - -GT() { (return 1); echo $?; } -LT() { (return -1); echo $?; } -EQ() { (return 0); echo $?; } - -compare_int_tuple() { - local x=($1) y=($2) - local lx=${#x[@]} ly=${#y[@]} - local i maxlen=$((lx > ly ? lx : ly)) - for ((i=0;i ~/.irods/irods_environment.json - iinit <<<"$1" 2>/dev/tty -} - -_end_pam_environment_and_password() { - rm -fr ~/.irods - mv ~/.irods.$$ ~/.irods -} - -setup_pam_login_for_alice() { - sudo useradd alice --create-home - local PASSWD=${1:-test123} - sudo chpasswd <<<"alice:$PASSWD" - iadmin mkuser alice rodsuser - _begin_pam_environment_and_password "$PASSWD" -} - -finalize_pam_login_for_alice() { - _end_pam_environment_and_password - iadmin rmuser alice - sudo userdel alice --remove -} - -test_specific_cleanup() { - eval "$CLEANUP" -} diff --git a/irods/test/pam.bats/test001_pam_password_expiration.bats b/irods/test/pam.bats/test001_pam_password_expiration.bats deleted file mode 100644 index 3e29100ef..000000000 --- a/irods/test/pam.bats/test001_pam_password_expiration.bats +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bats - -. "$BATS_TEST_DIRNAME"/test_support_functions -PYTHON=python3 - -# Setup/prerequisites are same as for login_auth_test. -# Run as ubuntu user with sudo; python_irodsclient must be installed (in either ~/.local or a virtualenv) -# - -PASSWD=test123 - -setup() -{ - setup_pam_login_for_alice $PASSWD -} - -teardown() -{ - finalize_pam_login_for_alice - test_specific_cleanup -} - -@test f001 { - - # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS - # without an exception being raised. - - local SCRIPT=" -import irods.test.helpers as h -ses = h.make_session() -ses.collections.get(h.home_collection(ses)) -print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) -" - - # Test that the first run of the code in $SCRIPT is successful, i.e. normal authenticated operations are possible. - - local OUTPUT=$($PYTHON -c "$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - SET_CLEANUP=yes \ - with_change_auth_params_for_test password_min_time 4 \ - password_max_time 5 - - # Test that running the $SCRIPT raises an exception if the PAM password has expired. - - iinit <<<"$PASSWD" - HOME_COLLECTION=$(ipwd) - sleep 9 - OUTPUT=$($PYTHON -c "$SCRIPT" 2>&1 >/dev/null || true) - grep 'RuntimeError: Time To Live' <<<"$OUTPUT" - - # Test that the $SCRIPT, when run with proper settings, can successfully reset the password. - - with_change_auth_params_for_test password_max_time 3600 - - OUTPUT=$($PYTHON -c "import irods.client_configuration as cfg -cfg.legacy_auth.pam.password_for_auto_renew = '$PASSWD' -cfg.legacy_auth.pam.time_to_live_in_hours = 1 -cfg.legacy_auth.pam.store_password_to_environment = True -$SCRIPT") - - [[ $OUTPUT =~ ^env_auth_scheme=pam_password$ ]] - - # Test that iCommands can authenticate with the newly written .irodsA file - - iquest "%s" "select COLL_NAME where COLL_NAME like '%/home/alice%'"| grep "^$HOME_COLLECTION\$" -} diff --git a/irods/test/pam_interactive_test.py b/irods/test/pam_interactive_test_must_run_manually.py similarity index 100% rename from irods/test/pam_interactive_test.py rename to irods/test/pam_interactive_test_must_run_manually.py diff --git a/irods/test/rule_test.py b/irods/test/rule_test.py index be95302b7..39b718a26 100644 --- a/irods/test/rule_test.py +++ b/irods/test/rule_test.py @@ -20,7 +20,7 @@ RE_Plugins_installed_run_condition_args = ( os.environ.get("PYTHON_RULE_ENGINE_INSTALLED", "*").lower()[:1] == "y", - "Test depends on server having Python-REP installed beyond the default options", + "Test depends on server having Python-REP installed (set PYTHON_RULE_ENGINE_INSTALLED=yes in environment)." ) @@ -420,7 +420,7 @@ def test_rulefile_in_file_like_object_1__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r".*\[Hello world!\]") + self.assertRegex(lines[0], r".*\[Hello world!\]") def test_rulefile_in_file_like_object_2__336(self): @@ -442,8 +442,8 @@ def test_rulefile_in_file_like_object_2__336(self): r = Rule(self.sess, rule_file=io.BytesIO(rule_file_contents.encode("utf-8"))) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[STRING\]\[\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[\]") + self.assertRegex(lines[0], r"\[STRING\]\[\]") + self.assertRegex(lines[1], r"\[STRING\]\[\]") r = Rule( self.sess, @@ -452,8 +452,8 @@ def test_rulefile_in_file_like_object_2__336(self): ) output = r.execute() lines = self.lines_from_stdout_buf(output) - self.assertRegexpMatches(lines[0], r"\[INTEGER\]\[5\]") - self.assertRegexpMatches(lines[1], r"\[STRING\]\[A String\]") + self.assertRegex(lines[0], r"\[INTEGER\]\[5\]") + self.assertRegex(lines[1], r"\[STRING\]\[A String\]") if __name__ == "__main__": diff --git a/irods/test/runner.py b/irods/test/runner.py index f9f9fa610..8782d6391 100644 --- a/irods/test/runner.py +++ b/irods/test/runner.py @@ -7,6 +7,7 @@ """ +import argparse import os import sys from unittest import TestLoader, TestSuite @@ -22,20 +23,74 @@ h.setFormatter(f) logger.addHandler(h) +parser = argparse.ArgumentParser() -# Load all tests in the current directory and run them +def abs_path(initial_dir, levels_up = 0): + directory = initial_dir + while levels_up > 0: + levels_up -= 1 + directory = os.path.join(directory,'..') + return os.path.abspath(directory) + +# Load all tests in the current directory and run them. if __name__ == "__main__": - # must set the path for the imported tests - sys.path.insert(0, os.path.abspath("../..")) + + # Get path to script directory for test import and/or discovery. + script_dir = os.path.abspath(os.path.dirname(sys.argv[0])) + + # Must set the path for the imported tests. + sys.path.insert(0, abs_path(script_dir, levels_up = 2)) + + parser.add_argument('--tests', '-t', + metavar='TESTS', + dest='tests', + nargs='+', + help='List of tests to run.') + + parser.add_argument('--environment_variable', '-e', + metavar='ENVIRONMENT_VARIABLE', + dest='env_var', + type=str, + help='Name of environment variable name to scan for in reason strings when filtering skipped test names to be output.') + + parser.add_argument('--output_tests_skipped', '-s', + metavar='SKIPPED_TESTS_OUTPUT_FILENAME', + dest='skipped_tests_output_filename', + type=str, + help='Name of a file into which to write names of skipped tests.') + + parser.add_argument('--tests_file', '-f', + metavar='TESTS_FILE', + dest='tests_file', + help='Name of a file containing a list of tests to run.') + + args = parser.parse_args() + + if args.tests_file: + if args.tests: + print ('Cannot specify both --tests and --tests_file', file = sys.stderr) + exit(2) + args.tests = filter(None,open(args.tests_file).read().split("\n")) loader = TestLoader() - suite = TestSuite( - loader.discover(start_dir=".", pattern="*_test.py", top_level_dir=".") - ) + + if args.tests: + suite = TestSuite(loader.loadTestsFromNames(args.tests)) + else: + suite = TestSuite(loader.discover(start_dir = script_dir, pattern = '*_test.py', top_level_dir = script_dir)) result = xmlrunner.XMLTestRunner( verbosity=2, output="/tmp/python-irodsclient/test-reports" ).run(suite) + + if args.skipped_tests_output_filename: + with open(args.skipped_tests_output_filename,'w') as skip_file: + do_output = (lambda reason: (args.env_var in reason) if args.env_var + else True) + for testinfo, reason in result.skipped: + if do_output(reason): + print(testinfo.test_id, file=skip_file) + if result.wasSuccessful(): sys.exit(0) diff --git a/irods/test/scripts/demo_script b/irods/test/scripts/demo_script new file mode 100755 index 000000000..7202d9cf3 --- /dev/null +++ b/irods/test/scripts/demo_script @@ -0,0 +1,2 @@ +#!/bin/bash +echo "hello from script [$0]" diff --git a/irods/test/scripts/experiment.sh b/irods/test/scripts/experiment.sh new file mode 100755 index 000000000..89bdd086e --- /dev/null +++ b/irods/test/scripts/experiment.sh @@ -0,0 +1,6 @@ +#!/bin/bash +DIR=$(dirname $0) +. $DIR/test_support_functions +cd "$DIR" +set_up_ssl sudo +add_irods_to_system_pam_configuration diff --git a/irods/test/scripts/fail.sh b/irods/test/scripts/fail.sh new file mode 100755 index 000000000..e84d70e10 --- /dev/null +++ b/irods/test/scripts/fail.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +DIR=$(dirname $0) +. $DIR/test_support_functions +cd "$DIR"; echo >&2 -n -- ; pwd +#echo "setting up" +$(up_from_script_dir 1)/demo_script +#set_up_ssl sudo +id -un +exit 12 diff --git a/irods/test/scripts/run_suite_locally.sh b/irods/test/scripts/run_suite_locally.sh new file mode 100755 index 000000000..79f02bdf8 --- /dev/null +++ b/irods/test/scripts/run_suite_locally.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -e + +SCRIPT_DIR=$(dirname "$0") +. "$SCRIPT_DIR"/test_support_functions + +run_tests() { + setup_pyN + su - testuser -c " + set -e + source /pyN/bin/activate + pip install -e /prc.rw[tests] + cd /prc.rw/irods/test + python /prc.rw/docker-testing/iinit.py \ + host localhost \ + port 1247 \ + user rods \ + zone tempZone \ + password rods + echo ; echo 'PRC under test: === iRODS [$IRODS_PACKAGE_VERSION] ; Python [$PYTHON_VERSION]' + python runner.py --output_tests_skipped /tmp/skipped.txt -e PYTHON_RULE_ENGINE_INSTALLED --tests irods.test.rule_test + " + + # Install PREP (Python Rule Engine Plugin). + ( + set -e + cd "$SCRIPT_DIR/../harness" + apt update + ./install_python_rule_engine + su irods -c './setup_python_rule_engine --wait' + ) + + # Run PREP-dependent tests that were previously skipped. + su - testuser -c " + set -e + source /pyN/bin/activate + cd /prc.rw/irods/test + env PYTHON_RULE_ENGINE_INSTALLED=yes python runner.py --tests_file /tmp/skipped.txt + " +} + +run_tests diff --git a/irods/test/scripts/test001_pam_password_expiration.bats b/irods/test/scripts/test001_pam_password_expiration.bats index 71da0c153..5eff0b990 100755 --- a/irods/test/scripts/test001_pam_password_expiration.bats +++ b/irods/test/scripts/test001_pam_password_expiration.bats @@ -16,8 +16,8 @@ setup() teardown() { - finalize_pam_login_for_alice - test_specific_cleanup + finalize_pam_login_for_alice + test_specific_cleanup } @test main { diff --git a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats index acf9c9594..8d84ddf82 100755 --- a/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats +++ b/irods/test/scripts/test003_write_pam_credentials_to_secrets_file.bats @@ -43,9 +43,13 @@ except irods.client_init.irodsA_already_exists: [ -n "$CONTENTS1" -a "$CONTENTS1" = "$CONTENTS2" ] # Now delete the already existing irodsA and repeat without negating overwrite. + TIMESTAMP_0=$(stat -c%Y $auth_file) + sleep 2 $PYTHON -c "import irods.client_init; irods.client_init.write_pam_irodsA_file('$ALICES_NEW_PAM_PASSWD')" - CONTENTS3=$(cat $auth_file) - [ "$CONTENTS2" != "$CONTENTS3" ] + TIMESTAMP=$(stat -c%Y $auth_file) + + # Test only the timestamp of the new auth_file, not the content, since that is implicitly asserted by the next step. + [ $(($TIMESTAMP-TIMESTAMP_0)) -ge 1 ] # Define the core Python to be run, basically a minimal code block ensuring that we can authenticate to iRODS # without an exception being raised. diff --git a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats index 23aecd8ce..de9304461 100755 --- a/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats +++ b/irods/test/scripts/test008_prc_write_irodsA_utility_in_native_mode.bats @@ -48,6 +48,9 @@ print ('env_auth_scheme=%s' % ses.pool.account._original_authentication_scheme) # Write another .irodsA prc_write_irodsA.py native <<<"rods" + CLIENT_JSON=~/.irods/irods_environment.json + jq '.["irods_client_server_policy"]="CS_NEG_REFUSE"' <$CLIENT_JSON >/tmp/client_json_test008.$$ + mv /tmp/client_json_test008.$$ $CLIENT_JSON # Verify new .irodsA for both iCommands and PRC use. ils >/tmp/stdout OUTPUT=$($PYTHON -c "$SCRIPT") diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions new file mode 100644 index 000000000..ce873aadc --- /dev/null +++ b/irods/test/scripts/test_support_functions @@ -0,0 +1,239 @@ +SCRIPTDIR=${BASH_SOURCE[0]} +up_from_script_dir() { + local x incr="" + for ((x=0;x<${1:-0};x++)); do incr+="/.."; done + realpath "$(dirname "$SCRIPTDIR")""$incr" +} + +# Sample usages: +# By user irods: set_up_ssl "" "-q" +# By sudo enabled user: set_up_ssl "sudo" "-q" +set_up_ssl() { + local SUDO=${1:-""} + local OPTS=${2:-""} + $SUDO su - irods -c "python3 $(up_from_script_dir 1)/setupssl.py $OPTS" +} + +# Clears out environment and resets to rodsadmin 'rods'. +# Meant mostly to allow initial steps by a rodsadminfor setting up tests. + +iinit_as_rods() { + rm -fr ~/.irods + iinit <<<$(hostname)$'\n1247\nrods\ntempZone\nrods' +} + +dot_to_space() { + sed 's/\./ /g'<<<"$1" +} + +CLEANUP=$':\n' + +GT() { (return 1); echo $?; } +LT() { (return -1); echo $?; } +EQ() { (return 0); echo $?; } + +compare_int_tuple() { + local x=($1) y=($2) + local lx=${#x[@]} ly=${#y[@]} + local i maxlen=$((lx > ly ? lx : ly)) + for ((i=0;i ~/.irods/irods_environment.json + + # TODO: check: it seems /dev/tty won't work if docker exec is not invoked with -t + if [ -n "$1" -a -z "$SKIP_IINIT_FOR_PASSWORD" ]; then + iinit <<<"$1" 2>/tmp/iinit_as_alice.log + fi +} + +_end_pam_environment_and_password() { + rm -fr ~/.irods + mv ~/.irods.$$ ~/.irods +} + +setup_pam_login_for_user() { + local user=${2:-alice} + sudo useradd $user --create-home + local PASSWD=${1:-test123} + sudo chpasswd <<<"$user:$PASSWD" + iadmin mkuser $user rodsuser + _begin_pam_environment_and_password "$PASSWD" $user +} + +setup_pam_login_for_alice() { + setup_pam_login_for_user "$1" alice +} + +finalize_pam_login_for_user() { + local USER=${1} + _end_pam_environment_and_password + iadmin rmuser "$USER" + sudo userdel "$USER" --remove +} + +finalize_pam_login_for_alice() { + finalize_pam_login_for_user alice +} + +test_specific_cleanup() { + eval "$CLEANUP" +} + +# PostgreSQL only +age_out_pam_password() { + # sets create_ts and modify_ts (timestamps) to older values, decreasing them by an amount of (offset + 1) where offset + # is the number of seconds for expiry_ts stored in the ICAT for the given user and password. In this way, we can + # artificially age out an existing pam password. + # Parameters: + # $1 - The username + # $2 - (optional) override the amount used for offsetting the create & modify timestamps. + local id=$(iquest %s "select USER_ID where USER_NAME = '$1'") + local offset=$(sudo su - postgres -c "psql -t ICAT -c 'select pass_expiry_ts from r_user_password where user_id = $id'") + local mtime=$(sudo su - postgres -c "psql -t ICAT -c 'select modify_ts from r_user_password where user_id = $id'") + mtime=$(sed 's/^\s*0//' <<<"$mtime") + [ -n "$2" ] && offset="$2" + ((offset+=1)) + local new_time=$((mtime - offset)) + sudo su - postgres -c "psql ICAT -c 'update r_user_password set create_ts=$new_time, modify_ts=$new_time where user_id=$id'" +} + +call_irodsctl() { + local arg=${1:-restart} + sudo su - irods -c "./irodsctl $arg" +} + +add_irods_to_system_pam_configuration() { + local tempfile=/tmp/irods-pam-config.$$ + cat <<-EOF >$tempfile + auth required pam_env.so + auth sufficient pam_unix.so + auth requisite pam_succeed_if.so uid >= 500 quiet + auth required pam_deny.so + EOF + sudo chown root.root $tempfile + sudo mv $tempfile /etc/pam.d/irods +} + +setup_preconnect_preference() { + sudo su irods -c "sed -i.orig 's/\(^\s*acPreConnect.*CS_NEG\)\([A-Z_]*\)/\1_$1/' /etc/irods/core.re" +} + +setup_pyN() { + if [ ! -d /pyN ]; then + mkdir /pyN ; chown testuser /pyN + su - testuser -c "/root/python/bin/python3 -m virtualenv /pyN" + + # TODO, REMOVE: Verbose to check proper version is being installed + echo "/pyN venv is python version => [$(. /pyN/bin/activate ; python -V)]" + + cp -r /prc{,.rw} + chown -R testuser /prc.rw + fi +} + +# requires image to descend from bats-python3 +activate_virtual_env_with_prc_installed() +{ + local py_venv=${1:-pyN} + [ "$py_venv" = pyN ] && sudo bash -c "$(declare -f setup_pyN); setup_pyN" + # install python client using copy of /prc so that bdist doesn't build in the readonly mount + sudo su - -c "source /${py_venv}/bin/activate && cp -rp /prc /prc-copy && \ + pip install '/prc-copy[tests]' && sudo rm -fr /prc-copy" + source /${py_venv}/bin/activate + echo "---> Python virtual environment activated. Interpreter Version is: $(python -V)" +} + +mtime_and_content() +{ + stat -c%y "$1" + cat "$1" +} + +irods_server_version() { + python -c "import irods.helpers as h +import operator,sys +if len(sys.argv) == 1: + (comparison,relto)=('','') +elif len(sys.argv) == 3: + (comparison,relto)=sys.argv[1:3] +fm_tuple = lambda tup: '.'.join(str(_) for _ in tup) +to_tuple = lambda vstr: tuple(int(_) for _ in vstr.split('.')) +svt = h.make_session().server_version_without_auth() +if relto: + exit(0 if vars(operator)[comparison](svt,to_tuple(relto)) else 1) +print(fm_tuple(svt)) +" $1 $2 +} diff --git a/irods/test/scripts/update_json_for_test b/irods/test/scripts/update_json_for_test new file mode 100644 index 000000000..3372fd48e --- /dev/null +++ b/irods/test/scripts/update_json_for_test @@ -0,0 +1,69 @@ +#!/bin/bash + +declare -A tls_server_items=( + [tls_server]='{"certificate_chain_file":"/etc/irods/ssl/irods.crt", + "certificate_key_file":"/etc/irods/ssl/irods.key", + "dh_params_file":"/etc/irods/ssl/dhparams.pem"}' +) + +declare -A tls_client_items=( + [tls_client]='{"ca_certificate_file":"/etc/irods/ssl/irods.crt", + "ca_certificate_path":"/etc/ssl/certs", + "verify_server":"cert"}' +) + +declare -A ssl_keys=( + [irods_client_server_negotiation]='"request_server_negotiation"' + [irods_client_server_policy]='"CS_NEG_REQUIRE"' + [irods_ssl_ca_certificate_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_chain_file]='"/etc/irods/ssl/irods.crt"' + [irods_ssl_certificate_key_file]='"/etc/irods/ssl/irods.key"' + [irods_ssl_dh_params_file]='"/etc/irods/ssl/dhparams.pem"' + [irods_ssl_verify_server]='"cert"' +) + +declare -A pam_keys=( + [irods_authentication_scheme]="\"$(pam_auth_string)\"" +) + +declare -A encrypt_keys=( + [irods_encryption_key_size]=16 + [irods_encryption_salt_size]=8 + [irods_encryption_num_hash_rounds]=16 + [irods_encryption_algorithm]='"AES-256-CBC"' +) + +declare -A RESTORE_FILES=() + +update_json_file() { + local file=$1 content=$2 + local bn=$(basename "$file") + local orig=/tmp/$bn.orig.$$ + local newfile=/tmp/$bn.new.$$ + echo "$content" >"$newfile" + sudo chmod --reference "$file" "$newfile" + sudo chown --reference "$file" "$newfile" + { sudo mv "$file" "$orig" && sudo mv "$newfile" "$file"; } || return 1 + RESTORE_FILES["$file"]="$orig" +} + +restore_json_files() { + local kk + for kk in ${!RESTORE_FILES[@]};do + sudo mv -f "${RESTORE_FILES["$kk"]}" "$kk" + done +} + +newcontent () { + local file=$1 + shift + local j=$(sudo cat "$file") + while [ $# -gt 0 ]; do + eval ' + for kk in ${!'$1'[@]}; do + j=$(jq ".$kk=${'$1'[$kk]}" <<<"$j") + done' + shift + done + echo "$j" +} diff --git a/irods/test/setupssl.py b/irods/test/setupssl.py index 3d3c20205..d14e682c0 100755 --- a/irods/test/setupssl.py +++ b/irods/test/setupssl.py @@ -60,7 +60,7 @@ def create_ssl_dir( # https://www.openssl.org/docs/man1.0.2/man1/dhparam.html#:~:text=DH%20parameter%20generation%20with%20the,that%20may%20be%20possible%20otherwise. if use_strong_primes_for_dh_generation: dhparam_generation_command = ( - "openssl dhparam -2 -out dhparams.pem" + "openssl dhparam -2 -out dhparams.pem 2048" ) else: dhparam_generation_command = (