From 3f959d9d4ac367cac6a7e6332c42ba7c07a3d612 Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Tue, 30 Jan 2024 11:07:48 -0500
Subject: [PATCH 01/13] [_502] experimental test harness using containers

testuser universal for all containers

login_auth*.py adjustments

[__502] scriptdir

[__502] patch for consistency in starting

document SKIP_IINIT_FOR_PASSWORD

"funcs" include file renamed to "test_support_functions"

remove fail.sh to scripts dir
---
 irods/test/demo.sh                            |   9 +
 irods/test/demo_A.sh                          |   1 +
 irods/test/demo_B.sh                          |   1 +
 irods/test/demo_hook.sh                       |   3 +
 .../test/harness/000_install-irods.Dockerfile |  14 ++
 .../test/harness/001_bats-python3.Dockerfile  |   5 +
 irods/test/harness/002_ssl-and-pam.Dockerfile |   1 +
 .../harness/003_prc-with-py3_12_2.Dockerfile  |  14 ++
 irods/test/harness/README.txt                 |  46 +++++
 irods/test/harness/build-docker.sh            |  21 ++
 irods/test/harness/docker_container_driver.sh | 109 ++++++++++
 irods/test/harness/install.sh                 | 139 +++++++++++++
 irods/test/harness/print_repo_root_location   |   5 +
 .../harness/start_postgresql_and_irods.sh     |  28 +++
 irods/test/harness/test_script_parameters     |  42 ++++
 irods/test/harness/tests/test_1.sh            |   6 +
 irods/test/harness/tests/test_2.sh            |   6 +
 irods/test/harness/tests/test_3.bats          |  21 ++
 irods/test/login_auth_test.sh                 |  39 ++++
 irods/test/login_auth_test_1.py               |   1 +
 irods/test/login_auth_test_2.py               |   1 +
 irods/test/pam.bats/funcs                     | 108 ----------
 irods/test/scripts/demo_script                |   2 +
 irods/test/scripts/experiment.sh              |   6 +
 irods/test/scripts/fail.sh                    |  10 +
 irods/test/scripts/test_support_functions     | 194 ++++++++++++++++++
 irods/test/scripts/update_json_for_test       |  55 +++++
 27 files changed, 779 insertions(+), 108 deletions(-)
 create mode 100755 irods/test/demo.sh
 create mode 120000 irods/test/demo_A.sh
 create mode 120000 irods/test/demo_B.sh
 create mode 100755 irods/test/demo_hook.sh
 create mode 100644 irods/test/harness/000_install-irods.Dockerfile
 create mode 100644 irods/test/harness/001_bats-python3.Dockerfile
 create mode 100644 irods/test/harness/002_ssl-and-pam.Dockerfile
 create mode 100644 irods/test/harness/003_prc-with-py3_12_2.Dockerfile
 create mode 100644 irods/test/harness/README.txt
 create mode 100755 irods/test/harness/build-docker.sh
 create mode 100755 irods/test/harness/docker_container_driver.sh
 create mode 100755 irods/test/harness/install.sh
 create mode 100755 irods/test/harness/print_repo_root_location
 create mode 100755 irods/test/harness/start_postgresql_and_irods.sh
 create mode 100644 irods/test/harness/test_script_parameters
 create mode 100755 irods/test/harness/tests/test_1.sh
 create mode 100755 irods/test/harness/tests/test_2.sh
 create mode 100755 irods/test/harness/tests/test_3.bats
 create mode 100755 irods/test/login_auth_test.sh
 create mode 120000 irods/test/login_auth_test_1.py
 create mode 120000 irods/test/login_auth_test_2.py
 delete mode 100644 irods/test/pam.bats/funcs
 create mode 100755 irods/test/scripts/demo_script
 create mode 100755 irods/test/scripts/experiment.sh
 create mode 100755 irods/test/scripts/fail.sh
 create mode 100644 irods/test/scripts/test_support_functions
 create mode 100644 irods/test/scripts/update_json_for_test

diff --git a/irods/test/demo.sh b/irods/test/demo.sh
new file mode 100755
index 000000000..7ef5b7c2e
--- /dev/null
+++ b/irods/test/demo.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+echo "$0 running"
+echo args:
+for arg in $*; do
+    echo $((++x)): "[$arg]"
+done
+
+exit 118
diff --git a/irods/test/demo_A.sh b/irods/test/demo_A.sh
new file mode 120000
index 000000000..52239f28f
--- /dev/null
+++ b/irods/test/demo_A.sh
@@ -0,0 +1 @@
+demo.sh
\ No newline at end of file
diff --git a/irods/test/demo_B.sh b/irods/test/demo_B.sh
new file mode 120000
index 000000000..52239f28f
--- /dev/null
+++ b/irods/test/demo_B.sh
@@ -0,0 +1 @@
+demo.sh
\ No newline at end of file
diff --git a/irods/test/demo_hook.sh b/irods/test/demo_hook.sh
new file mode 100755
index 000000000..8a9f855fe
--- /dev/null
+++ b/irods/test/demo_hook.sh
@@ -0,0 +1,3 @@
+#!/bin/bash
+echo "-- HOOK RUNNING --"
+command "/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" $*
diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile
new file mode 100644
index 000000000..7071aa1e0
--- /dev/null
+++ b/irods/test/harness/000_install-irods.Dockerfile
@@ -0,0 +1,14 @@
+FROM ubuntu:18.04
+COPY install.sh /
+ARG irods_package_version
+ENV IRODS_PACKAGE_VERSION "$irods_package_version"
+RUN for phase in initialize install-essential-packages add-package-repo; do \
+        bash /install.sh --w=$phase 0; \
+    done
+RUN /install.sh 4
+COPY start_postgresql_and_irods.sh /
+RUN apt install -y sudo
+RUN useradd -ms/bin/bash testuser
+RUN echo 'testuser ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers
+RUN apt install -y faketime
+CMD bash /start_postgresql_and_irods.sh
diff --git a/irods/test/harness/001_bats-python3.Dockerfile b/irods/test/harness/001_bats-python3.Dockerfile
new file mode 100644
index 000000000..1c1e7b69b
--- /dev/null
+++ b/irods/test/harness/001_bats-python3.Dockerfile
@@ -0,0 +1,5 @@
+from install-irods
+run apt update; apt install -y python3-pip bats
+run python3 -m pip install --upgrade pip
+run python3 -m pip install virtualenv
+run python3 -m virtualenv /py3
diff --git a/irods/test/harness/002_ssl-and-pam.Dockerfile b/irods/test/harness/002_ssl-and-pam.Dockerfile
new file mode 100644
index 000000000..810d10e86
--- /dev/null
+++ b/irods/test/harness/002_ssl-and-pam.Dockerfile
@@ -0,0 +1 @@
+FROM bats-python3
diff --git a/irods/test/harness/003_prc-with-py3_12_2.Dockerfile b/irods/test/harness/003_prc-with-py3_12_2.Dockerfile
new file mode 100644
index 000000000..9a77a104a
--- /dev/null
+++ b/irods/test/harness/003_prc-with-py3_12_2.Dockerfile
@@ -0,0 +1,14 @@
+from install-irods
+run apt update
+run apt install -y wget build-essential libssl-dev zlib1g-dev
+run apt install wget build-essential
+run wget https://www.python.org/ftp/python/3.12.2/Python-3.12.2.tar.xz
+run tar xf Python-3.12.2.tar.xz
+workdir /Python-3.12.2
+run ./configure --prefix /root/python --with-ensurepip=install
+run make -j
+run mkdir /root/python
+run make install
+workdir /
+run /root/python/bin/python3 -m pip install python-irodsclient
+run chmod a+rx /root
diff --git a/irods/test/harness/README.txt b/irods/test/harness/README.txt
new file mode 100644
index 000000000..31f848963
--- /dev/null
+++ b/irods/test/harness/README.txt
@@ -0,0 +1,46 @@
+SAMPLE RUNS
+
+To build required images
+------------------------
+Examples
+
+    1) ./build-docker.sh 
+       DEFAULT: build  single-node system based on latest iRODS release
+
+    2) IRODS_PACKAGE_VERSION="4.2.12-1~bionic" NO_CACHE='1' ./build-docker.sh 
+       Build (ignoring docker cache) single-node system based on specified package version string.
+
+simple examples
+---------------
+./docker_container_driver.sh  tests/test_1.sh 
+./docker_container_driver.sh  tests/test_2.sh 
+
+Any script in a subdirectory of the repo (mounted at /prc within the container) can be
+executed and will be able to find other scripts and source include files within the tree.
+[See "experiment.sh" example below.]
+
+Examples of options in driver script
+------------------------------------
+
+  1. To start container and run test script:
+     C=$(  ./docker_container_driver.sh -c -L -u testuser  ../scripts/experiment.sh )
+
+  2. To manually examine results afterward:
+     docker exec -it $C bash 
+
+
+Demo / Demo hook  / args
+------------------------
+
+$ ~/python-irodsclient/irods/test/harness$ ./docker_container_driver.sh  ../demo.sh 
+ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[irods/test/demo.sh]
+image=[ssl-and-pam]
+.......-- HOOK RUNNING --
+/prc/irods/test/demo.sh running
+args:
+1: [arg1]
+2: [arg2]
+Killed: 1358fbff6eadac24f0915ffb414f0367deedc84b0c3e4de69a23bd3a8726298f
+daniel@prec3431:~/python-irodsclient/irods/test/harness$ echo $?
+118
+
diff --git a/irods/test/harness/build-docker.sh b/irods/test/harness/build-docker.sh
new file mode 100755
index 000000000..252b87fef
--- /dev/null
+++ b/irods/test/harness/build-docker.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# IRODS_PACKAGE_VERSION if defined is like "4.3.1-0~bionic"
+# (but contains no '~' suffix for irods versions <= 4.2.10)
+
+BASE=$(basename "$0")
+DIR=$(realpath "$(dirname "$0")")
+cd "$DIR"
+DOCKER=docker
+for dockerfile in [0-9]*.Dockerfile; do 
+    image_name=${dockerfile#[0-9]*_}
+    image_name=${image_name%.Dockerfile}
+    if [ "$image_name" = "install-irods" ];then
+        package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"}
+    else
+        package_version_option=""
+    fi
+    $DOCKER build -f $dockerfile -t $image_name . $package_version_option \
+                                                  ${NO_CACHE+"--no-cache"} || 
+                                                  { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; }
+done
diff --git a/irods/test/harness/docker_container_driver.sh b/irods/test/harness/docker_container_driver.sh
new file mode 100755
index 000000000..c340360ff
--- /dev/null
+++ b/irods/test/harness/docker_container_driver.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+KILL_TEST_CONTAINER=1
+RUN_AS_USER=""
+ECHO_CONTAINER=""
+
+EXPLICIT_WORKDIR=""
+while [[ $1  = -* ]]; do
+    if [ "$1" = -c ]; then
+        ECHO_CONTAINER=1
+        shift
+    fi
+    if [ "$1" = -L ]; then
+        KILL_TEST_CONTAINER=0
+        shift
+    fi
+    if [ "$1" = -u ]; then
+        RUN_AS_USER="$2"
+        shift 2
+    fi
+    if [ "$1" = -w ]; then
+        EXPLICIT_WORKDIR="$2"
+        shift 2
+    fi
+done
+
+if [ "$1" = "" ]; then
+    echo >&2 "Usage: $0 [options] /path/to/script"
+    echo >&2 "With options: [-L] to leak, [-u username] to run as non-root user"
+    exit 1
+fi
+
+DIR=$(dirname $0)
+. "$DIR"/test_script_parameters
+
+testscript=${1}
+
+testscript_basename=$(basename "$testscript")
+arglist=${wrapper_arglist[$testscript_basename]}  # arglist dominated by symbolic link name if any
+
+if [ -L "$testscript" ]; then
+    testscript=$(realpath "$testscript")
+    testscript_basename=$(basename "$testscript")
+fi
+
+original_testscript_abspath=$(realpath "$testscript")
+
+wrapped=${wrappers["$testscript_basename"]}
+
+if [ -n "$wrapped" ]; then
+    # wrapped is assumed to contain a leading path element relative to the referencing script's containing directory
+    testscript="$(dirname "$testscript")/$wrapped"
+    testscript_basename=$(basename "$testscript")
+fi
+
+testscript_abspath=$(realpath "$testscript")
+
+cd "$DIR"
+
+image=${images[$testscript_basename]}
+
+if [ -z "$RUN_AS_USER" ]; then
+    RUN_AS_USER=${user[$testscript_basename]}
+fi
+
+# Tests are run as testuser by default
+: ${RUN_AS_USER:='testuser'}
+
+WORKDIR=""
+if [ -n "$EXPLICIT_WORKDIR" ]; then
+    WORKDIR="$EXPLICIT_WORKDIR"
+else
+    WORKDIR=${workdirs[$RUN_AS_USER]}
+fi
+
+reporoot=$(./print_repo_root_location)
+ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to $reporoot "$original_testscript_abspath")
+
+echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" 
+INNER_MOUNT=/prc
+
+# Start the container.
+echo image="[$image]"
+CONTAINER=$(docker run -d -v $reporoot:$INNER_MOUNT:ro --rm $image)
+
+# Wait for iRODS and database to start up.
+TIME0=$(date +%s)
+while :; do
+    [ `date +%s` -gt $((TIME0 + 30)) ] && { echo >&2 "Waited too long for DB and iRODS to start"; exit 124; }
+    sleep 1 
+    docker exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null
+    [ $? -ne 0 ] && { echo -n . >&2; continue; }
+    break
+done
+
+docker exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \
+            ${WORKDIR:+"-w$WORKDIR"} \
+	    -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \
+            $CONTAINER \
+            $INNER_MOUNT/$(realpath --relative-to $reporoot "$testscript_abspath") \
+            $arglist
+STATUS=$?
+
+if [ $((0+KILL_TEST_CONTAINER)) -ne 0 ]; then
+    echo >&2 'Killed:' $(docker stop --time=0 $CONTAINER)
+fi
+
+[ -n "$ECHO_CONTAINER" ] && echo $CONTAINER
+exit $STATUS
diff --git a/irods/test/harness/install.sh b/irods/test/harness/install.sh
new file mode 100755
index 000000000..cd840a98a
--- /dev/null
+++ b/irods/test/harness/install.sh
@@ -0,0 +1,139 @@
+#!/bin/bash
+
+IRODS_HOME=/var/lib/irods
+DEV_HOME="$HOME"
+: ${DEV_REPOS:="$DEV_HOME/github"}
+
+add_package_repo()
+{
+      local R="/etc/apt/sources.list.d/renci-irods.list"
+      echo >&2 "... installing package repo"
+      sudo apt update
+      sudo apt install -y lsb-release apt-transport-https
+      wget -qO - https://packages.irods.org/irods-signing-key.asc | sudo apt-key add - && \
+      echo "deb [arch=amd64] https://packages.irods.org/apt/ $(lsb_release -sc) main" |\
+          sudo tee "$R"
+      sudo apt update
+}
+
+DIST_NAME=$(lsb_release -sc)
+
+: ${IRODS_VSN:=4.3.1-0~$DIST_NAME}
+
+while [[ "$1" = -* ]]; do
+  ARG="$1"
+  shift
+  case $ARG in
+    --i=* | --irods=* |\
+    --irods-version=*) IRODS_PACKAGE_VERSION=${ARG#*=};;
+    --w=* | --with=* | --with-options=* ) withopts=${ARG#*=} ;;
+    -v) VERBOSE=1;;
+  esac
+done
+
+
+run_phase() {
+
+ local PHASE=$1
+ local with_opts=" $2 "
+
+ case "$PHASE" in
+
+ 0)
+
+    if [[ $with_opts = *\ initialize\ * ]]; then
+        apt-get -y update
+        apt-get  install -y apt-transport-https wget lsb-release sudo jq
+    fi
+
+    if [[ $with_opts = *\ sudo-without-pw\ * ]]; then
+      if [ `id -u` = 0 -a "${USER:-root}" = root ] ; then
+        echo >&2 "root authorization for 'sudo' is automatic - no /etc/sudoers modification needed"
+      else
+        if [ -f "/etc/sudoers" ]; then
+           if [ -n "$USER" ] ; then
+             # add a line with our USER name to /etc/sudoers if not already there
+             sudo su -c "sed -n '/^\s*[^#]/p' /etc/sudoers | grep '^$USER\s*ALL=(ALL)\s*NOPASSWD:\s*ALL\s*$' >/dev/null" || \
+             sudo su -c "echo '$USER ALL=(ALL) NOPASSWD: ALL' >>/etc/sudoers"
+           else
+             echo >&2 "user login is '$USER' - can this be right?"
+           fi
+        else
+           echo >&2 "WARNING - Could not modify sudoers files"
+           echo -n >&2 "           (hit 'Enter' to continue)"
+           read key
+        fi
+      fi # not root
+    fi # with-opts
+
+    #------ (needed for both package install and build from source)
+
+    if [[ $with_opts = *\ install-essential-packages\ * ]]; then
+
+        if ! dpkg -l tzdata >/dev/null 2>&1 ; then
+          sudo su - root -c \
+           "env DEBIAN_FRONTEND=noninteractive bash -c 'apt-get install -y tzdata'"
+        fi
+        sudo apt-get update
+        sudo apt-get install -y software-properties-common postgresql
+        sudo apt-get update && \
+        sudo apt-get install -y libfuse2 unixodbc rsyslog ################### less python-pip 
+    fi
+
+
+    if [[ $with_opts = *\ add-package-repo\ * ]]; then
+            add_package_repo -f
+    fi
+
+
+    if [[ $with_opts = *\ create-db\ * ]]; then
+    sudo su - postgres -c "
+        { dropdb --if-exists ICAT
+          dropuser --if-exists irods ; } >/dev/null 2>&1"
+    sudo su - postgres -c "psql <<\\
+________
+        CREATE DATABASE \"ICAT\";
+        CREATE USER irods WITH PASSWORD 'testpassword';
+        GRANT ALL PRIVILEGES ON DATABASE \"ICAT\" to irods;
+________"
+    echo >&2 "-- status of create-db =  $? -- "
+    fi
+    ;;
+
+ 4)
+   sudo apt install -y irods-{dev,runtime}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"}
+   if [[ $with_opts != *\ basic\ * ]]; then
+     sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"}
+   fi
+ ;;
+
+ 5)
+ if [ ! "$IRODS_VSN" '<' "4.3" ]; then
+    PYTHON=python3
+ else
+    PYTHON=python2
+ fi
+ sudo $PYTHON /var/lib/irods/scripts/setup_irods.py < /var/lib/irods/packaging/localhost_setup_postgres.input
+ ;;
+
+ *) echo >&2 "unrecognized phase: '$PHASE'." ; QUIT=1 ;;
+ esac
+ return $?
+}
+
+#-------------------------- main
+
+QUIT=0
+while [ $# -gt 0 ] ; do
+  ARG=$1 ; shift
+  NOP="" ; run_phase $ARG " $withopts "; sts=$?
+  [ $QUIT != 0 ] && break
+  [ -n "$NOP" ] && continue
+  echo -n "== $ARG == "
+  if [ $sts -eq 0 ]; then
+    echo Y >&2
+  else
+    [ $quit_on_phase_err ] && { echo >&2 "N - quitting"; exit 1; }
+    echo N >&2
+  fi
+done
diff --git a/irods/test/harness/print_repo_root_location b/irods/test/harness/print_repo_root_location
new file mode 100755
index 000000000..adcf3fc73
--- /dev/null
+++ b/irods/test/harness/print_repo_root_location
@@ -0,0 +1,5 @@
+#!/bin/bash
+# The following line needs be kept updated to reflect true position relative to repository root,
+# in the event this script or any of its chain of containing directories (up to but not including the repo root) are moved.
+REPO_ROOT_RELATIVE_TO_THIS_SCRIPT=../../..
+realpath "$(dirname "$0")/$REPO_ROOT_RELATIVE_TO_THIS_SCRIPT"
diff --git a/irods/test/harness/start_postgresql_and_irods.sh b/irods/test/harness/start_postgresql_and_irods.sh
new file mode 100755
index 000000000..fd41dfec3
--- /dev/null
+++ b/irods/test/harness/start_postgresql_and_irods.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+service postgresql start
+x=${DB_WAIT_SEC:-20}
+while [ $x -ge 0 ] && { ! $SUDO su - postgres -c "psql -c '\l' >/dev/null 2>&1" || x=""; }
+do
+  [ -z "$x" ] && break
+  echo >&2 "$((x--)) secs til database timeout"; sleep 1
+done
+[ -z "$x" ] || { echo >&2 "Error -- database didn't start" ; exit 1; }
+if ! id -u irods >/dev/null 2>&1 ; then
+    /install.sh --w=create-db 0
+    VERSION_file=$(ls /var/lib/irods/{VERSION,version}.json.dist 2>/dev/null)
+    IRODS_VSN=$(jq -r '.irods_version' $VERSION_file) /install.sh 5
+fi
+su - irods -c '~/irodsctl restart'
+IRODS_WAIT_SEC=20
+x=$IRODS_WAIT_SEC
+SLEEPTIME=""
+while [ $((x--)) -gt 0 ]; do
+  sleep $((SLEEPTIME+0))
+  pgrep irodsServer
+  STATUS=$?
+  [ $STATUS -eq 0 ] && break
+  SLEEPTIME=1
+done
+echo "($STATUS)" >/tmp/irods_status
+[ $STATUS -eq 0 ] || exit 125
+tail -f /dev/null
diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters
new file mode 100644
index 000000000..77ddb3032
--- /dev/null
+++ b/irods/test/harness/test_script_parameters
@@ -0,0 +1,42 @@
+# keys for Arglist refer to argument given, which could be a symlink.
+
+declare -A wrapper_arglist=(
+    [demo.sh]="arg1 arg2"
+    [demo_A.sh]="arg1-a arg2-a"
+    [login_auth_test.py]="TestLogins"
+    [login_auth_test_1.py]="-v TestAnonymousUser TestMiscellaneous"
+    [login_auth_test_2.py]="-v TestWithSSL"
+)
+
+# keys for Wrapper refer to argument after resolution of any symlinks
+
+declare -A wrappers=(
+    [login_auth_test_must_run_manually.py]=./login_auth_test.sh
+    [PRC_issue_362.bats]=./login_auth_test.sh
+    [test001_pam_password_expiration.bats]=../login_auth_test.sh
+    [demo.sh]=./demo_hook.sh
+)
+
+# keys for Image and User refer to the basename after resolution to a wrapper if one is used
+
+declare -A images=(
+    [login_auth_test.sh]=bats-python3
+    [test_1.sh]=install-irods
+    [test_2.sh]=bats-python3
+    [test_3.bats]=bats-python3
+    [experiment.sh]=ssl-and-pam
+    [fail.sh]=ssl-and-pam
+    [login_auth_test_must_run_manually.py]=ssl-and-pam
+    [demo_hook.sh]=ssl-and-pam
+)
+
+declare -A user=(
+)
+
+# keys for WorkDir refer to user
+
+declare -A workdirs=(
+    [testuser]=/home/testuser
+    [irods]=/var/lib/irods
+    [root]=/
+)
diff --git a/irods/test/harness/tests/test_1.sh b/irods/test/harness/tests/test_1.sh
new file mode 100755
index 000000000..798302f15
--- /dev/null
+++ b/irods/test/harness/tests/test_1.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+run() {
+  echo dir of this = $(realpath "$(dirname "${BASH_SOURCE[0]}")/repo")
+}
+
+echo hello_there
diff --git a/irods/test/harness/tests/test_2.sh b/irods/test/harness/tests/test_2.sh
new file mode 100755
index 000000000..f5b9279f4
--- /dev/null
+++ b/irods/test/harness/tests/test_2.sh
@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+run() {
+  echo dir of this = $(realpath "$(dirname "${BASH_SOURCE[0]}")/repo")
+}
+
+echo later_alligator
diff --git a/irods/test/harness/tests/test_3.bats b/irods/test/harness/tests/test_3.bats
new file mode 100755
index 000000000..18be2ad07
--- /dev/null
+++ b/irods/test/harness/tests/test_3.bats
@@ -0,0 +1,21 @@
+#!/usr/bin/env bats
+
+. "$BATS_TEST_DIRNAME"/../setup_pam_and_ssl.funcs
+
+setup() {
+  echo setup >>/tmp/log
+  setup_preconnect_preference DONT_CARE
+  python3 "$BATS_TEST_DIRNAME"/repo/irods/test/setupssl.py
+:
+}
+
+teardown() {
+  echo teardown >>/tmp/log
+:
+}
+
+@test mytest {
+  grep acPreConn /etc/irods/core.re >>/tmp/log
+  echo test proper >>/tmp/log
+:
+}
diff --git a/irods/test/login_auth_test.sh b/irods/test/login_auth_test.sh
new file mode 100755
index 000000000..399a1c2ea
--- /dev/null
+++ b/irods/test/login_auth_test.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+. $(dirname $0)/scripts/test_support_functions
+. $(dirname $0)/scripts/update_json_for_test
+
+IRODS_SERVICE_ACCOUNT_ENV_FILE=~irods/.irods/irods_environment.json 
+LOCAL_ACCOUNT_ENV_FILE=~/.irods/irods_environment.json 
+
+setup_preconnect_preference DONT_CARE
+
+add_irods_to_system_pam_configuration
+
+# set up /etc/irods/ssl directory and files
+set_up_ssl sudo -q
+
+sudo useradd -ms/bin/bash alissa 
+sudo chpasswd <<<"alissa:test123"
+
+update_json_file $IRODS_SERVICE_ACCOUNT_ENV_FILE \
+                 "$(newcontent $IRODS_SERVICE_ACCOUNT_ENV_FILE ssl_keys)"
+
+# This is mostly so we can call python3 as just "python"
+activate_virtual_env_with_prc_installed >/dev/null 2>&1 || { echo >&2 "couldn't set up virtual environment"; exit 1; }
+
+# Set up testuser with rods+SSL so we never have to run login_auth_tests.py as the service account.
+iinit_as_rods >/dev/null 2>&1 || { echo >&2 "couldn't iinit as rods"; exit 2; }
+update_json_file $LOCAL_ACCOUNT_ENV_FILE \
+                 "$(newcontent $LOCAL_ACCOUNT_ENV_FILE ssl_keys encrypt_keys)"
+
+original_script=/prc/$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT
+# Run tests.
+if [ -x "$original_script" ]; then 
+  command "$original_script" $*
+elif [[ $original_script =~ \.py$ ]]; then
+  python "$original_script" $*
+elif [[ $original_script =~ \.bats$ ]]; then
+  bats "$original_script"
+else
+  echo >&2 "I don't know how to run this: original_script=[$original_script]"
+fi
diff --git a/irods/test/login_auth_test_1.py b/irods/test/login_auth_test_1.py
new file mode 120000
index 000000000..23402ed84
--- /dev/null
+++ b/irods/test/login_auth_test_1.py
@@ -0,0 +1 @@
+login_auth_test_must_run_manually.py
\ No newline at end of file
diff --git a/irods/test/login_auth_test_2.py b/irods/test/login_auth_test_2.py
new file mode 120000
index 000000000..23402ed84
--- /dev/null
+++ b/irods/test/login_auth_test_2.py
@@ -0,0 +1 @@
+login_auth_test_must_run_manually.py
\ No newline at end of file
diff --git a/irods/test/pam.bats/funcs b/irods/test/pam.bats/funcs
deleted file mode 100644
index 30539a03a..000000000
--- a/irods/test/pam.bats/funcs
+++ /dev/null
@@ -1,108 +0,0 @@
-dot_to_space() {
-  sed 's/\./ /g'<<<"$1"
-}
-
-CLEANUP=$':\n'
-
-GT() { (return 1); echo $?; }
-LT() { (return -1); echo $?; }
-EQ() { (return 0); echo $?; }
-
-compare_int_tuple() {
-    local x=($1) y=($2)
-    local lx=${#x[@]} ly=${#y[@]}
-    local i maxlen=$((lx > ly ? lx : ly))
-    for ((i=0;i<maxlen;i++)) {
-        if [ $i -ge $lx ]; then return `LT`; fi
-        if [ $i -ge $ly ]; then return `GT`; fi
-        if [ ${x[$i]} -lt ${y[$i]} ]; then return `LT`; fi
-        if [ ${x[$i]} -gt ${y[$i]} ]; then return `GT`; fi
-    }
-    return `EQ`
-}
-
-irods_version()
-{
-  local X=''
-  [[ `ihelp -h|tail -1` =~ [0-9]+(\.[0-9]+)+ ]] && X=${BASH_REMATCH[0]}
-  echo "$X"
-}
-
-pam_test_would_take_too_long()
-{
-    compare_int_tuple "$(dot_to_space `irods_version`)" "$(dot_to_space 4.3.1)"
-    [ $? = `LT` ]
-}
-
-get_auth_param () { iadmin get_grid_configuration authentication $1; }
-
-with_change_auth_params_for_test()
-{
-  local restore_cmd=""
-
-  # Use saved environment (must be rodsadmin).
-  if [ -e ~/.irods.$$ ]; then
-      export IRODS_ENVIRONMENT_FILE=~/.irods.$$/irods_environment.json
-      export IRODS_AUTHENTICATION_FILE=~/.irods.$$/.irodsA
-  fi
-
-  while [ $# -ge 2 ]; do
-    local reset_value=$(iadmin get_grid_configuration authentication $1)
-    restore_cmd+=$'\n'"iadmin set_grid_configuration authentication $1 $reset_value"
-    [ -n "$2" ] && iadmin set_grid_configuration authentication $1 $2
-    shift 2
-  done
-
-  unset IRODS_ENVIRONMENT_FILE IRODS_AUTHENTICATION_FILE
-
-  if [[ $SET_CLEANUP = [yY]* ]]; then
-    CLEANUP+="$restore_cmd"
-  fi
-}
-
-_begin_pam_environment_and_password() {
-    local ENV='{
-    "irods_host": "localhost",
-    "irods_zone_name": "tempZone",
-    "irods_port": 1247,
-    "irods_user_name": "alice",
-    "irods_authentication_scheme": "pam_password",
-    "irods_client_server_negotiation": "request_server_negotiation",
-    "irods_client_server_policy": "CS_NEG_REQUIRE",
-    "irods_ssl_ca_certificate_file": "/etc/irods/ssl/irods.crt",
-    "irods_ssl_verify_server": "cert",
-    "irods_encryption_key_size": 16,
-    "irods_encryption_salt_size": 8,
-    "irods_encryption_num_hash_rounds": 16,
-    "irods_encryption_algorithm": "AES-256-CBC"
-    }'
-
-    rm -fr ~/.irods.$$
-    mv ~/.irods ~/.irods.$$
-    mkdir ~/.irods
-    echo "$ENV" > ~/.irods/irods_environment.json
-    iinit <<<"$1" 2>/dev/tty
-}
-
-_end_pam_environment_and_password() {
-    rm -fr ~/.irods
-    mv ~/.irods.$$ ~/.irods
-}
-
-setup_pam_login_for_alice() {
-    sudo useradd alice --create-home
-    local PASSWD=${1:-test123}
-    sudo chpasswd <<<"alice:$PASSWD"
-    iadmin mkuser alice rodsuser
-    _begin_pam_environment_and_password "$PASSWD"
-}
-
-finalize_pam_login_for_alice() {
-    _end_pam_environment_and_password
-    iadmin rmuser alice
-    sudo userdel alice --remove
-}
-
-test_specific_cleanup() {
-  eval "$CLEANUP"
-}
diff --git a/irods/test/scripts/demo_script b/irods/test/scripts/demo_script
new file mode 100755
index 000000000..7202d9cf3
--- /dev/null
+++ b/irods/test/scripts/demo_script
@@ -0,0 +1,2 @@
+#!/bin/bash
+echo "hello from script [$0]"
diff --git a/irods/test/scripts/experiment.sh b/irods/test/scripts/experiment.sh
new file mode 100755
index 000000000..89bdd086e
--- /dev/null
+++ b/irods/test/scripts/experiment.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+DIR=$(dirname $0)
+. $DIR/test_support_functions
+cd "$DIR"
+set_up_ssl sudo
+add_irods_to_system_pam_configuration
diff --git a/irods/test/scripts/fail.sh b/irods/test/scripts/fail.sh
new file mode 100755
index 000000000..e84d70e10
--- /dev/null
+++ b/irods/test/scripts/fail.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+DIR=$(dirname $0)
+. $DIR/test_support_functions
+cd "$DIR"; echo >&2 -n -- ; pwd
+#echo "setting up"
+$(up_from_script_dir 1)/demo_script
+#set_up_ssl sudo
+id -un
+exit 12
diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions
new file mode 100644
index 000000000..73a2db385
--- /dev/null
+++ b/irods/test/scripts/test_support_functions
@@ -0,0 +1,194 @@
+SCRIPTDIR=${BASH_SOURCE[0]}
+up_from_script_dir() {
+  local x incr=""
+  for ((x=0;x<${1:-0};x++)); do incr+="/.."; done
+  realpath "$(dirname "$SCRIPTDIR")""$incr"
+}
+
+# Sample usages:
+#   By user irods: set_up_ssl "" "-q"
+#   By sudo enabled user: set_up_ssl "sudo" "-q"
+set_up_ssl() {
+  local SUDO=${1:-""}
+  local OPTS=${2:-""}
+  $SUDO su - irods  -c "python3 $(up_from_script_dir 1)/setupssl.py $OPTS"
+}
+
+# Clears out environment and resets to rodsadmin 'rods'.
+# Meant mostly to allow initial steps by a rodsadminfor setting up tests.
+
+iinit_as_rods() {
+  rm -fr ~/.irods
+  iinit <<<$(hostname)$'\n1247\nrods\ntempZone\nrods'
+}
+
+dot_to_space() {
+  sed 's/\./ /g'<<<"$1"
+}
+
+CLEANUP=$':\n'
+
+GT() { (return 1); echo $?; }
+LT() { (return -1); echo $?; }
+EQ() { (return 0); echo $?; }
+
+compare_int_tuple() {
+    local x=($1) y=($2)
+    local lx=${#x[@]} ly=${#y[@]}
+    local i maxlen=$((lx > ly ? lx : ly))
+    for ((i=0;i<maxlen;i++)) {
+        if [ $i -ge $lx ]; then return `LT`; fi
+        if [ $i -ge $ly ]; then return `GT`; fi
+        if [ ${x[$i]} -lt ${y[$i]} ]; then return `LT`; fi
+        if [ ${x[$i]} -gt ${y[$i]} ]; then return `GT`; fi
+    }
+    return `EQ`
+}
+
+irods_version()
+{
+  local X=''
+  [[ `ihelp -h|tail -1` =~ [0-9]+(\.[0-9]+)+ ]] && X=${BASH_REMATCH[0]}
+  echo "$X"
+}
+
+pam_auth_string()
+{
+    compare_int_tuple "$(dot_to_space `irods_version`)" "$(dot_to_space 4.3)"
+    if [ $? = `GT` ]; then
+        echo "pam_password"
+    else
+        echo "pam"
+    fi
+}
+
+get_auth_param () { iadmin get_grid_configuration authentication $1; }
+
+with_change_auth_params_for_test()
+{
+  local restore_cmd=""
+
+  # Use saved environment (must be rodsadmin).
+  if [ -e ~/.irods.$$ ]; then
+      export IRODS_ENVIRONMENT_FILE=~/.irods.$$/irods_environment.json
+      export IRODS_AUTHENTICATION_FILE=~/.irods.$$/.irodsA
+  fi
+
+  while [ $# -ge 2 ]; do
+    local reset_value=$(iadmin get_grid_configuration authentication $1)
+    restore_cmd+=$'\n'"iadmin set_grid_configuration authentication $1 $reset_value"
+    [ -n "$2" ] && iadmin set_grid_configuration authentication $1 $2
+    shift 2
+  done
+
+  unset IRODS_ENVIRONMENT_FILE IRODS_AUTHENTICATION_FILE
+
+  if [[ $SET_CLEANUP = [yY]* ]]; then
+    CLEANUP+="$restore_cmd"
+  fi
+}
+
+_begin_pam_environment_and_password() {
+    # If the environment variable SKIP_IINIT_FOR_PASSWORD is set to a non-null length string, then
+    # the step of generating the .irodsA containing the PAM authentication secret will be skipped.
+
+    local username=${2:-alice}
+    local ENV='{
+    "irods_host": "localhost",
+    "irods_zone_name": "tempZone",
+    "irods_port": 1247,
+    "irods_user_name": "'"$username"'",
+    "irods_authentication_scheme": "'$(pam_auth_string)'",
+    "irods_client_server_negotiation": "request_server_negotiation",
+    "irods_client_server_policy": "CS_NEG_REQUIRE",
+    "irods_ssl_ca_certificate_file": "/etc/irods/ssl/irods.crt",
+    "irods_ssl_verify_server": "cert",
+    "irods_encryption_key_size": 16,
+    "irods_encryption_salt_size": 8,
+    "irods_encryption_num_hash_rounds": 16,
+    "irods_encryption_algorithm": "AES-256-CBC"
+    }'
+    rm -fr ~/.irods.$$
+    mv ~/.irods ~/.irods.$$
+    mkdir ~/.irods
+    echo "$ENV" > ~/.irods/irods_environment.json
+
+    # TODO: check: it seems /dev/tty won't work if docker exec is not invoked with -t
+    if [ -n "$1" ]; then
+        iinit <<<"$1" 2>/tmp/iinit_as_alice.log
+    fi
+}
+
+_end_pam_environment_and_password() {
+    rm -fr ~/.irods
+    mv ~/.irods.$$ ~/.irods
+}
+
+setup_pam_login_for_user() {
+    local user=${2:-alice}
+    sudo useradd $user --create-home
+    local PASSWD=${1:-test123}
+    sudo chpasswd <<<"$user:$PASSWD"
+    iadmin mkuser $user rodsuser
+    _begin_pam_environment_and_password "$PASSWD" $user
+}
+
+setup_pam_login_for_alice() {
+    setup_pam_login_for_user "$1" alice
+}
+
+finalize_pam_login_for_user() {
+    local USER=${1}
+    _end_pam_environment_and_password
+    iadmin rmuser "$USER"
+    sudo userdel "$USER" --remove
+}
+
+finalize_pam_login_for_alice() {
+    finalize_pam_login_for_user alice
+}
+
+test_specific_cleanup() {
+  eval "$CLEANUP"
+}
+
+# PostgreSQL only
+age_out_pam_password() {
+  local id=$(iquest %s "select USER_ID where USER_NAME = '$1'")
+  local offset=$(sudo su - postgres -c "psql -t ICAT -c 'select pass_expiry_ts from r_user_password where user_id = $id'")
+  local mtime=$(sudo su - postgres -c "psql -t ICAT -c 'select modify_ts from r_user_password where user_id = $id'")
+  mtime=$(sed 's/^\s*0//' <<<"$mtime")
+  ((offset+=1))
+  local new_time=$((mtime - offset))
+  sudo su - postgres -c "psql ICAT -c 'update r_user_password set create_ts=$new_time, modify_ts=$new_time where user_id=$id'"
+}
+
+call_irodsctl() {
+    local arg=${1:-restart}
+    sudo su - irods -c "./irodsctl $arg"
+}
+
+add_irods_to_system_pam_configuration() {
+    local tempfile=/tmp/irods-pam-config.$$
+    cat <<-EOF >$tempfile
+	auth        required      pam_env.so
+	auth        sufficient    pam_unix.so
+	auth        requisite     pam_succeed_if.so uid >= 500 quiet
+	auth        required      pam_deny.so
+	EOF
+    sudo chown root.root $tempfile
+    sudo mv $tempfile /etc/pam.d/irods
+}
+
+setup_preconnect_preference() {
+    sudo su irods -c "sed -i.orig 's/\(^\s*acPreConnect.*CS_NEG\)\([A-Z_]*\)/\1_$1/' /etc/irods/core.re"
+}
+
+# requires image to descend from bats-python3
+activate_virtual_env_with_prc_installed()
+{
+    # install python client using copy of /prc so that bdist doesn't build in the readonly mount
+    sudo su - -c "source /py3/bin/activate && cp -rp /prc /prc-copy && \
+                  pip install '/prc-copy[tests]' && sudo rm -fr /prc-copy"
+    source /py3/bin/activate
+}
diff --git a/irods/test/scripts/update_json_for_test b/irods/test/scripts/update_json_for_test
new file mode 100644
index 000000000..3dc84dbad
--- /dev/null
+++ b/irods/test/scripts/update_json_for_test
@@ -0,0 +1,55 @@
+#!/bin/bash
+declare -A ssl_keys=(
+    [irods_client_server_negotiation]='"request_server_negotiation"'
+    [irods_client_server_policy]='"CS_NEG_REQUIRE"'
+    [irods_ssl_ca_certificate_file]='"/etc/irods/ssl/irods.crt"'
+    [irods_ssl_certificate_chain_file]='"/etc/irods/ssl/irods.crt"'
+    [irods_ssl_certificate_key_file]='"/etc/irods/ssl/irods.key"'
+    [irods_ssl_dh_params_file]='"/etc/irods/ssl/dhparams.pem"'
+    [irods_ssl_verify_server]='"cert"'
+)
+
+declare -A pam_keys=(
+    [irods_authentication_scheme]="\"$(pam_auth_string)\""
+)
+declare -A encrypt_keys=(
+    [irods_encryption_key_size]=16
+    [irods_encryption_salt_size]=8
+    [irods_encryption_num_hash_rounds]=16
+    [irods_encryption_algorithm]='"AES-256-CBC"'
+)
+
+declare -A RESTORE_FILES=()
+
+update_json_file() {
+    local file=$1 content=$2
+    local bn=$(basename "$file")
+    local orig=/tmp/$bn.orig.$$
+    local newfile=/tmp/$bn.new.$$
+    echo "$content" >"$newfile"
+    sudo chmod --reference "$file" "$newfile"
+    sudo chown --reference "$file" "$newfile"
+    { sudo mv "$file" "$orig" && sudo mv "$newfile" "$file"; } || return 1
+    RESTORE_FILES["$file"]="$orig"
+}
+
+restore_json_files() {
+    local kk
+    for kk in ${!RESTORE_FILES[@]};do
+        sudo mv -f "${RESTORE_FILES["$kk"]}" "$kk"
+    done
+}
+
+newcontent () {
+  local file=$1
+  shift
+  local j=$(sudo cat "$file")
+  while [ $# -gt 0 ]; do
+    eval '
+    for kk in ${!'$1'[@]}; do
+        j=$(jq ".$kk=${'$1'[$kk]}" <<<"$j")
+    done'
+    shift
+  done
+  echo "$j"
+}

From 1a6ce6463907475f61d501046ab19b5f3909d8b6 Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Tue, 18 Mar 2025 09:21:46 -0400
Subject: [PATCH 02/13] extra time parameter to age_out_pam_password

---
 irods/test/scripts/test_support_functions | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions
index 73a2db385..9144b20b7 100644
--- a/irods/test/scripts/test_support_functions
+++ b/irods/test/scripts/test_support_functions
@@ -154,10 +154,17 @@ test_specific_cleanup() {
 
 # PostgreSQL only
 age_out_pam_password() {
+  # sets create_ts and modify_ts (timestamps) to older values, decreasing them by an amount of (offset + 1) where offset
+  # is the number of seconds for expiry_ts stored in the ICAT for the given user and password.  In this way, we can
+  # artificially age out an existing pam password.
+  # Parameters:
+  #  $1 - The username
+  #  $2 - (optional) override the amount used for offsetting the create & modify timestamps.
   local id=$(iquest %s "select USER_ID where USER_NAME = '$1'")
   local offset=$(sudo su - postgres -c "psql -t ICAT -c 'select pass_expiry_ts from r_user_password where user_id = $id'")
   local mtime=$(sudo su - postgres -c "psql -t ICAT -c 'select modify_ts from r_user_password where user_id = $id'")
   mtime=$(sed 's/^\s*0//' <<<"$mtime")
+  [ -n "$2" ] && offset="$2"
   ((offset+=1))
   local new_time=$((mtime - offset))
   sudo su - postgres -c "psql ICAT -c 'update r_user_password set create_ts=$new_time, modify_ts=$new_time where user_id=$id'"

From eaa5395f006bfcee503a826f270fa91c19d0779b Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Tue, 18 Mar 2025 09:23:43 -0400
Subject: [PATCH 03/13] SKIP_IINIT_FOR_PASSWORD option for pam user env setup

---
 irods/test/scripts/test_support_functions | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions
index 9144b20b7..e1dada01e 100644
--- a/irods/test/scripts/test_support_functions
+++ b/irods/test/scripts/test_support_functions
@@ -114,7 +114,7 @@ _begin_pam_environment_and_password() {
     echo "$ENV" > ~/.irods/irods_environment.json
 
     # TODO: check: it seems /dev/tty won't work if docker exec is not invoked with -t
-    if [ -n "$1" ]; then
+    if [ -n "$1" -a -z "$SKIP_IINIT_FOR_PASSWORD" ]; then
         iinit <<<"$1" 2>/tmp/iinit_as_alice.log
     fi
 }

From 9f0286c30e3d87936ed6fefd561368a0023425bf Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Tue, 18 Mar 2025 09:24:59 -0400
Subject: [PATCH 04/13] add mtime_and_content

---
 irods/test/scripts/test_support_functions | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions
index e1dada01e..5994b70c5 100644
--- a/irods/test/scripts/test_support_functions
+++ b/irods/test/scripts/test_support_functions
@@ -199,3 +199,9 @@ activate_virtual_env_with_prc_installed()
                   pip install '/prc-copy[tests]' && sudo rm -fr /prc-copy"
     source /py3/bin/activate
 }
+
+mtime_and_content()
+{
+    stat -c%y "$1"
+    cat "$1"
+}

From 4421f804fc7f8ff145d358c50007ba72b336d260 Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Tue, 18 Mar 2025 09:23:59 -0400
Subject: [PATCH 05/13] add tests 001 thru 009 to test params

---
 irods/test/harness/test_script_parameters | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters
index 77ddb3032..67faf3e09 100644
--- a/irods/test/harness/test_script_parameters
+++ b/irods/test/harness/test_script_parameters
@@ -13,8 +13,16 @@ declare -A wrapper_arglist=(
 declare -A wrappers=(
     [login_auth_test_must_run_manually.py]=./login_auth_test.sh
     [PRC_issue_362.bats]=./login_auth_test.sh
-    [test001_pam_password_expiration.bats]=../login_auth_test.sh
     [demo.sh]=./demo_hook.sh
+    [test001_pam_password_expiration.bats]=../login_auth_test.sh
+    [test002_write_native_credentials_to_secrets_file.bats]=../login_auth_test.sh
+    [test003_write_pam_credentials_to_secrets_file.bats]=../login_auth_test.sh
+    [test004_prc_pam_password_internal_secrets_file_generation.bats]=../login_auth_test.sh
+    [test005_test_special_characters_in_pam_passwords.bats]=../login_auth_test.sh
+    [test006_connection_timeout_on_ssl_socket.bats]=../login_auth_test.sh
+    [test007_pam_features_in_new_auth_framework.bats]=../login_auth_test.sh
+    [test008_prc_write_irodsA_utility_in_native_mode.bats]=../login_auth_test.sh
+    [test009_test_special_characters_in_pam_passwords_auth_fmwk.bats]=../login_auth_test.sh
 )
 
 # keys for Image and User refer to the basename after resolution to a wrapper if one is used

From 3832ff0bc489e77be214885626e8c40c0e70611c Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Mon, 17 Mar 2025 12:23:21 -0400
Subject: [PATCH 06/13] add irods_server_version to test_support_functions

---
 irods/test/scripts/test_support_functions | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/irods/test/scripts/test_support_functions b/irods/test/scripts/test_support_functions
index 5994b70c5..a6c2d8fec 100644
--- a/irods/test/scripts/test_support_functions
+++ b/irods/test/scripts/test_support_functions
@@ -205,3 +205,19 @@ mtime_and_content()
     stat -c%y "$1"
     cat "$1"
 }
+
+irods_server_version() {
+  python -c "import irods.helpers as h
+import operator,sys
+if len(sys.argv) == 1:
+    (comparison,relto)=('','')
+elif len(sys.argv) == 3:
+    (comparison,relto)=sys.argv[1:3]
+fm_tuple = lambda tup: '.'.join(str(_) for  _ in tup)
+to_tuple = lambda vstr: tuple(int(_) for _ in vstr.split('.'))
+svt = h.make_session().server_version_without_auth()
+if relto:
+  exit(0 if vars(operator)[comparison](svt,to_tuple(relto)) else 1)
+print(fm_tuple(svt))
+" $1 $2
+}

From cd9e1bb9aafa46fc2725e07ca824b0d79e2dc06a Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Wed, 17 Sep 2025 07:45:53 -0400
Subject: [PATCH 07/13] can now give exact filename of dockerfiles to build

---
 irods/test/harness/build-docker.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/irods/test/harness/build-docker.sh b/irods/test/harness/build-docker.sh
index 252b87fef..3a5c28ecf 100755
--- a/irods/test/harness/build-docker.sh
+++ b/irods/test/harness/build-docker.sh
@@ -6,8 +6,13 @@
 BASE=$(basename "$0")
 DIR=$(realpath "$(dirname "$0")")
 cd "$DIR"
-DOCKER=docker
-for dockerfile in [0-9]*.Dockerfile; do 
+: ${DOCKER:=docker}
+if [ $# -gt 0 ]; then
+    ARGS=("$@")
+else
+    ARGS=([0-9]*.Dockerfile)
+fi
+for dockerfile in "${ARGS[@]}"; do 
     image_name=${dockerfile#[0-9]*_}
     image_name=${image_name%.Dockerfile}
     if [ "$image_name" = "install-irods" ];then

From 99cf753ce65ee1a724d57c3a869d968606e56842 Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Thu, 18 Sep 2025 08:41:22 -0400
Subject: [PATCH 08/13] firm up package defs; allow run under podman

---
 .../test/harness/000_install-irods.Dockerfile |  2 +-
 irods/test/harness/docker_container_driver.sh | 20 +++++++-------
 irods/test/harness/install.sh                 | 26 +++++++++++++++----
 3 files changed, 33 insertions(+), 15 deletions(-)

diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile
index 7071aa1e0..50f95a577 100644
--- a/irods/test/harness/000_install-irods.Dockerfile
+++ b/irods/test/harness/000_install-irods.Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 COPY install.sh /
 ARG irods_package_version
 ENV IRODS_PACKAGE_VERSION "$irods_package_version"
diff --git a/irods/test/harness/docker_container_driver.sh b/irods/test/harness/docker_container_driver.sh
index c340360ff..c5041bc4e 100755
--- a/irods/test/harness/docker_container_driver.sh
+++ b/irods/test/harness/docker_container_driver.sh
@@ -79,30 +79,32 @@ ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$(realpath --relative-to $reporoot "$original_t
 echo "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT]" 
 INNER_MOUNT=/prc
 
+: ${DOCKER:=docker}
+
 # Start the container.
 echo image="[$image]"
-CONTAINER=$(docker run -d -v $reporoot:$INNER_MOUNT:ro --rm $image)
+CONTAINER=$($DOCKER run -d -v $reporoot:$INNER_MOUNT:ro --rm $image)
 
 # Wait for iRODS and database to start up.
 TIME0=$(date +%s)
 while :; do
     [ `date +%s` -gt $((TIME0 + 30)) ] && { echo >&2 "Waited too long for DB and iRODS to start"; exit 124; }
     sleep 1 
-    docker exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null
+    $DOCKER exec $CONTAINER grep '(0)' /tmp/irods_status 2>/dev/null >/dev/null
     [ $? -ne 0 ] && { echo -n . >&2; continue; }
     break
 done
 
-docker exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \
-            ${WORKDIR:+"-w$WORKDIR"} \
-	    -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \
-            $CONTAINER \
-            $INNER_MOUNT/$(realpath --relative-to $reporoot "$testscript_abspath") \
-            $arglist
+$DOCKER exec ${RUN_AS_USER:+"-u$RUN_AS_USER"} \
+             ${WORKDIR:+"-w$WORKDIR"} \
+	     -e "ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=$ORIGINAL_SCRIPT_RELATIVE_TO_ROOT" \
+             $CONTAINER \
+             $INNER_MOUNT/$(realpath --relative-to $reporoot "$testscript_abspath") \
+             $arglist
 STATUS=$?
 
 if [ $((0+KILL_TEST_CONTAINER)) -ne 0 ]; then
-    echo >&2 'Killed:' $(docker stop --time=0 $CONTAINER)
+    echo >&2 'Killed:' $($DOCKER stop --time=0 $CONTAINER)
 fi
 
 [ -n "$ECHO_CONTAINER" ] && echo $CONTAINER
diff --git a/irods/test/harness/install.sh b/irods/test/harness/install.sh
index cd840a98a..d81a3f852 100755
--- a/irods/test/harness/install.sh
+++ b/irods/test/harness/install.sh
@@ -16,10 +16,25 @@ add_package_repo()
       sudo apt update
 }
 
-DIST_NAME=$(lsb_release -sc)
-
+DIST_NAME=$(grep '^VERSION_CODENAME=' /etc/os-release|sed 's/.*=//')
 : ${IRODS_VSN:=4.3.1-0~$DIST_NAME}
 
+# Expand a spec of the leading version tuple eg. 4.3.4 out  to the full name of
+# the most recent matching version of the package
+
+irods_package_vsn() {
+  apt list -a irods-server 2>/dev/null|awk '{print $2}'|grep '\w'|sort|\
+      grep "$(perl -e 'print quotemeta($ARGV[0])' "$IRODS_PACKAGE_VERSION")"|tail -1
+}
+
+# Report irods server version installed, or failing that, the requested version.
+
+irods_vsn() {
+  local V=$(dpkg -l irods-server 2>/dev/null|grep '^ii\s'|awk '{print $3}')
+  V=
+  echo "${V:-$IRODS_VSN}"
+}
+
 while [[ "$1" = -* ]]; do
   ARG="$1"
   shift
@@ -101,14 +116,15 @@ ________"
     ;;
 
  4)
-   sudo apt install -y irods-{dev,runtime}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"}
+   IRODS_TO_INSTALL=`irods_package_version`
+   sudo apt install -y irods-{dev,runtime}${IRODS_TO_INSTALL=:+"=$IRODS_TO_INSTALL="}
    if [[ $with_opts != *\ basic\ * ]]; then
-     sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_PACKAGE_VERSION:+"=$IRODS_PACKAGE_VERSION"}
+     sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_TO_INSTALL=:+"=$IRODS_TO_INSTALL="}
    fi
  ;;
 
  5)
- if [ ! "$IRODS_VSN" '<' "4.3" ]; then
+ if [ ! `irods_vsn` '<' "4.3" ]; then
     PYTHON=python3
  else
     PYTHON=python2

From 6cf1bc7db9445e80e62d2952eb365d20a56e8c37 Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Thu, 18 Sep 2025 15:02:31 -0400
Subject: [PATCH 09/13] correct test009 name

---
 irods/test/harness/test_script_parameters | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters
index 67faf3e09..d5b21d527 100644
--- a/irods/test/harness/test_script_parameters
+++ b/irods/test/harness/test_script_parameters
@@ -22,7 +22,7 @@ declare -A wrappers=(
     [test006_connection_timeout_on_ssl_socket.bats]=../login_auth_test.sh
     [test007_pam_features_in_new_auth_framework.bats]=../login_auth_test.sh
     [test008_prc_write_irodsA_utility_in_native_mode.bats]=../login_auth_test.sh
-    [test009_test_special_characters_in_pam_passwords_auth_fmwk.bats]=../login_auth_test.sh
+    [test009_test_special_characters_in_pam_passwords_auth_framework.bats]=../login_auth_test.sh
 )
 
 # keys for Image and User refer to the basename after resolution to a wrapper if one is used

From 24c710e0cfe94df8aa1880d0135d36cd3db32395 Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Fri, 19 Sep 2025 01:31:52 -0400
Subject: [PATCH 10/13] bats tests - IRODS version macros, params

---
 irods/test/harness/000_install-irods.Dockerfile | 2 +-
 irods/test/harness/install.sh                   | 6 +++---
 irods/test/harness/test_script_parameters       | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/irods/test/harness/000_install-irods.Dockerfile b/irods/test/harness/000_install-irods.Dockerfile
index 50f95a577..a978aafba 100644
--- a/irods/test/harness/000_install-irods.Dockerfile
+++ b/irods/test/harness/000_install-irods.Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 COPY install.sh /
 ARG irods_package_version
 ENV IRODS_PACKAGE_VERSION "$irods_package_version"
diff --git a/irods/test/harness/install.sh b/irods/test/harness/install.sh
index d81a3f852..9e8f79c9e 100755
--- a/irods/test/harness/install.sh
+++ b/irods/test/harness/install.sh
@@ -116,10 +116,10 @@ ________"
     ;;
 
  4)
-   IRODS_TO_INSTALL=`irods_package_version`
-   sudo apt install -y irods-{dev,runtime}${IRODS_TO_INSTALL=:+"=$IRODS_TO_INSTALL="}
+   IRODS_TO_INSTALL=`irods_package_vsn`
+   sudo apt install -y irods-{dev,runtime}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"}
    if [[ $with_opts != *\ basic\ * ]]; then
-     sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_TO_INSTALL=:+"=$IRODS_TO_INSTALL="}
+     sudo apt install -y irods-{icommands,server,database-plugin-postgres}${IRODS_TO_INSTALL:+"=$IRODS_TO_INSTALL"}
    fi
  ;;
 
diff --git a/irods/test/harness/test_script_parameters b/irods/test/harness/test_script_parameters
index d5b21d527..a4abdf42e 100644
--- a/irods/test/harness/test_script_parameters
+++ b/irods/test/harness/test_script_parameters
@@ -3,7 +3,7 @@
 declare -A wrapper_arglist=(
     [demo.sh]="arg1 arg2"
     [demo_A.sh]="arg1-a arg2-a"
-    [login_auth_test.py]="TestLogins"
+    [login_auth_test_must_run_manually.py]="-v TestLogins"
     [login_auth_test_1.py]="-v TestAnonymousUser TestMiscellaneous"
     [login_auth_test_2.py]="-v TestWithSSL"
 )

From a455662452195f344fdc1802f915699cfce009af Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Fri, 19 Sep 2025 01:40:22 -0400
Subject: [PATCH 11/13] README.txt

---
 irods/test/harness/README.txt | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/irods/test/harness/README.txt b/irods/test/harness/README.txt
index 31f848963..7883bfdf3 100644
--- a/irods/test/harness/README.txt
+++ b/irods/test/harness/README.txt
@@ -4,16 +4,16 @@ To build required images
 ------------------------
 Examples
 
-    1) ./build-docker.sh 
+    1) ./build-docker.sh
        DEFAULT: build  single-node system based on latest iRODS release
 
-    2) IRODS_PACKAGE_VERSION="4.2.12-1~bionic" NO_CACHE='1' ./build-docker.sh 
+    2) IRODS_PACKAGE_VERSION=4.2.12 NO_CACHE='1' ./build-docker.sh [ ... optional in-directory dockerfiles in sequence ... ]
        Build (ignoring docker cache) single-node system based on specified package version string.
 
 simple examples
 ---------------
-./docker_container_driver.sh  tests/test_1.sh 
-./docker_container_driver.sh  tests/test_2.sh 
+./docker_container_driver.sh  tests/test_1.sh
+./docker_container_driver.sh  tests/test_2.sh
 
 Any script in a subdirectory of the repo (mounted at /prc within the container) can be
 executed and will be able to find other scripts and source include files within the tree.
@@ -26,13 +26,14 @@ Examples of options in driver script
      C=$(  ./docker_container_driver.sh -c -L -u testuser  ../scripts/experiment.sh )
 
   2. To manually examine results afterward:
-     docker exec -it $C bash 
+     docker exec -it $C bash
 
+For both scripts, the environment variable DOCKER may be set to "podman" to run the alternative virtualizer.
 
 Demo / Demo hook  / args
 ------------------------
 
-$ ~/python-irodsclient/irods/test/harness$ ./docker_container_driver.sh  ../demo.sh 
+$ ~/python-irodsclient/irods/test/harness$ ./docker_container_driver.sh  ../demo.sh
 ORIGINAL_SCRIPT_RELATIVE_TO_ROOT=[irods/test/demo.sh]
 image=[ssl-and-pam]
 .......-- HOOK RUNNING --

From 63cc566b80de2ac81efaf8a449d8aef10856e3dc Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Fri, 19 Sep 2025 08:59:46 -0400
Subject: [PATCH 12/13] version-string-driven building of python interpreter
 builder image

---
 .../003_compile-specific-python.Dockerfile    | 15 +++++++++++
 .../harness/003_prc-with-py3_12_2.Dockerfile  | 14 -----------
 irods/test/harness/build-docker.sh            | 19 +++++++++++---
 irods/test/harness/most_recent_python.sh      | 25 +++++++++++++++++++
 4 files changed, 55 insertions(+), 18 deletions(-)
 create mode 100644 irods/test/harness/003_compile-specific-python.Dockerfile
 delete mode 100644 irods/test/harness/003_prc-with-py3_12_2.Dockerfile
 create mode 100755 irods/test/harness/most_recent_python.sh

diff --git a/irods/test/harness/003_compile-specific-python.Dockerfile b/irods/test/harness/003_compile-specific-python.Dockerfile
new file mode 100644
index 000000000..6baa0d85f
--- /dev/null
+++ b/irods/test/harness/003_compile-specific-python.Dockerfile
@@ -0,0 +1,15 @@
+from ssl-and-pam
+run apt update
+run apt install -y wget build-essential
+run apt install -y libssl-dev zlib1g-dev libffi-dev libncurses-dev wget build-essential
+arg python_version
+run wget https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tar.xz
+run tar xf Python-${python_version}.tar.xz
+workdir /Python-${python_version}
+run ./configure --prefix /root/python --with-ensurepip=install
+run make -j
+run mkdir /root/python
+run make install
+workdir /
+run /root/python/bin/python3 -m pip install python-irodsclient
+run chmod a+rx /root
diff --git a/irods/test/harness/003_prc-with-py3_12_2.Dockerfile b/irods/test/harness/003_prc-with-py3_12_2.Dockerfile
deleted file mode 100644
index 9a77a104a..000000000
--- a/irods/test/harness/003_prc-with-py3_12_2.Dockerfile
+++ /dev/null
@@ -1,14 +0,0 @@
-from install-irods
-run apt update
-run apt install -y wget build-essential libssl-dev zlib1g-dev
-run apt install wget build-essential
-run wget https://www.python.org/ftp/python/3.12.2/Python-3.12.2.tar.xz
-run tar xf Python-3.12.2.tar.xz
-workdir /Python-3.12.2
-run ./configure --prefix /root/python --with-ensurepip=install
-run make -j
-run mkdir /root/python
-run make install
-workdir /
-run /root/python/bin/python3 -m pip install python-irodsclient
-run chmod a+rx /root
diff --git a/irods/test/harness/build-docker.sh b/irods/test/harness/build-docker.sh
index 3a5c28ecf..c005f0c75 100755
--- a/irods/test/harness/build-docker.sh
+++ b/irods/test/harness/build-docker.sh
@@ -1,7 +1,10 @@
 #!/usr/bin/env bash
 
-# IRODS_PACKAGE_VERSION if defined is like "4.3.1-0~bionic"
+# environment variables for build
+# IRODS_PACKAGE_VERSION if defined is like "4.3.4" or "5.0.1".
 # (but contains no '~' suffix for irods versions <= 4.2.10)
+# PYTHON_VERSION is usually two dot-separated numbers: example "3.13", but could also have zero, one or three version numbers.
+# (Do not specify the triple form, X.Y.Z, if that release is not known to exist - not counting alphas and release candidates)
 
 BASE=$(basename "$0")
 DIR=$(realpath "$(dirname "$0")")
@@ -15,12 +18,20 @@ fi
 for dockerfile in "${ARGS[@]}"; do 
     image_name=${dockerfile#[0-9]*_}
     image_name=${image_name%.Dockerfile}
-    if [ "$image_name" = "install-irods" ];then
-        package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"}
+    irods_package_version_option=""
+    python_version_option=""
+    if [ "$image_name" = "install-irods" ]; then
+        irods_package_version_option=${IRODS_PACKAGE_VERSION:+"--build-arg=irods_package_version=$IRODS_PACKAGE_VERSION"}
+    elif [ "$image_name" = "compile-specific-python" ]; then
+        temp=$(./most_recent_python.sh $PYTHON_VERSION)
+        if [ -n "$temp" ]; then
+            PYTHON_VERSION="$temp"
+        fi
+        python_version_option=${PYTHON_VERSION:+"--build-arg=python_version=$PYTHON_VERSION"}
     else
         package_version_option=""
     fi
-    $DOCKER build -f $dockerfile -t $image_name . $package_version_option \
+    $DOCKER build -f $dockerfile -t $image_name . $irods_package_version_option $python_version_option \
                                                   ${NO_CACHE+"--no-cache"} || 
                                                   { STATUS=$?; echo "*** Failure while building [$image_name]"; exit $STATUS; }
 done
diff --git a/irods/test/harness/most_recent_python.sh b/irods/test/harness/most_recent_python.sh
new file mode 100755
index 000000000..278f3ce7c
--- /dev/null
+++ b/irods/test/harness/most_recent_python.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+usage() {
+    echo >&2 "Usage:
+    $0 major.minor"
+    echo >&2 "Output:
+    prints full latest python version inclusive of the patch level."
+    exit 2
+}
+MAJOR_MINOR=$1
+if [ -z "${MAJOR_MINOR}" ]; then                      # allow blank specification: most recent overall
+    MAJOR_MINOR='[0-9]\+\.[0-9]\+'
+elif [[ $MAJOR_MINOR =~ ^[0-9]+$ ]]; then             # allow single integer, eg. 3 for most recent 3.y.z
+    MAJOR_MINOR+='\.[0-9]\+'
+elif [[ $MAJOR_MINOR =~ [0-9]+\.[0-9]+ ]]; then       # allow x.y form, will yield output of most recent x.y.z
+    MAJOR_MINOR=$(sed 's/\./\\./'<<<"${MAJOR_MINOR}") # insert backslash in front of "."
+elif ! [[ $MAJOR_MINOR =~ [0-9]+\\?.[0-9]+ ]]; then
+    usage
+fi
+
+url='https://www.python.org/ftp/python/'
+
+# Fetch the directory listing, extract version numbers, sort them to find the largest numerically.
+curl --silent "$url"|\
+sed -n 's!.*href="\('"${MAJOR_MINOR}"'\.[0-9]\+\)/".*!\1!p'|sort -rV|\
+head -n 1

From 62be0025f81de2520029a480d3f51b2306f042b9 Mon Sep 17 00:00:00 2001
From: d-w-moore <dmoore@renci.org>
Date: Mon, 22 Sep 2025 13:57:26 -0400
Subject: [PATCH 13/13] script to start server under iRODS 5

---
 irods/test/harness/manage_irods5_procs | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100755 irods/test/harness/manage_irods5_procs

diff --git a/irods/test/harness/manage_irods5_procs b/irods/test/harness/manage_irods5_procs
new file mode 100755
index 000000000..06baa8322
--- /dev/null
+++ b/irods/test/harness/manage_irods5_procs
@@ -0,0 +1,10 @@
+if [ "$1" = "start" ]; then
+  sudo su - irods -c 'irodsServer -d -p /tmp/irods.pid'
+elif [ "$1" = "status" ]; then
+  pgrep -afl "irods(Delay|Agent|Server)"
+elif [ "$1" = "stop" ]; then
+  sudo su - irods -c 'kill -QUIT $(cat /tmp/irods.pid)'
+else
+  echo >&2 "usage: $0 [start|status|stop]"
+  exit 2
+fi