ci: add task tests

.github/workflows/checks.yaml

@@ -25,3 +25,48 @@ jobs:
         run: poetry run ruff check tests/* crypt4gh_middleware/*
       - name: Type checking with mypy
         run: poetry run mypy crypt4gh_middleware
+
+  test:
+    name: Run tests
+    runs-on: ubuntu-latest
+    services:
+      funnel:
+        image: athitheyag/funnel:test
+        ports:
+          - 8000:8000
+      minio:
+        image: minio/minio:edge-cicd
+        ports:
+          - 9000:9000
+        env:
+          MINIO_ROOT_USER: minioadmin
+          MINIO_ROOT_PASSWORD: minioadmin
+    steps:
+      - name: Fetch mc client
+        run: curl -o ./mc -# https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x ./mc
+      - name: Create input and output buckets
+        run: | 
+          ./mc alias set minio http://localhost:9000 minioadmin minioadmin \
+          && ./mc mb minio/inputs \
+          && ./mc mb minio/outputs
+      - name: Set bucket permissions to public
+        run: |
+          ./mc anonymous set public minio/inputs \
+          && ./mc anonymous set public minio/outputs
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Upload input files
+        run: |
+            for file in inputs/*; do
+              curl http://localhost:9000/inputs/$(basename $file) --upload-file $file
+            done
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install poetry
+        run: pip install poetry
+      - name: Install dependencies
+        run: poetry install
+      - name: Run tests
+        run: poetry run pytest tests/tasks

tests/tasks/task_bodies.py

@@ -1,26 +1,26 @@
 """TES task request bodies used by tests"""
 
-from tests.utils import INPUT_DIR
+from tests.utils import S3_BUCKET_ENDPOINT
 
-def get_uppercase_task_body(tmp_dir):
+def get_uppercase_task_body(output_bucket_name):
     """Returns TES task body that makes the contents of a file uppercase."""
     return {
         "name": "Hello world",
         "inputs": [
             {
-                "url": f"file://{INPUT_DIR}/hello.txt",
+                "url": f"s3://inputs/hello.txt",
                 "path": "/inputs/hello.txt",
                 "type": "FILE"
             },
             {
-                "url": f"file://{INPUT_DIR}/make_uppercase.py",
+                "url": f"s3://inputs/make_uppercase.py",
                 "path": "/inputs/make_uppercase.py",
                 "type": "FILE"
             }
         ],
         "outputs": [
             {
-                "url": f"file://{tmp_dir}/hello-upper.txt",
+                "url": f"s3://{output_bucket_name}/hello-upper.txt",
                 "path": "/outputs/hello-upper.txt",
                 "type": "FILE"
             }
@@ -39,30 +39,30 @@ def get_uppercase_task_body(tmp_dir):
     }
 
 
-def get_decryption_task_body(tmp_dir):
+def get_decryption_task_body(output_bucket_name):
     """Returns TES task body that decrypts a crypt4GH file."""
     return {
         "name": "Decrypt with secret key as environment variable",
         "inputs": [
             {
-                "url": f"file://{INPUT_DIR}/hello.c4gh",
+                "url": f"http://{S3_BUCKET_ENDPOINT}/inputs/hello.c4gh",
                 "path": "/inputs/hello.c4gh",
                 "type": "FILE"
             },
             {
-                "url": f"file://{INPUT_DIR}/alice.sec",
+                "url": f"http://{S3_BUCKET_ENDPOINT}/inputs/alice.sec",
                 "path": "/inputs/alice.sec",
                 "type": "FILE"
             },
             {
-                "url": f"file://{INPUT_DIR}/decrypt.sh",
+                "url": f"http://{S3_BUCKET_ENDPOINT}/inputs/decrypt.sh",
                 "path": "/inputs/decrypt.sh",
                 "type": "FILE"
             }
         ],
         "outputs": [
             {
-                "url": f"file://{tmp_dir}/hello-decrypted.txt",
+                "url": f"s3://{output_bucket_name}/hello-decrypted.txt",
                 "path": "/outputs/hello-decrypted.txt",
                 "type": "FILE"
             }
@@ -81,35 +81,35 @@ def get_decryption_task_body(tmp_dir):
     }
 
 
-def get_uppercase_task_with_decryption_body(tmp_dir):
+def get_uppercase_task_with_decryption_body(output_bucket_name):
     """Returns TES task body that makes the contents of a crypt4GH file uppercase."""
     return {
         "name": "Decrypt with secret key as environment variable",
         "inputs": [
             {
-                "url": f"file://{INPUT_DIR}/hello.c4gh",
+                "url": f"http://{S3_BUCKET_ENDPOINT}/inputs/hello.c4gh",
                 "path": "/inputs/hello.c4gh",
                 "type": "FILE"
             },
             {
-                "url": f"file://{INPUT_DIR}/alice.sec",
+                "url": f"http://{S3_BUCKET_ENDPOINT}/inputs/alice.sec",
                 "path": "/inputs/alice.sec",
                 "type": "FILE"
             },
             {
-                "url": f"file://{INPUT_DIR}/decrypt.sh",
+                "url": f"http://{S3_BUCKET_ENDPOINT}/inputs/decrypt.sh",
                 "path": "/inputs/decrypt.sh",
                 "type": "FILE"
             },
             {
-                "url": f"file://{INPUT_DIR}/make_uppercase.py",
+                "url": f"http://{S3_BUCKET_ENDPOINT}/inputs/make_uppercase.py",
                 "path": "/inputs/make_uppercase.py",
                 "type": "FILE"
             }
         ],
         "outputs": [
             {
-                "url": f"file://{tmp_dir}/hello-upper-decrypt.txt",
+                "url": f"s3://{output_bucket_name}/hello-upper-decrypt.txt",
                 "path": "/outputs/hello-upper-decrypt.txt",
                 "type": "FILE"
             }

tests/tasks/test_tasks.py

@@ -2,7 +2,6 @@
 
 import json
 from pathlib import Path
-from tempfile import TemporaryDirectory
 from time import sleep
 
 import pytest
@@ -13,9 +12,9 @@
     get_uppercase_task_body,
     get_uppercase_task_with_decryption_body,
 )
-from tests.utils import INPUT_TEXT, timeout
+from tests.utils import INPUT_TEXT, S3_BUCKET_ENDPOINT, timeout
 
-TES_URL = "http://localhost:8090/ga4gh/tes/v1"
+TES_URL = "http://localhost:8000/v1"
 HEADERS = {"accept": "application/json", "Content-Type": "application/json"}
 WAIT_STATUSES = ("UNKNOWN", "INITIALIZING", "RUNNING", "QUEUED")
 
@@ -26,17 +25,17 @@ def wait_for_file_download(filename, output_path):
         sleep(1)
 
 
-@pytest.fixture(name="output_dir")
-def fixture_output_dir():
+@pytest.fixture(name="output_bucket")
+def fixture_output_bucket():
     """Returns temporary directory to store task outputs."""
-    return Path(TemporaryDirectory().name)
+    return Path("outputs")
 
 
 @pytest.fixture(name="task")
-def fixture_task(request, output_dir):
+def fixture_task(request, output_bucket):
     """Returns response received after creating task."""
     return requests.post(
-        url=f"{TES_URL}/tasks", headers=HEADERS, json=request.param(output_dir)
+        url=f"{TES_URL}/tasks", headers=HEADERS, json=request.param(output_bucket)
     )
 
 
@@ -65,13 +64,11 @@ def fixture_final_task_info(task):
     (get_uppercase_task_with_decryption_body, "hello-upper-decrypt.txt", INPUT_TEXT.upper())
 ], indirect=["task"])
 @timeout(time_limit=10)
-def test_task(task, final_task_info, filename, expected_output, output_dir):  # pylint: disable=unused-argument
+def test_task(task, final_task_info, filename, expected_output, output_bucket):  # pylint: disable=unused-argument
     """Test tasks for successful completion and intended behavior."""
     assert final_task_info["state"] == "COMPLETE"
 
-    wait_for_file_download(filename=filename, output_path=output_dir)
-    with open(output_dir/filename, encoding="utf-8") as f:
-        output = f.read()
-        assert output == expected_output
-        true_result = output.isupper() if "upper" in filename else not output.isupper()
-        assert true_result
+    output = requests.get(url=f"http://{S3_BUCKET_ENDPOINT}/outputs/{filename}").text
+    assert output == expected_output
+    true_result = output.isupper() if "upper" in filename else not output.isupper()
+    assert true_result

tests/utils.py

@@ -7,6 +7,7 @@
 
 INPUT_DIR = Path(__file__).parents[1]/"inputs"
 INPUT_TEXT = "hello world from the input!"
+S3_BUCKET_ENDPOINT = "host.docker.internal:9000"
 
 
 @contextlib.contextmanager