tests: end-to-end with Oras CP (#3)

Signed-off-by: tarilabs <[email protected]>

.github/workflows/e2e.yaml

@@ -37,3 +37,32 @@ jobs:
     - name: Run E2E tests
       run: |
         make test-e2e-skopeo
+  e2e-oras:
+    name: E2E using Oras CP
+    runs-on: ubuntu-24.04
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Install oras
+      uses: oras-project/setup-oras@v1
+    - run: oras version
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9' 
+    - name: Install Poetry
+      run: |
+        pipx install poetry
+    - name: Install dependencies
+      run: |
+        make install
+    - name: Start Kind Cluster
+      uses: helm/kind-action@v1
+      with:
+        cluster_name: kind
+    - name: Start distribution-registry
+      run: |
+        ./e2e/deploy_distribution_registry.sh
+    - name: Run E2E tests
+      run: |
+        make test-e2e-oras

Makefile

@@ -15,6 +15,10 @@ test:
 test-e2e-skopeo:
 	poetry run pytest --e2e-skopeo -s -x -rA
 
+.PHONY: test-e2e-oras
+test-e2e-oras:
+	poetry run pytest --e2e-oras -s -x -rA
+
 .PHONY: lint
 lint: install
 	poetry run ruff check --fix

olot/backend/oras_cp.py

@@ -0,0 +1,24 @@
+import os
+import shutil
+import subprocess
+import typing
+
+def is_oras() -> bool :
+    return shutil.which("oras") is not None
+
+
+def oras_pull(base_image: str, dest: typing.Union[str, os.PathLike]):
+    if isinstance(dest, os.PathLike):
+        dest = str(dest)
+    subprocess.run(["oras", "copy", "--to-oci-layout", base_image, dest+":latest"], check=True)
+    blobs_dir = os.path.join(dest, "blobs", "sha256")
+    for _, _, files in os.walk(blobs_dir):
+        for file in files:
+            os.chmod(os.path.join(blobs_dir, file), 0o664) # TODO eventually avoid this by refactor manifest change logic
+
+
+
+def oras_push(src: typing.Union[str, os.PathLike], oci_ref: str):
+    if isinstance(src, os.PathLike):
+        src = str(src)
+    return subprocess.run(["oras", "copy", "--from-oci-layout", "--to-plain-http", src+":latest", oci_ref], check=True)

pyproject.toml

@@ -23,6 +23,7 @@ pytest = "^8.3.4"
 [tool.pytest.ini_options]
 markers = [
     "e2e_skopeo: end-to-end testing with skopeo",
+    "e2e_oras: end-to-end testing with oras",
 ]
 
 

tests/backend/test_oras_cp.py

@@ -0,0 +1,79 @@
+import os
+import subprocess
+import time
+import docker # type: ignore
+from pathlib import Path
+import pytest
+from olot.backend.oras_cp import is_oras, oras_pull, oras_push
+from olot.basics import check_ocilayout, oci_layers_on_top, read_ocilayout_root_index
+
+@pytest.mark.e2e_oras
+def test_is_oras():
+    assert is_oras()
+
+
+@pytest.mark.e2e_oras
+def test_oras_pull(tmp_path):
+    """Test oras to pull/dl a known base-image to an oci-layout
+    """
+    oras_pull("quay.io/mmortari/hello-world-wait:latest", tmp_path)
+
+    assert check_ocilayout(tmp_path)
+
+    mut = read_ocilayout_root_index(tmp_path)
+    assert mut.schemaVersion == 2
+    assert len(mut.manifests) == 3
+    manifest0 = mut.manifests[0]
+    assert manifest0.mediaType == "application/vnd.oci.image.index.v1+json"
+    assert manifest0.digest == "sha256:d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b"
+    assert manifest0.size == 491
+
+
+@pytest.mark.e2e_oras
+def test_oras_scenario(tmp_path):
+    """Test oras with an end-to-end scenario
+    """
+    oras_pull("quay.io/mmortari/hello-world-wait:latest", tmp_path)
+    model_joblib = Path(__file__).parent / ".." / "data" / "model.joblib"
+    model_files = [
+        model_joblib,
+        Path(__file__).parent / ".." / "data" / "hello.md",
+    ]
+    oci_layers_on_top(tmp_path, model_files)
+    oras_push(tmp_path, "localhost:5001/nstestorg/modelcar:latest")
+
+    # show what has been copied in Container Registry
+    subprocess.run(["skopeo","list-tags","--tls-verify=false","docker://localhost:5001/nstestorg/modelcar"], check=True)
+
+    # copy from Container Registry to Docker daemon for local running the modelcar as-is
+    result = subprocess.run("skopeo inspect --tls-verify=false --raw docker://localhost:5001/nstestorg/modelcar | jq -r '.manifests[] | select(.platform.architecture == \"amd64\") | .digest'", shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    assert result.returncode == 0
+    digest = result.stdout.strip()
+    print(digest)
+    # use by convention the linux/amd64
+    subprocess.run(["skopeo", "copy", "--src-tls-verify=false", f"docker://localhost:5001/nstestorg/modelcar@{digest}", "docker-daemon:localhost:5001/nstestorg/modelcar:latest"], check=True)
+    client = docker.from_env()
+    container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True, remove=True)
+    print(container.logs())
+    _, stat = container.get_archive('/models/model.joblib')
+    print(str(stat["size"]))
+    # assert the model.joblib from the KServe modelcar is in expected location (above) and expected size
+    assert stat["size"] == os.stat(model_joblib).st_size
+    container.kill()
+    max_attempts = 5
+    attempt = 0
+    while attempt < max_attempts:
+        try:
+            if client.containers.get(container.id):
+                container.kill()
+                time.sleep(2**attempt)
+            else:
+                break
+        except docker.errors.NotFound:
+            print("test container terminated")
+        except Exception as e:
+            print(f"Attempt to terminate {attempt + 1} failed: {e}")
+        attempt += 1
+    if attempt == max_attempts:
+        print("Was unable to terminate the test container")
+    client.images.remove("localhost:5001/nstestorg/modelcar")

tests/backend/test_skopeo.py

@@ -1,5 +1,6 @@
 import os
 import subprocess
+import time
 import docker # type: ignore
 from pathlib import Path
 import pytest
@@ -52,9 +53,27 @@ def test_skopeo_scenario(tmp_path):
     # use by convention the linux/amd64
     subprocess.run(["skopeo", "copy", "--src-tls-verify=false", f"docker://localhost:5001/nstestorg/modelcar@{digest}", "docker-daemon:localhost:5001/nstestorg/modelcar:latest"], check=True)
     client = docker.from_env()
-    container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True)
+    container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True, remove=True)
     print(container.logs())
     _, stat = container.get_archive('/models/model.joblib')
     print(str(stat["size"]))
     # assert the model.joblib from the KServe modelcar is in expected location (above) and expected size
     assert stat["size"] == os.stat(model_joblib).st_size
+    container.kill()
+    max_attempts = 5
+    attempt = 0
+    while attempt < max_attempts:
+        try:
+            if client.containers.get(container.id):
+                container.kill()
+                time.sleep(2**attempt)
+            else:
+                break
+        except docker.errors.NotFound:
+            print("test container terminated")
+        except Exception as e:
+            print(f"Attempt to terminate {attempt + 1} failed: {e}")
+        attempt += 1
+    if attempt == max_attempts:
+        print("Was unable to terminate the test container")
+    client.images.remove("localhost:5001/nstestorg/modelcar")

tests/conftest.py

@@ -2,23 +2,26 @@
 
 
 def pytest_collection_modifyitems(config, items):
-    for item in items:
+    for item in items: 
         skip_e2e_skopeo = pytest.mark.skip(
             reason="this is an end-to-end test, requires explicit opt-in --e2e-skopeo option to run."
         )
+        skip_e2e_oras = pytest.mark.skip(
+            reason="this is an end-to-end test, requires explicit opt-in --e2e-oras option to run."
+        )
         skip_not_e2e = pytest.mark.skip(
             reason="skipping non-e2e tests; opt-out of --e2e -like options to run."
         )
         if "e2e_skopeo" in item.keywords:
             if not config.getoption("--e2e-skopeo"):
                 item.add_marker(skip_e2e_skopeo)
             continue
-        # elif "e2e_model_registry" in item.keywords:
-        #     if not config.getoption("--e2e-model-registry"):
-        #         item.add_marker(skip_e2e_model_registry)
-        #     continue
+        elif "e2e_oras" in item.keywords:
+            if not config.getoption("--e2e-oras"):
+                item.add_marker(skip_e2e_oras)
+            continue
 
-        if config.getoption("--e2e-skopeo"): # or config.getoption("--e2e-model-registry"):
+        if config.getoption("--e2e-skopeo") or config.getoption("--e2e-oras"):
             item.add_marker(skip_not_e2e)
 
 
@@ -29,3 +32,9 @@ def pytest_addoption(parser):
         default=False,
         help="opt-in to run tests marked with e2e_skopeo",
     )
+    parser.addoption(
+        "--e2e-oras",
+        action="store_true",
+        default=False,
+        help="opt-in to run tests marked with e2e_oras",
+    )