core: modelcard in its layer (#4)

* core: modelcard in its layer Signed-off-by: tarilabs <[email protected]> * fix mypy errors Signed-off-by: tarilabs <[email protected]> --------- Signed-off-by: tarilabs <[email protected]>
containers · Dec 19, 2024 · 3ef324c · 3ef324c
1 parent 1efd485
commit 3ef324c
Show file tree

Hide file tree

Showing 4 changed files with 333 additions and 8 deletions.
diff --git a/olot/basics.py b/olot/basics.py
@@ -4,7 +4,8 @@
 from pathlib import Path
 from pprint import pprint
 import tarfile
-from typing import Dict
+from typing import Dict, List
+import typing
 import click
 import gzip
 
@@ -39,34 +40,40 @@ def get_file_hash(path) -> str:
     return h.hexdigest()
 
 
-def oci_layers_on_top(ocilayout: Path, model_files):
+def oci_layers_on_top(ocilayout: Path, model_files: List[os.PathLike], modelcard: typing.Union[os.PathLike, None] = None):
     check_ocilayout(ocilayout)
     ocilayout_root_index = read_ocilayout_root_index(ocilayout)
     ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout, ocilayout_root_index)
     ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout, ocilayout_indexes)
-    new_layers = []
+    new_layers = {} # layer digest : diff_id
     for model in model_files:
         model = Path(model)
         new_layer = tar_into_ocilayout(ocilayout, model)
-        new_layers.append(new_layer)
+        new_layers[new_layer] = new_layer
+    if modelcard is not None:
+        modelcard_layer_diffid = targz_into_ocilayout(ocilayout, Path(modelcard))
+        new_layers[modelcard_layer_diffid[0]] = modelcard_layer_diffid[1]
     new_ocilayout_manifests: Dict[str, str] = {}
     for manifest_hash, manifest in ocilayout_manifests.items():
         print(manifest_hash, manifest.mediaType)
         config_sha = manifest.config.digest.removeprefix("sha256:")
         mc = None
         with open(ocilayout / "blobs" / "sha256" / config_sha, "r") as cf:
             mc = OCIManifestConfig.model_validate_json(cf.read())
-        for layer in new_layers:
+        for layer, diffid in new_layers.items():
             size = os.stat(ocilayout / "blobs" / "sha256" / layer).st_size
+            mt = "application/vnd.oci.image.layer.v1.tar" if layer == diffid else "application/vnd.oci.image.layer.v1.tar+gzip"
+            la = None if layer == diffid else {"io.opendatahub.temp.layer.type":"modelcard"}
             cd = ContentDescriptor(
-                mediaType="application/vnd.oci.image.layer.v1.tar",
+                mediaType=mt,
                 digest="sha256:"+layer,
                 size=size,
                 urls=None,
                 data=None,
-                artifactType=None           
+                artifactType=None,
+                annotations=la
             )
-            mc.rootfs.diff_ids.append("sha256:"+layer)
+            mc.rootfs.diff_ids.append("sha256:"+diffid)
             manifest.layers.append(cd)
         # TODO: add to Manifest.config the history/author of this project.
         mc_json = mc.model_dump_json(exclude_none=True)

diff --git a/tests/backend/test_oras_cp.py b/tests/backend/test_oras_cp.py
@@ -71,6 +71,65 @@ def test_oras_scenario(tmp_path):
                 break
         except docker.errors.NotFound:
             print("test container terminated")
+            break
+        except Exception as e:
+            print(f"Attempt to terminate {attempt + 1} failed: {e}")
+        attempt += 1
+    if attempt == max_attempts:
+        print("Was unable to terminate the test container")
+    client.images.remove("localhost:5001/nstestorg/modelcar")
+
+
+@pytest.mark.e2e_oras
+def test_oras_scenario_modelcard(tmp_path):
+    """Test oras with an end-to-end scenario with modelcard as separate layer
+    """
+    oras_pull("quay.io/mmortari/hello-world-wait:latest", tmp_path)
+    model_joblib = Path(__file__).parent / ".." / "data" / "model.joblib"
+    model_files = [
+        model_joblib,
+        Path(__file__).parent / ".." / "data" / "hello.md",
+    ]
+    modelcard = Path(__file__).parent / ".." / "data" / "README.md"
+    oci_layers_on_top(tmp_path, model_files, modelcard)
+    oras_push(tmp_path, "localhost:5001/nstestorg/modelcar:latest")
+
+    # show what has been copied in Container Registry
+    subprocess.run(["skopeo","list-tags","--tls-verify=false","docker://localhost:5001/nstestorg/modelcar"], check=True)
+
+    # copy from Container Registry to Docker daemon for local running the modelcar as-is
+    result = subprocess.run("skopeo inspect --tls-verify=false --raw docker://localhost:5001/nstestorg/modelcar | jq -r '.manifests[] | select(.platform.architecture == \"amd64\") | .digest'", shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    assert result.returncode == 0
+    digest = result.stdout.strip()
+    print(digest)
+    # use by convention the linux/amd64
+    subprocess.run(["skopeo", "copy", "--src-tls-verify=false", f"docker://localhost:5001/nstestorg/modelcar@{digest}", "docker-daemon:localhost:5001/nstestorg/modelcar:latest"], check=True)
+    client = docker.from_env()
+    container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True, remove=True)
+    print(container.logs())
+    _, stat = container.get_archive('/models/model.joblib')
+    print(str(stat["size"]))
+    # assert the model.joblib from the KServe modelcar is in expected location (above) and expected size
+    assert stat["size"] == os.stat(model_joblib).st_size
+
+    # assert the README.md modelcard is in expected location and expected size
+    _, stat = container.get_archive('/models/README.md')
+    print(str(stat["size"]))
+    assert stat["size"] == os.stat(modelcard).st_size
+
+    container.kill()
+    max_attempts = 5
+    attempt = 0
+    while attempt < max_attempts:
+        try:
+            if client.containers.get(container.id):
+                container.kill()
+                time.sleep(2**attempt)
+            else:
+                break
+        except docker.errors.NotFound:
+            print("test container terminated")
+            break
         except Exception as e:
             print(f"Attempt to terminate {attempt + 1} failed: {e}")
         attempt += 1

diff --git a/tests/backend/test_skopeo.py b/tests/backend/test_skopeo.py
@@ -71,6 +71,66 @@ def test_skopeo_scenario(tmp_path):
                 break
         except docker.errors.NotFound:
             print("test container terminated")
+            break
+        except Exception as e:
+            print(f"Attempt to terminate {attempt + 1} failed: {e}")
+        attempt += 1
+    if attempt == max_attempts:
+        print("Was unable to terminate the test container")
+    client.images.remove("localhost:5001/nstestorg/modelcar")
+
+
+@pytest.mark.e2e_skopeo
+def test_skopeo_scenario_modelcard(tmp_path):
+    """Test skopeo with an end-to-end scenario with modelcard as separate layer
+    """
+    skopeo_pull("quay.io/mmortari/hello-world-wait", tmp_path)
+    model_joblib = Path(__file__).parent / ".." / "data" / "model.joblib"
+    model_files = [
+        model_joblib,
+        Path(__file__).parent / ".." / "data" / "hello.md",
+    ]
+    modelcard = Path(__file__).parent / ".." / "data" / "README.md"
+    oci_layers_on_top(tmp_path, model_files, modelcard)
+    skopeo_push(tmp_path, "localhost:5001/nstestorg/modelcar")
+
+    # show what has been copied in Container Registry
+    subprocess.run(["skopeo","list-tags","--tls-verify=false","docker://localhost:5001/nstestorg/modelcar"], check=True)
+
+    # copy from Container Registry to Docker daemon for local running the modelcar as-is
+    result = subprocess.run("skopeo inspect --tls-verify=false --raw docker://localhost:5001/nstestorg/modelcar | jq -r '.manifests[] | select(.platform.architecture == \"amd64\") | .digest'", shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    assert result.returncode == 0
+    digest = result.stdout.strip()
+    print(digest)
+    # use by convention the linux/amd64
+    subprocess.run(["skopeo", "copy", "--src-tls-verify=false", f"docker://localhost:5001/nstestorg/modelcar@{digest}", "docker-daemon:localhost:5001/nstestorg/modelcar:latest"], check=True)
+    client = docker.from_env()
+    container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True, remove=True)
+    print(container.logs())
+
+    _, stat = container.get_archive('/models/model.joblib')
+    print(str(stat["size"]))
+    # assert the model.joblib from the KServe modelcar is in expected location (above) and expected size
+    assert stat["size"] == os.stat(model_joblib).st_size
+
+    # assert the README.md modelcard is in expected location and expected size
+    _, stat = container.get_archive('/models/README.md')
+    print(str(stat["size"]))
+    assert stat["size"] == os.stat(modelcard).st_size
+
+    container.kill()
+    max_attempts = 5
+    attempt = 0
+    while attempt < max_attempts:
+        try:
+            if client.containers.get(container.id):
+                container.kill()
+                time.sleep(2**attempt)
+            else:
+                break
+        except docker.errors.NotFound:
+            print("test container terminated")
+            break
         except Exception as e:
             print(f"Attempt to terminate {attempt + 1} failed: {e}")
         attempt += 1

diff --git a/tests/data/README.md b/tests/data/README.md
@@ -0,0 +1,199 @@
+---
+license: apache-2.0
+tags:
+- example
+- demo
+---
+
+# Model Card for Very Simple Example
+
+This is just a template model card.
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+{{ model_description | default("", true) }}
+
+- **Developed by:** {{ developers | default("[More Information Needed]", true)}}
+- **Funded by [optional]:** {{ funded_by | default("[More Information Needed]", true)}}
+- **Shared by [optional]:** {{ shared_by | default("[More Information Needed]", true)}}
+- **Model type:** {{ model_type | default("[More Information Needed]", true)}}
+- **Language(s) (NLP):** {{ language | default("[More Information Needed]", true)}}
+- **License:** {{ license | default("[More Information Needed]", true)}}
+- **Finetuned from model [optional]:** {{ base_model | default("[More Information Needed]", true)}}
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** {{ repo | default("[More Information Needed]", true)}}
+- **Paper [optional]:** {{ paper | default("[More Information Needed]", true)}}
+- **Demo [optional]:** {{ demo | default("[More Information Needed]", true)}}
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+{{ direct_use | default("[More Information Needed]", true)}}
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+{{ downstream_use | default("[More Information Needed]", true)}}
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+{{ out_of_scope_use | default("[More Information Needed]", true)}}
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+{{ bias_risks_limitations | default("[More Information Needed]", true)}}
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+{{ bias_recommendations | default("Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.", true)}}
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+{{ get_started_code | default("[More Information Needed]", true)}}
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+{{ training_data | default("[More Information Needed]", true)}}
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+{{ preprocessing | default("[More Information Needed]", true)}}
+
+
+#### Training Hyperparameters
+
+- **Training regime:** {{ training_regime | default("[More Information Needed]", true)}} <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+{{ speeds_sizes_times | default("[More Information Needed]", true)}}
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+{{ testing_data | default("[More Information Needed]", true)}}
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+{{ testing_factors | default("[More Information Needed]", true)}}
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+{{ testing_metrics | default("[More Information Needed]", true)}}
+
+### Results
+
+{{ results | default("[More Information Needed]", true)}}
+
+#### Summary
+
+{{ results_summary | default("", true) }}
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+{{ model_examination | default("[More Information Needed]", true)}}
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** {{ hardware_type | default("[More Information Needed]", true)}}
+- **Hours used:** {{ hours_used | default("[More Information Needed]", true)}}
+- **Cloud Provider:** {{ cloud_provider | default("[More Information Needed]", true)}}
+- **Compute Region:** {{ cloud_region | default("[More Information Needed]", true)}}
+- **Carbon Emitted:** {{ co2_emitted | default("[More Information Needed]", true)}}
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+{{ model_specs | default("[More Information Needed]", true)}}
+
+### Compute Infrastructure
+
+{{ compute_infrastructure | default("[More Information Needed]", true)}}
+
+#### Hardware
+
+{{ hardware_requirements | default("[More Information Needed]", true)}}
+
+#### Software
+
+{{ software | default("[More Information Needed]", true)}}
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+{{ citation_bibtex | default("[More Information Needed]", true)}}
+
+**APA:**
+
+{{ citation_apa | default("[More Information Needed]", true)}}
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+{{ glossary | default("[More Information Needed]", true)}}
+
+## More Information [optional]
+
+{{ more_information | default("[More Information Needed]", true)}}
+
+## Model Card Authors [optional]
+
+{{ model_card_authors | default("[More Information Needed]", true)}}
+
+## Model Card Contact
+
+{{ model_card_contact | default("[More Information Needed]", true)}}