added local catch for EFS OS/permissions issues

jose-guevarra · Dec 7, 2023 · 11ab35b · 11ab35b
1 parent 5407543
commit 11ab35b
Show file tree

Hide file tree

Showing 7 changed files with 446 additions and 393 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@ stores/
 mlflow/
 results/
 workspaces/
+efs/
 
 # VSCode
 .vscode/

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -2,7 +2,7 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.5.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer

diff --git a/Makefile b/Makefile
@@ -12,6 +12,7 @@ style:
 # Cleaning
 .PHONY: clean
 clean: style
+	python notebooks/clear_cell_nums.py
 	find . -type f -name "*.DS_Store" -ls -delete
 	find . | grep -E "(__pycache__|\.pyc|\.pyo)" | xargs rm -rf
 	find . | grep -E ".pytest_cache" | xargs rm -rf

diff --git a/madewithml/config.py b/madewithml/config.py
@@ -11,6 +11,11 @@
 LOGS_DIR = Path(ROOT_DIR, "logs")
 LOGS_DIR.mkdir(parents=True, exist_ok=True)
 EFS_DIR = Path(f"/efs/shared_storage/madewithml/{os.environ.get('GITHUB_USERNAME', '')}")
+try:
+    Path(EFS_DIR).mkdir(parents=True, exist_ok=True)
+except OSError:
+    EFS_DIR = Path(ROOT_DIR, "efs")
+    Path(EFS_DIR).mkdir(parents=True, exist_ok=True)
 
 # Config MLflow
 MODEL_REGISTRY = Path(f"{EFS_DIR}/mlflow")

diff --git a/notebooks/benchmarks.ipynb b/notebooks/benchmarks.ipynb
@@ -58,7 +58,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "e2c96931-d511-4c6e-b582-87d24455a11e",
    "metadata": {
     "tags": []
@@ -79,7 +79,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "id": "953a577e-3cd0-4c6b-81f9-8bc32850214d",
    "metadata": {
     "tags": []
@@ -101,7 +101,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "id": "1790e2f5-6b8b-425c-8842-a2b0ea8f3f07",
    "metadata": {
     "tags": []
@@ -113,7 +113,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": null,
    "id": "6b9bfadb-ba49-4f5a-b216-4db14c8888ab",
    "metadata": {
     "tags": []
@@ -208,7 +208,7 @@
        "4  A PyTorch Implementation of \"Watch Your Step: ...            other  "
       ]
      },
-     "execution_count": 4,
+     "execution_count": null,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -222,7 +222,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "aa5b95d5-d61e-48e4-9100-d9d2fc0d53fa",
    "metadata": {
     "tags": []
@@ -234,7 +234,7 @@
        "['computer-vision', 'other', 'natural-language-processing', 'mlops']"
       ]
      },
-     "execution_count": 5,
+     "execution_count": null,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -247,7 +247,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "3c828129-8248-4e38-93a4-cabb097e7ba5",
    "metadata": {
     "tags": []
@@ -279,7 +279,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "8e3c3f44-2c19-4c32-9bc5-e9a7a917d19d",
    "metadata": {},
    "outputs": [],
@@ -295,7 +295,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "4950bdb4",
    "metadata": {},
    "outputs": [
@@ -337,7 +337,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "id": "b2aae14c-9870-4a27-b5ad-90f339686620",
    "metadata": {
     "tags": []
@@ -364,7 +364,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": null,
    "id": "03ee23e5",
    "metadata": {},
    "outputs": [
@@ -401,7 +401,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "id": "71c43e8c",
    "metadata": {},
    "outputs": [
@@ -416,7 +416,7 @@
        "  'description': 'A PyTorch implementation of \"Capsule Graph Neural Network\" (ICLR 2019).'}]"
       ]
      },
-     "execution_count": 11,
+     "execution_count": null,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -429,7 +429,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": null,
    "id": "c9359a91-ac19-48a4-babb-e65d53f39b42",
    "metadata": {
     "tags": []
@@ -462,7 +462,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": null,
    "id": "5fac795e",
    "metadata": {},
    "outputs": [
@@ -486,7 +486,7 @@
        "['other', 'computer-vision', 'computer-vision']"
       ]
      },
-     "execution_count": 13,
+     "execution_count": null,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -507,7 +507,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": null,
    "id": "e4cb38a8-44cb-4cea-828c-590f223d4063",
    "metadata": {
     "tags": []
@@ -543,7 +543,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": null,
    "id": "de2d0416",
    "metadata": {},
    "outputs": [],
@@ -576,7 +576,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": null,
    "id": "ff3c37fb",
    "metadata": {},
    "outputs": [],
@@ -618,7 +618,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": null,
    "id": "972fee2f-86e2-445e-92d0-923f5690132a",
    "metadata": {},
    "outputs": [],
@@ -647,7 +647,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": null,
    "id": "9ee4e745-ef56-4b76-8230-fcbe56ac46aa",
    "metadata": {
     "tags": []
@@ -663,7 +663,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": null,
    "id": "73780054-afeb-4ce6-8255-51bf91f9f820",
    "metadata": {
     "tags": []
@@ -709,7 +709,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": null,
    "id": "24af6d04-d29e-4adb-a289-4c34c2cc7ec8",
    "metadata": {
     "tags": []
@@ -780,7 +780,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": null,
    "id": "e22ed1e1-b34d-43d1-ae8b-32b1fd5be53d",
    "metadata": {
     "tags": []
@@ -815,7 +815,7 @@
        "  'tag': 'mlops'}]"
       ]
      },
-     "execution_count": 22,
+     "execution_count": null,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -833,7 +833,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": null,
    "id": "294548a5-9edf-4dea-ab8d-dc7464246810",
    "metadata": {
     "tags": []
@@ -864,7 +864,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": null,
    "id": "29bca273-3ea8-4ce0-9fa9-fe19062b7c5b",
    "metadata": {
     "tags": []
@@ -917,7 +917,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": null,
    "id": "3e59a3b9-69d9-4bb5-8b88-0569fcc72f0c",
    "metadata": {
     "tags": []
@@ -1001,7 +1001,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": null,
    "id": "15ea136e",
    "metadata": {},
    "outputs": [],
@@ -1020,7 +1020,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": null,
    "id": "ec0b498a-97c1-488c-a6b9-dc63a8a9df4d",
    "metadata": {
     "tags": []
@@ -1065,7 +1065,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": null,
    "id": "4cc80311",
    "metadata": {},
    "outputs": [],
@@ -1080,7 +1080,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 30,
+   "execution_count": null,
    "id": "6771b1d2",
    "metadata": {},
    "outputs": [

diff --git a/notebooks/clear_cell_nums.py b/notebooks/clear_cell_nums.py
@@ -0,0 +1,23 @@
+from pathlib import Path
+
+import nbformat
+
+
+def clear_execution_numbers(nb_path):
+    with open(nb_path, "r", encoding="utf-8") as f:
+        nb = nbformat.read(f, as_version=4)
+    for cell in nb["cells"]:
+        if cell["cell_type"] == "code":
+            cell["execution_count"] = None
+            for output in cell["outputs"]:
+                if "execution_count" in output:
+                    output["execution_count"] = None
+    with open(nb_path, "w", encoding="utf-8") as f:
+        nbformat.write(nb, f)
+
+
+if __name__ == "__main__":
+    NOTEBOOK_DIR = Path(__file__).parent
+    notebook_fps = list(NOTEBOOK_DIR.glob("**/*.ipynb"))
+    for fp in notebook_fps:
+        clear_execution_numbers(fp)