refactoring of giskard deployment

Giskard-AI · Mar 28, 2023 · 4156b99 · 4156b99
1 parent 70afa92
commit 4156b99
Show file tree

Hide file tree

Showing 23 changed files with 226 additions and 16 deletions.
diff --git a/Giskard-deployment/README.md b/Giskard-deployment/README.md
diff --git a/Giskard-deployment/.gitignore → deployment/.gitignore b/Giskard-deployment/.gitignore → deployment/.gitignore
diff --git a/Giskard-deployment/Giskard_test.py → deployment/Giskard_test.py b/Giskard-deployment/Giskard_test.py → deployment/Giskard_test.py
diff --git a/deployment/README.md b/deployment/README.md
@@ -0,0 +1,16 @@
+[![Deployment workflow](https://github.com/Giskard-AI/giskard-examples/actions/workflows/deployment.yml/badge.svg?branch=deployment)](https://github.com/Giskard-AI/giskard-examples/actions/workflows/deployment.yml)
+
+# Deployment
+Example of how to deploy the [credit scoring](https://github.com/Giskard-AI/giskard-examples/blob/main/Credit%20scoring%20classification%20model.ipynb) once it passes [Giskard](https://www.giskard.ai/)'s tests.
+
+## Workflow
+The deployment happens according to three steps:
+- [Training](https://github.com/rabah-khalek/deployment/blob/0f2902bf653adb04551f7be11a700e0e0e95f327/.github/workflows/deployment.yml#L5-L48): execution of `train.py` script every time a `push` occurs.
+- [Testing](https://github.com/rabah-khalek/deployment/blob/0f2902bf653adb04551f7be11a700e0e0e95f327/.github/workflows/deployment.yml#L50-L101): execution of `Giskard_test.py` script after the training is done. During this step, the model produced by `train.py` is uploaded to the Giskard UI (where the user can define his custom test suite). Then all the tests are performed. If at least 50% of the tests pass (this can be tuned [here](https://github.com/rabah-khalek/deployment/blob/0f2902bf653adb04551f7be11a700e0e0e95f327/Giskard_test.py#L85) in `Giskard_test.py`), the model is tagged as verified.
+- [Deployment](https://github.com/rabah-khalek/deployment/blob/0f2902bf653adb04551f7be11a700e0e0e95f327/.github/workflows/deployment.yml#L103-L142): If the model is verified, it is committed to the repo under the folder `deployed_model` that contains:
+  - a subfolder named as the date of the model training which itself contains: 
+    - `logistic_regression_model.pkl`: a pickle of the verified model.
+    - `test_data.zip`: a pandas dataframe that was used to verify the model.
+    - `Giskard_tests.json`: a summary of all the tests and metrics that were run by Giskard.
+
+<img width="950" alt="Screenshot 2022-12-08 at 15 54 18" src="https://user-images.githubusercontent.com/32709181/206478480-d3831099-ca5b-4a95-9282-598f6a433639.png">
diff --git a/deployment/create_project.py b/deployment/create_project.py
@@ -0,0 +1,32 @@
+"""Main module."""
+import os
+os.environ['GSK_URL']="https://dev.giskard.ai/"
+os.environ['GSK_TOKEN']="eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJhZG1pbiIsInRva2VuX3R5cGUiOiJBUEkiLCJhdXRoIjoiUk9MRV9BRE1JTixST0xFX0FJVEVTVEVSIiwiZXhwIjoxNjgzNTQxNjI5fQ.7CXD98Yil36eaHHJuTxwjY_5vEIT8BcyWxD9Gk3U-HI"
+os.environ['GSK_PROJECT_KEY']="test"
+os.environ['GSK_PROJECT_NAME']="test"
+os.environ['GSK_PROJECT_DESCRIPTION']="test"
+import warnings
+warnings.filterwarnings("ignore")
+
+from giskard import GiskardClient
+
+if __name__ == "__main__":
+
+    url = os.environ['GSK_URL'] # read from https://github.com/Giskard-AI/giskard-examples secrets
+    token = os.environ['GSK_TOKEN'] # read from https://github.com/Giskard-AI/giskard-examples secrets
+
+    client = GiskardClient(url, token)
+
+    try:
+        credit_scoring = client.create_project(os.environ['GSK_PROJECT_KEY'], os.environ['GSK_PROJECT_NAME'], os.environ['GSK_PROJECT_DESCRIPTION'])
+        print(f"Your project \"{os.environ['GSK_PROJECT_KEY']}\" is now created on {url}")
+    except:
+        credit_scoring = client.get_project(os.environ['GSK_PROJECT_KEY'])
+        print(f"Your project \"{os.environ['GSK_PROJECT_KEY']}\" has already been created on {url}")
+
+    try:
+        test_suite_id = credit_scoring.list_test_suites()[0]['id']
+        print("There's already a test suite created")
+    except:
+        print(f"Before the release of the test-API v2.0, you would need to create a test suite yourself from the \
+              {url} in the project \"{os.environ['GSK_PROJECT_KEY']}\" in order for the deployment to work")
diff --git a/Giskard-deployment/data.csv → deployment/data.csv b/Giskard-deployment/data.csv → deployment/data.csv
diff --git a/...el/12-02-2023_17.32.51/Giskard_tests.json → ...el/12-02-2023_17.32.51/Giskard_tests.json b/...el/12-02-2023_17.32.51/Giskard_tests.json → ...el/12-02-2023_17.32.51/Giskard_tests.json
diff --git a/...23_17.32.51/logistic_regression_model.pkl → ...23_17.32.51/logistic_regression_model.pkl b/...23_17.32.51/logistic_regression_model.pkl → ...23_17.32.51/logistic_regression_model.pkl
diff --git a/...d_model/12-02-2023_17.32.51/test_data.zip → ...d_model/12-02-2023_17.32.51/test_data.zip b/...d_model/12-02-2023_17.32.51/test_data.zip → ...d_model/12-02-2023_17.32.51/test_data.zip
diff --git a/...ls/08-12-2022_16.35.01/Giskard_tests.json → ...ls/08-12-2022_16.35.01/Giskard_tests.json b/...ls/08-12-2022_16.35.01/Giskard_tests.json → ...ls/08-12-2022_16.35.01/Giskard_tests.json
diff --git a/...22_16.35.01/logistic_regression_model.pkl → ...22_16.35.01/logistic_regression_model.pkl b/...22_16.35.01/logistic_regression_model.pkl → ...22_16.35.01/logistic_regression_model.pkl
diff --git a/..._models/08-12-2022_16.35.01/test_data.zip → ..._models/08-12-2022_16.35.01/test_data.zip b/..._models/08-12-2022_16.35.01/test_data.zip → ..._models/08-12-2022_16.35.01/test_data.zip
diff --git a/...ls/31-01-2023_17.34.46/Giskard_tests.json → ...ls/31-01-2023_17.34.46/Giskard_tests.json b/...ls/31-01-2023_17.34.46/Giskard_tests.json → ...ls/31-01-2023_17.34.46/Giskard_tests.json
diff --git a/...23_17.34.46/logistic_regression_model.pkl → ...23_17.34.46/logistic_regression_model.pkl b/...23_17.34.46/logistic_regression_model.pkl → ...23_17.34.46/logistic_regression_model.pkl
diff --git a/..._models/31-01-2023_17.34.46/test_data.zip → ..._models/31-01-2023_17.34.46/test_data.zip b/..._models/31-01-2023_17.34.46/test_data.zip → ..._models/31-01-2023_17.34.46/test_data.zip
diff --git a/Giskard-deployment/requirements.txt → deployment/requirements.txt b/Giskard-deployment/requirements.txt → deployment/requirements.txt
diff --git a/Giskard-deployment/train.py → deployment/train.py b/Giskard-deployment/train.py → deployment/train.py
diff --git a/deployment/trained_model/07-02-2023_12.56.36/logistic_regression_model.pkl b/deployment/trained_model/07-02-2023_12.56.36/logistic_regression_model.pkl
diff --git a/deployment/trained_model/07-02-2023_12.56.36/test_data.zip b/deployment/trained_model/07-02-2023_12.56.36/test_data.zip
diff --git a/deployment/trained_model/26-01-2023_12.40.27/Giskard_tests.json b/deployment/trained_model/26-01-2023_12.40.27/Giskard_tests.json
@@ -0,0 +1,178 @@
+[
+    {
+        "id": 57,
+        "name": "Custom Test",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:32.002Z",
+        "message": null,
+        "metric": 1.0
+    },
+    {
+        "id": 67,
+        "name": "Recall difference",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:32Z",
+        "message": null,
+        "metric": 0.011697275
+    },
+    {
+        "id": 52,
+        "name": "Precision difference",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:32Z",
+        "message": null,
+        "metric": 0.030879991
+    },
+    {
+        "id": 62,
+        "name": "Increasing (t-test)",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:31.998Z",
+        "message": null,
+        "metric": 7e-45
+    },
+    {
+        "id": 72,
+        "name": "Increasing (Wilcoxon)",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:32.002Z",
+        "message": null,
+        "metric": 7.180732e-35
+    },
+    {
+        "id": 63,
+        "name": "Invariance (proportion)",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:33.011Z",
+        "message": null,
+        "metric": 0.85
+    },
+    {
+        "id": 53,
+        "name": "Precision",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:32.926Z",
+        "message": null,
+        "metric": 0.82191783
+    },
+    {
+        "id": 73,
+        "name": "Decreasing (proportion)",
+        "status": "FAILED",
+        "executionDate": "2023-01-30T12:40:33.050Z",
+        "message": null,
+        "metric": 0.0
+    },
+    {
+        "id": 58,
+        "name": "Decreasing (t-test)",
+        "status": "FAILED",
+        "executionDate": "2023-01-30T12:40:32.815Z",
+        "message": null,
+        "metric": 1.0
+    },
+    {
+        "id": 68,
+        "name": "Decreasing (Wilcoxon)",
+        "status": "FAILED",
+        "executionDate": "2023-01-30T12:40:32.900Z",
+        "message": null,
+        "metric": 1.0
+    },
+    {
+        "id": 64,
+        "name": "Classification Probability in Range",
+        "status": "FAILED",
+        "executionDate": "2023-01-30T12:40:33.270Z",
+        "message": null,
+        "metric": 0.23
+    },
+    {
+        "id": 59,
+        "name": "Increasing (proportion)",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:33.643Z",
+        "message": null,
+        "metric": 1.0
+    },
+    {
+        "id": 54,
+        "name": "Accuracy",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:33.328Z",
+        "message": null,
+        "metric": 0.76
+    },
+    {
+        "id": 65,
+        "name": "Invariance (Wilcoxon)",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:33.799Z",
+        "message": null,
+        "metric": 2.9326454e-18
+    },
+    {
+        "id": 69,
+        "name": "AUC",
+        "status": "FAILED",
+        "executionDate": "2023-01-30T12:40:33.742Z",
+        "message": null,
+        "metric": 0.6983973
+    },
+    {
+        "id": 60,
+        "name": "Recall",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:33.970Z",
+        "message": null,
+        "metric": 0.8450704
+    },
+    {
+        "id": 55,
+        "name": "F1",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:34.038Z",
+        "message": null,
+        "metric": 0.8333333
+    },
+    {
+        "id": 66,
+        "name": "F1 difference",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:34.202Z",
+        "message": null,
+        "metric": 0.009887002
+    },
+    {
+        "id": 61,
+        "name": "Right Label",
+        "status": "FAILED",
+        "executionDate": "2023-01-30T12:40:34.293Z",
+        "message": null,
+        "metric": 0.27
+    },
+    {
+        "id": 70,
+        "name": "Accuracy difference",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:34.236Z",
+        "message": null,
+        "metric": 0.013333321
+    },
+    {
+        "id": 56,
+        "name": "Invariance (t-test)",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:34.365Z",
+        "message": null,
+        "metric": 9.099929e-22
+    },
+    {
+        "id": 71,
+        "name": "Disparate Impact",
+        "status": "PASSED",
+        "executionDate": "2023-01-30T12:40:34.738Z",
+        "message": null,
+        "metric": 1.0384616
+    }
+]
diff --git a/deployment/trained_model/26-01-2023_12.40.27/logistic_regression_model.pkl b/deployment/trained_model/26-01-2023_12.40.27/logistic_regression_model.pkl
diff --git a/deployment/trained_model/26-01-2023_12.40.27/test_data.zip b/deployment/trained_model/26-01-2023_12.40.27/test_data.zip
diff --git a/Giskard-deployment/verified_model.tar.gz → deployment/verified_model.tar.gz b/Giskard-deployment/verified_model.tar.gz → deployment/verified_model.tar.gz