diff --git a/.gitignore b/.gitignore index cb0a5b4..388482b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,11 @@ test*.py *checkpoint* *.xml -Fracture_* + *py_cache* .* !.gitignore !.gitkeep -!*best_experiment* -*.csv \ No newline at end of file +*.csv +logs.log +experiments diff --git a/README.md b/README.md index 799a9cb..8fcc6e7 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,39 @@ It uses the [HAIM multimodal dataset](https://physionet.org/content/haim-multimo (tabular, time-series, text and images) and 11 unique sources to perform 12 predictive tasks (10 chest pathologies, length-of-stay and 48 h mortality predictions). -This package is our own adaptation of the [HAIM GitHub package](https://github.com/lrsoenksen/HAIM.git). +This [HAIM GitHub package](https://github.com/MEDomics-UdeS/HAIM) is MEDomicsLab’s own adaptation of the [HAIM GitHub package](https://github.com/lrsoenksen/HAIM.git). +This version has the same purpose with differents tools, as we are incorporating our custom PyCaretEvaluator class for the training part. + +The PyCaretEvaluator class is designed to streamline and enhance the model evaluation process by integrating the PyCaret library with Ray for parallel execution, allowing for efficient memory management and performance optimization. PyCaret is an open-source, low-code machine learning library in Python that simplifies the process of building, training, and deploying machine learning models. This class is particularly useful for cases involving extensive model evaluations or hyperparameter tuning across multiple cross-validation folds. By leveraging Ray, the class executes each fold in parallel, reducing computation time and improving scalability on larger datasets. + +## 2. How to use the package? +The dataset used to replicate this study is publicly available on [physionet](https://physionet.org/content/haim-multimodal/1.0.1/). +To run this package on **Python 3.11**, you need to set up a Conda environment to manage dependencies. + +### 2.1 Creating and Activating a Conda Environment + +1. **Create the environment** with the required Python version: + + - + ```bash + $ conda create --name haim_env python=3.11 + ``` + +2. **Activate the Conda environment**: + + ```bash + $ conda activate haim_env + + ``` + +### 2.2 Installing the requirements + + ```bash + $ pip install -r requirements.txt + + ``` + -## 2. How to use the package ? -The dataset used to replicate this study is publicly available in [physionet](https://physionet.org/content/haim-multimodal/1.0.1/). To run this package: -- Download the dataset and move the file ``cxr_ic_fusion_1103.csv`` to [csvs](csvs). -- Install the requirements under **Python 3.9.13** as following: -``` -$ pip install requirements.txt -``` The package can be used with different sources combinations to predict one of the 12 predictive tasks defined above. Here is a code snippet which uses one combination of sources to predict patient's length-of-stay: ```python @@ -66,13 +90,6 @@ run the following command: ``` $ python run_experiments.py ``` - -> **Warning** -> -> The HAIM experiment performs 14324 evaluations (1023 evaluations for each of the chest pathologies prediction tasks and 2047 for the length-of-stay and 48h mortality). We didn't run the experiment but we approximate the execution time to 200 days run with the current implementation using only 10 CPUs. - -The experiments results (metrics values and figures) will be stored in the [``experiments``](experiments) directory where the name of each folder is structured as ``TaskName_NumberOfTheExperiment`` -(ex. Fracture_25). For each prediction task, the sources combination with the best AUC will be stored in the directory ``TaskName_best_experiment``. To reproduce the HAIM exepriment on one single predictive task, run the following command: ``` @@ -106,22 +123,23 @@ Below are the ``AUC`` values reported from our experiments compared to those rep -Task | AUC from our experiment | AUC from the paper | ----------| -----------| ----------- | -Fracture | 0.828 +- 0.110 | 0.838 | -Pneumothorax| 0.811 +- 0.021 | 0.836 | -Pneumonia | 0.871 +- 0.013 | 0.883 | -Lung opacity | 0.797 +- 0.015 | 0.816 | -Lung lesion | 0.829 +- 0.053 | 0.844 | -Enlarged Cardiomediastinum | 0.877 +- 0.035 | 0.876 | -Edema | 0.915 +- 0.007 |0.917 | -Consolidation | 0.918 +- 0.018 | 0.929 | -Cardiomegaly | 0.908 +- 0.004 | 0.914 | -Atelectasis | 0.765 +- 0.013 | 0.779 | -Length of stay | 0.932 +- 0.012 | 0.939| -48 hours mortality | 0.907 +- 0.007 | 0.912 | +| Task | AUC from our 2nd experiment | AUC from our 1st experiment | AUC from the paper | +|-------------------------|-----------------------------|-----------------------------|--------------------| +| Fracture | 0.731 +- 0.134 | 0.828 +- 0.110 | 0.838 | +| Pneumothorax | 0.898 +- 0.012 | 0.811 +- 0.021 | 0.836 | +| Pneumonia | 0.877 +- 0.012 | 0.871 +- 0.013 | 0.883 | +| Lung opacity | 0.809 +- 0.012 | 0.797 +- 0.015 | 0.816 | +| Lung lesion | 0.888 +- 0.069 | 0.829 +- 0.053 | 0.844 | +| Enlarged Cardiomediastinum | 0.888 +- 0.019 | 0.877 +- 0.035 | 0.876 | +| Edema | 0.915 +- 0.005 | 0.915 +- 0.007 | 0.917 | +| Consolidation | 0.912 +- 0.015 | 0.918 +- 0.018 | 0.929 | +| Cardiomegaly | 0.922 +- 0.005 | 0.908 +- 0.004 | 0.914 | +| Atelectasis | 0.796 +- 0.022 | 0.765 +- 0.013 | 0.779 | +| Length of stay | 0.959 +- 0.003 | 0.932 +- 0.012 | 0.939 | +| 48 hours mortality | 0.960 +- 0.004 | 0.907 +- 0.007 | 0.912 | -More statistics and metrics are reported from each of the 12 experiments above and can be found in the ``experiments`` directory. Each experiment directory is named after the task on which the prediction model was evaluated. + +More statistics and metrics are reported from each of the 12 experiments above and can be found in the ``results`` directory. Each experiment directory is named after the task on which the prediction model was evaluated. > **Note** > @@ -132,19 +150,52 @@ More statistics and metrics are reported from each of the 12 experiments above a We tried to reproduce the HAIM experiment and used all the 1023 possible sources combinations to predict the presence or absence of a fracture in a patient and select the one resulting in the best ``AUC``. Below the ``AUC`` value reported from our experiments compared to the one reported in the HAIM paper. - AUC from our experiment | AUC from the paper | - -----------| ----------- | -0.862 +- 0.112 | 0.838 | + | AUC from our experiment with PyCaret | AUC from our experiment | AUC from the paper | +|--------------------------------------|-------------------------|---------------------| +| 0.731 ± 0.134 | 0.862 ± 0.112 | 0.838 | + The above experiment can be performed using the following command ``` $ python run_experiments.py -t "Fracture" ``` -A recap of the experiment named [``Fracture_best_experiment``](experiments/Fracture_best_experiment) is generated at the end of the experiment containing more statistics and metrics values. +A recap of the experiment (results/fracture) is generated at the end of the experiment containing : + +#### 1. ```{task}_results.json``` : + +This JSON file stores detailed results for each fold in the cross-validation process. +For each fold, it includes: + - Train Results: Performance metrics obtained on the training data, captured after training and tuning the model. + - Test Predictions: Predictions generated by the model on the test data for this fold. + - Best Hyperparameters: The best hyperparameters found during tuning for this specific fold (if tuning was performed). +This file serves as a comprehensive record of all results and configurations for each fold. + +#### 2. ```CP_{task}_final_metrics.csv``` : + +This CSV file consolidates the mean and standard deviation of key performance metrics calculated across all folds. +Included columns: + - Metric: The name of each evaluation metric (e.g., AUC, F1 Score, Precision, Recall, MCC and Kappa). + - Mean: The average value of each metric across all folds. + - Std Dev: The standard deviation of each metric. +This file gives an overall view of the model’s performance and consistency across folds. + +> **Note** +> +> The Matthews Correlation Coefficient (MCC) is a metric that evaluates how well a model’s predictions match actual outcomes, balancing correct and incorrect predictions even when classes are imbalanced. + Cohen’s Kappa measures the level of agreement between two raters or classifiers, showing how often they agree beyond what would be expected by chance. + + +#### 3. ```best_model_fold_X.pkl``` (where X is the fold number): + +For each fold, the best-performing model is saved as a .pkl file. +Each model file can be reloaded independently if further analysis or testing is needed. +Saving the best models for each fold allows you to compare models or even ensemble them if desired. + +All the results can be accessed via this link: https://usherbrooke-my.sharepoint.com/:f:/g/personal/kalm7073_usherbrooke_ca/EtKnOhTN1kdJmRbkziHYX9EBkgMSWzhfvXMV4lb_fZw3uQ?e=Olejde. ## 5. Future work -The next step of our package is to regenerate the embeddings for each source type. For each modality (tabular, time-series, image, text), we will also explore new embeddings generators. +This adaptation aims to evaluate HAIM with PyCaret to better understand its effectiveness within the MEDomicsLab platform. Moving forward, we plan to test additional task variations to further optimize performance. ## Project Tree ``` @@ -155,10 +206,8 @@ The next step of our package is to regenerate the embeddings for each source typ │   ├── data │   │   ├── constants.py <- Constants related to the HAIM study │   │   ├── datasets.py <- Custom dataset implementation for the HAIM study -│   │   └── sampling.py <- Samples the dataset to test, train and validation │   ├── evaluation -│   │   ├── tuning.py <- Hyper-parameters optimizations using different optimizers -│   │   └── evaluating.py <- Skeleton of each experiment process +│   │   └── pycaret_evaluator.py <- Skeleton of each experiment process │   └── utils │   │   └── metric_scores.py <- Custom metrics implementations and wrappers ├── requirements.txt <- All the requirements to install to run the project diff --git a/notebooks/48h-mortality.ipynb b/notebooks/48h-mortality.ipynb index d92896b..60b81c9 100644 --- a/notebooks/48h-mortality.ipynb +++ b/notebooks/48h-mortality.ipynb @@ -49,14 +49,11 @@ "import os\n", "os.chdir('../')\n", "\n", - "from xgboost import XGBClassifier\n", "from pandas import read_csv\n", "\n", "from src.data import constants\n", "from src.data.dataset import HAIMDataset\n", - "from src.data.sampling import Sampler\n", - "from src.evaluation.evaluating import Evaluator\n", - "from src.evaluation.tuning import SklearnTuner\n", + "from src.evaluation.pycaret_evaluator import PyCaretEvaluator\n", "from src.utils.metric_scores import *" ] }, @@ -71,7 +68,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 2, "id": "a948c18c", "metadata": {}, "outputs": [], @@ -97,7 +94,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 3, "id": "239db8a3", "metadata": {}, "outputs": [], @@ -110,86 +107,17 @@ " constants.GLOBAL_ID)" ] }, - { - "cell_type": "markdown", - "id": "0d9f8354", - "metadata": {}, - "source": [ - "#### Create the sampler\n" - ] - }, - { - "cell_type": "markdown", - "id": "2fe30725", - "metadata": {}, - "source": [ - "Sample the data using a 5 folds cross-validation method based on unique ``haim_id`` " - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "d8cc844e", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "100%|███████████████████████████████████████████| 5/5 [00:00<00:00, 2788.77it/s]\n" - ] - } - ], - "source": [ - "sampler = Sampler(dataset, constants.GLOBAL_ID, 5)\n", - "_, masks = sampler()" - ] - }, - { - "cell_type": "markdown", - "id": "6abdd3c8", - "metadata": {}, - "source": [ - "#### Select the evaluation metrics" - ] - }, - { - "cell_type": "markdown", - "id": "f935bbb8", - "metadata": {}, - "source": [ - "Initilialize a list containing the evaluation metrics to report" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "3ceec6b8", - "metadata": {}, - "outputs": [], - "source": [ - "# Initialization of the list containing the evaluation metrics\n", - "evaluation_metrics = [BinaryAccuracy(), \n", - " BinaryBalancedAccuracy(),\n", - " BinaryBalancedAccuracy(Reduction.GEO_MEAN),\n", - " Sensitivity(), \n", - " Specificity(), \n", - " AUC(), \n", - " BrierScore(),\n", - " BinaryCrossEntropy()]" - ] - }, { "cell_type": "markdown", "id": "114f173f", "metadata": {}, "source": [ - "#### Set hyper-parameters and fixed parameters" + "#### Set hyper-parameters" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 4, "id": "8d6cb174", "metadata": {}, "outputs": [], @@ -198,13 +126,7 @@ "grid_hps = {'max_depth': [5, 6, 7, 8],\n", " 'n_estimators': [200, 300],\n", " 'learning_rate': [0.3, 0.1, 0.05],\n", - " }\n", - "\n", - "# Save the fixed parameters of the model\n", - "fixed_params = {'seed': 42,\n", - " 'eval_metric': 'logloss',\n", - " 'verbosity': 0\n", - " }" + " }" ] }, { @@ -244,169 +166,301 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "id": "726c2332", "metadata": {}, - "outputs": [], - "source": [ - "evaluation = Evaluator(dataset=dataset,\n", - " masks=masks,\n", - " metrics=evaluation_metrics,\n", - " model=XGBClassifier,\n", - " tuner=SklearnTuner,\n", - " tuning_metric=AUC(),\n", - " hps=grid_hps,\n", - " n_tuning_splits=5,\n", - " fixed_params=fixed_params,\n", - " filepath=constants.EXPERIMENT_PATH,\n", - " weight='scale_pos_weight',\n", - " evaluation_name=''\n", - " evaluation_name='48h mortality'\n", - " )\n", - "evaluation.evaluate()\n" - ] - }, - { - "cell_type": "markdown", - "id": "6787a8cc", - "metadata": {}, - "source": [ - "#### Comparison with the paper results:\n", - "\n", - "\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "b517ec5e", - "metadata": {}, "outputs": [ { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0 +- 0.00.0005 +- 0.0004
test_metrics0.9737 +- 0.0020.5182 +- 0.00370.1896 +- 0.01960.0363 +- 0.00741.0 +- 0.00.9066 +- 0.00720.0195 +- 0.00110.1098 +- 0.0052
HAIM----------0.912----
NON_HAIM----------0.889----
\n", - "
" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9737 +- 0.002 0.5182 +- 0.0037 0.1896 +- 0.0196 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.0363 +- 0.0074 1.0 +- 0.0 0.9066 +- 0.0072 \n", - "HAIM -- -- 0.912 \n", - "NON_HAIM -- -- 0.889 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0 +- 0.0 0.0005 +- 0.0004 \n", - "test_metrics 0.0195 +- 0.0011 0.1098 +- 0.0052 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-04 13:54:33,961\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=631326)\u001b[0m Outer fold 1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(raylet)\u001b[0m Spilled 3869 MiB, 2 objects, write throughput 589 MiB/s. Set RAY_verbose_spill_logs=0 to disable this message.\n", + "\u001b[36m(raylet)\u001b[0m Spilled 7738 MiB, 4 objects, write throughput 583 MiB/s.\n", + "\u001b[36m(raylet)\u001b[0m Spilled 11607 MiB, 5 objects, write throughput 579 MiB/s.\n", + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0005 +- 0.00020.0097 +- 0.0032
test_metrics0.9797 +- 0.00440.6456 +- 0.01620.5407 +- 0.02930.9973 +- 0.00130.294 +- 0.03220.7654 +- 0.01320.0186 +- 0.00460.0974 +- 0.0243
HAIM----------0.779----
NON_HAIM----------0.767----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9797 +- 0.0044 0.6456 +- 0.0162 0.5407 +- 0.0293 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9973 +- 0.0013 0.294 +- 0.0322 0.7654 +- 0.0132 \n", - "HAIM -- -- 0.779 \n", - "NON_HAIM -- -- 0.767 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0005 +- 0.0002 0.0097 +- 0.0032 \n", - "test_metrics 0.0186 +- 0.0046 0.0974 +- 0.0243 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-23 10:10:30,609\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=19477)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=19477)\u001b[0m Train indices: [ 1 2 3 ... 15209 15210 15212]\n", + "\u001b[36m(run_fold pid=19477)\u001b[0m Test indices: [ 0 4 7 ... 15194 15203 15211]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0003 +- 0.00.0089 +- 0.001
test_metrics0.8826 +- 0.00710.814 +- 0.00860.8079 +- 0.00980.9125 +- 0.00910.7155 +- 0.01970.908 +- 0.00380.0736 +- 0.00420.2708 +- 0.0133
HAIM----------0.914----
NON_HAIM----------0.912----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.8826 +- 0.0071 0.814 +- 0.0086 0.8079 +- 0.0098 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9125 +- 0.0091 0.7155 +- 0.0197 0.908 +- 0.0038 \n", - "HAIM -- -- 0.914 \n", - "NON_HAIM -- -- 0.912 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0003 +- 0.0 0.0089 +- 0.001 \n", - "test_metrics 0.0736 +- 0.0042 0.2708 +- 0.0133 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-24 08:49:09,601\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=124826)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=124826)\u001b[0m Train indices: [ 0 3 4 ... 18565 18568 18570]\n", + "\u001b[36m(run_fold pid=124826)\u001b[0m Test indices: [ 1 2 9 ... 18566 18567 18569]\n", + "\u001b[36m(run_fold pid=124826)\u001b[0m Configuring PyCaret for outer fold 1\n", + "\u001b[36m(run_fold pid=124833)\u001b[0m Outer fold 3\u001b[32m [repeated 2x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n", + "\u001b[36m(run_fold pid=124833)\u001b[0m Train indices: [ 0 1 2 ... 18567 18568 18569]\u001b[32m [repeated 2x across cluster]\u001b[0m\n", + "\u001b[36m(run_fold pid=124833)\u001b[0m Test indices: [ 6 18 19 ... 18542 18549 18570]\u001b[32m [repeated 2x across cluster]\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0009 +- 0.00130.0151 +- 0.0135
test_metrics0.8712 +- 0.02330.8451 +- 0.01930.8437 +- 0.01970.8856 +- 0.02980.8046 +- 0.03830.9181 +- 0.01830.0794 +- 0.01560.2927 +- 0.0787
HAIM----------0.929----
NON_HAIM----------0.92----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.8712 +- 0.0233 0.8451 +- 0.0193 0.8437 +- 0.0197 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.8856 +- 0.0298 0.8046 +- 0.0383 0.9181 +- 0.0183 \n", - "HAIM -- -- 0.929 \n", - "NON_HAIM -- -- 0.92 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0009 +- 0.0013 0.0151 +- 0.0135 \n", - "test_metrics 0.0794 +- 0.0156 0.2927 +- 0.0787 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-09-26 15:25:14,862\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8266 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=469104)\u001b[0m Outer fold 2\n", + "\u001b[36m(run_fold pid=469104)\u001b[0m Configuring PyCaret for outer fold 2\n", + "\u001b[36m(run_fold pid=469091)\u001b[0m Outer fold 1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0011 +- 0.00120.0222 +- 0.0129
test_metrics0.8084 +- 0.01480.8244 +- 0.00530.8208 +- 0.0070.7594 +- 0.04590.8895 +- 0.0380.9147 +- 0.00720.1151 +- 0.00590.383 +- 0.0182
HAIM----------0.917----
NON_HAIM----------0.912----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.8084 +- 0.0148 0.8244 +- 0.0053 0.8208 +- 0.007 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.7594 +- 0.0459 0.8895 +- 0.038 0.9147 +- 0.0072 \n", - "HAIM -- -- 0.917 \n", - "NON_HAIM -- -- 0.912 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0011 +- 0.0012 0.0222 +- 0.0129 \n", - "test_metrics 0.1151 +- 0.0059 0.383 +- 0.0182 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-09-24 16:07:16,490\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=121391)\u001b[0m Outer fold 1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(raylet)\u001b[0m Spilled 2750 MiB, 3 objects, write throughput 586 MiB/s. Set RAY_verbose_spill_logs=0 to disable this message.\n", + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0005 +- 0.00050.0145 +- 0.0081
test_metrics0.788 +- 0.03060.799 +- 0.03140.7981 +- 0.03160.7794 +- 0.03660.8186 +- 0.05430.8768 +- 0.0350.1208 +- 0.02250.4129 +- 0.0921
HAIM----------0.876----
NON_HAIM----------0.868----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.788 +- 0.0306 0.799 +- 0.0314 0.7981 +- 0.0316 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.7794 +- 0.0366 0.8186 +- 0.0543 0.8768 +- 0.035 \n", - "HAIM -- -- 0.876 \n", - "NON_HAIM -- -- 0.868 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0005 +- 0.0005 0.0145 +- 0.0081 \n", - "test_metrics 0.1208 +- 0.0225 0.4129 +- 0.0921 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-09-26 11:11:21,998\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=409777)\u001b[0m Outer fold 2\n", + "\u001b[36m(run_fold pid=409777)\u001b[0m Configuring PyCaret for outer fold 2\n", + "\u001b[36m(run_fold pid=409766)\u001b[0m Outer fold 4\u001b[32m [repeated 3x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0029 +- 0.00060.039 +- 0.0037
test_metrics0.9052 +- 0.04340.6851 +- 0.16050.5897 +- 0.23930.9363 +- 0.03430.4339 +- 0.32280.828 +- 0.11030.0516 +- 0.02140.1901 +- 0.067
HAIM----------0.838----
NON_HAIM----------0.787----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9052 +- 0.0434 0.6851 +- 0.1605 0.5897 +- 0.2393 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9363 +- 0.0343 0.4339 +- 0.3228 0.828 +- 0.1103 \n", - "HAIM -- -- 0.838 \n", - "NON_HAIM -- -- 0.787 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0029 +- 0.0006 0.039 +- 0.0037 \n", - "test_metrics 0.0516 +- 0.0214 0.1901 +- 0.067 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-21 11:46:41,343\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8267 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=329239)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m Train indices: [ 0 3 5 6 7 8 9 10 11 12 13 15 16 18 19 20 21 22\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 24 25 26 27 30 31 32 33 34 35 37 39 40 41 42 44 46 47\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 48 49 51 52 53 54 56 57 58 59 63 64 65 66 68 71 77 79\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 80 81 83 84 86 87 88 89 91 93 94 95 96 97 98 99 100 101\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 102 103 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 121 122 123 124 126 128 130 131 132 133 134 137 139 140 141 142 144 146\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 147 148 149 150 151 152 154 155 156 157 159 162 164 165 166 168 169 171\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 172 173 175 177 179 180 182 183 185 186 187 189 190 191 192 195 196 197\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 198 199 200 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 217 218 219 220 222 223 224 225 226 227 228 230 231 232 234 235 236 237\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 240 241 242 243 245 246 247 248 250 251 252 253 254 255 256 257 258 259\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 260 261 263 264 265 266 268 270 271 272 273 275 276 277 278 279 280 281\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 282 283 284 285 287 288 289 290 291 293 294 295 296 297 298 300 301 302\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 358 359 360 362 364 366 367 368 369 373 374 375 376 377 380 381 382 383\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 384 386 387 388 389 390 391 392 394 395 397 398 399 400 401 402 404 405\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 409 410 411 413 415 416 417 418 419 420 421 422 423 425 426 427 429 430\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 432 434 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 452 453 454 455 456 457 458 460 461 462 463 464 465 467 469 470 472 474\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 476 477 478 479 480 482 483 485 486 487 489 491 492 493 494 495 496 497\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 498 499 500 501 503 505 506 507 508 509 510 511 512 513 514 515 516 518\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 520 521 522 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 539 540 541 542 544 546 547 548 549 550 553 554 555]\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m Test indices: [ 1 2 4 14 17 23 28 29 36 38 43 45 50 55 60 61 62 67\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 69 70 72 73 74 75 76 78 82 85 90 92 104 125 127 129 135 136\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 138 143 145 153 158 160 161 163 167 170 174 176 178 181 184 188 193 194\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 201 221 229 233 238 239 244 249 262 267 269 274 286 292 299 303 361 363\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 365 370 371 372 378 379 385 393 396 403 406 407 408 412 414 424 428 431\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 433 435 459 466 468 471 473 475 481 484 488 490 502 504 517 519 523 543\n", + "\u001b[36m(run_fold pid=329239)\u001b[0m 545 551 552 556]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0 +- 0.00.0009 +- 0.0004
test_metrics0.924 +- 0.00380.5532 +- 0.0060.3266 +- 0.01820.1071 +- 0.01180.9993 +- 0.00030.9323 +- 0.01150.0461 +- 0.00330.2077 +- 0.0293
HAIM----------0.939----
NON_HAIM----------0.919----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.924 +- 0.0038 0.5532 +- 0.006 0.3266 +- 0.0182 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.1071 +- 0.0118 0.9993 +- 0.0003 0.9323 +- 0.0115 \n", - "HAIM -- -- 0.939 \n", - "NON_HAIM -- -- 0.919 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0 +- 0.0 0.0009 +- 0.0004 \n", - "test_metrics 0.0461 +- 0.0033 0.2077 +- 0.0293 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-25 22:05:42,189\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=249894)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=249894)\u001b[0m Train indices: [ 0 1 3 ... 45046 45047 45049]\n", + "\u001b[36m(run_fold pid=249894)\u001b[0m Test indices: [ 2 13 21 ... 45039 45045 45048]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(raylet)\u001b[0m Spilled 3869 MiB, 1 objects, write throughput 563 MiB/s. Set RAY_verbose_spill_logs=0 to disable this message.\n", + "\u001b[36m(raylet)\u001b[0m Spilled 7738 MiB, 3 objects, write throughput 564 MiB/s.\n", + "\u001b[36m(raylet)\u001b[0m Spilled 11607 MiB, 5 objects, write throughput 555 MiB/s.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=249894)\u001b[0m Configuring PyCaret for outer fold 1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0035 +- 0.00080.0353 +- 0.0042
test_metrics0.9601 +- 0.01130.6165 +- 0.09910.3786 +- 0.30940.9922 +- 0.00240.2407 +- 0.19730.8286 +- 0.05290.0374 +- 0.00820.147 +- 0.0333
HAIM----------0.844----
NON_HAIM----------0.831----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9601 +- 0.0113 0.6165 +- 0.0991 0.3786 +- 0.3094 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9922 +- 0.0024 0.2407 +- 0.1973 0.8286 +- 0.0529 \n", - "HAIM -- -- 0.844 \n", - "NON_HAIM -- -- 0.831 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0035 +- 0.0008 0.0353 +- 0.0042 \n", - "test_metrics 0.0374 +- 0.0082 0.147 +- 0.0333 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-09-26 10:03:33,478\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=396015)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=396015)\u001b[0m Configuring PyCaret for outer fold 1\n", + "\u001b[36m(run_fold pid=396009)\u001b[0m Outer fold 2\u001b[32m [repeated 3x across cluster] (Ray deduplicates logs by default. Set RAY_DEDUP_LOGS=0 to disable log deduplication, or see https://docs.ray.io/en/master/ray-observability/user-guides/configure-logging.html#log-deduplication for more options.)\u001b[0m\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0008 +- 0.00070.0139 +- 0.0071
test_metrics0.9735 +- 0.00220.6665 +- 0.03190.5759 +- 0.05730.9968 +- 0.00270.3362 +- 0.06590.7971 +- 0.01520.0246 +- 0.00180.1173 +- 0.0055
HAIM----------0.816----
NON_HAIM----------0.813----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9735 +- 0.0022 0.6665 +- 0.0319 0.5759 +- 0.0573 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.9968 +- 0.0027 0.3362 +- 0.0659 0.7971 +- 0.0152 \n", - "HAIM -- -- 0.816 \n", - "NON_HAIM -- -- 0.813 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0008 +- 0.0007 0.0139 +- 0.0071 \n", - "test_metrics 0.0246 +- 0.0018 0.1173 +- 0.0055 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-22 09:17:01,087\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8266 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=419355)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=419355)\u001b[0m Train indices: [ 0 1 2 ... 14132 14133 14135]\n", + "\u001b[36m(run_fold pid=419355)\u001b[0m Test indices: [ 8 20 21 ... 14118 14126 14134]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[36m(raylet)\u001b[0m Spilled 2262 MiB, 2 objects, write throughput 473 MiB/s. Set RAY_verbose_spill_logs=0 to disable this message.\n", + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0016 +- 0.00190.0288 +- 0.0191
test_metrics0.7282 +- 0.02930.7398 +- 0.02440.7095 +- 0.04030.5433 +- 0.08850.9362 +- 0.04210.8714 +- 0.01260.1528 +- 0.01020.4679 +- 0.0308
HAIM----------0.883----
NON_HAIM----------0.876----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.7282 +- 0.0293 0.7398 +- 0.0244 0.7095 +- 0.0403 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.5433 +- 0.0885 0.9362 +- 0.0421 0.8714 +- 0.0126 \n", - "HAIM -- -- 0.883 \n", - "NON_HAIM -- -- 0.876 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0016 +- 0.0019 0.0288 +- 0.0191 \n", - "test_metrics 0.1528 +- 0.0102 0.4679 +- 0.0308 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-09-27 17:42:40,522\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=678263)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=678263)\u001b[0m Configuring PyCaret for outer fold 1\n", + "\u001b[36m(run_fold pid=678258)\u001b[0m Outer fold 2\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
AccuracyBalancedAccGeoBalancedAccSensitivitySpecificityAUCBrierScoreBCE
train_metrics1.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.01.0 +- 0.00.0002 +- 0.00020.0081 +- 0.0047
test_metrics0.8754 +- 0.01650.6423 +- 0.0350.5295 +- 0.06420.2845 +- 0.06991.0 +- 0.00.8114 +- 0.02080.1066 +- 0.01280.4283 +- 0.0531
HAIM----------0.836----
NON_HAIM----------0.804----
\n", - "" - ], - "text/plain": [ - " Accuracy BalancedAcc GeoBalancedAcc \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.8754 +- 0.0165 0.6423 +- 0.035 0.5295 +- 0.0642 \n", - "HAIM -- -- -- \n", - "NON_HAIM -- -- -- \n", - "\n", - " Sensitivity Specificity AUC \\\n", - "train_metrics 1.0 +- 0.0 1.0 +- 0.0 1.0 +- 0.0 \n", - "test_metrics 0.2845 +- 0.0699 1.0 +- 0.0 0.8114 +- 0.0208 \n", - "HAIM -- -- 0.836 \n", - "NON_HAIM -- -- 0.804 \n", - "\n", - " BrierScore BCE \n", - "train_metrics 0.0002 +- 0.0002 0.0081 +- 0.0047 \n", - "test_metrics 0.1066 +- 0.0128 0.4283 +- 0.0531 \n", - "HAIM -- -- \n", - "NON_HAIM -- -- " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-21 16:37:00,227\tINFO worker.py:1777 -- Started a local Ray instance. View the dashboard at \u001b[1m\u001b[32mhttp://127.0.0.1:8265 \u001b[39m\u001b[22m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[36m(run_fold pid=352119)\u001b[0m Outer fold 1\n", + "\u001b[36m(run_fold pid=352119)\u001b[0m Train indices: [ 0 1 2 ... 17156 17157 17158]\n", + "\u001b[36m(run_fold pid=352119)\u001b[0m Test indices: [ 13 14 15 ... 17135 17147 17155]\n", + "\u001b[36m(run_fold pid=352119)\u001b[0m Configuring PyCaret for outer fold 1\n", + "\u001b[36m(run_fold pid=352122)\u001b[0m Outer fold 2\n", + "\u001b[36m(run_fold pid=352122)\u001b[0m Train indices: [ 0 1 2 ... 17155 17157 17158]\n", + "\u001b[36m(run_fold pid=352122)\u001b[0m Test indices: [ 3 6 8 ... 17152 17154 17156]\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing: 0%| | 0/4 [00:00 List[List[str]]: """ - Function to extract all possible combinations of sources + Function to extract all possible combinations of sources - Args: - sources(List[str]): list of sources types + Args: + sources(List[str]): list of sources types - Returns: list of combinations + Returns: list of combinations """ comb = [] for i in range(len(sources)): @@ -50,15 +47,20 @@ def run_single_experiment(prediction_task: str, dataset: Optional[DataFrame] = None, evaluation_name: Optional[str] = None) -> None: """ - Function to perform one single experiment - - Args: - prediction_task(task): task label, must be a HAIM prediction task - sources_predictors(List[str]): predictors to use for prediction, each source has one or more predictors - sources_modalities(List[str]): the modalities of the sources used for prediction - dataset(Optional[DataFrame]): HAIM dataframe - evaluation_name(Optional[str]): name of the experiment + Function to perform one single experiment + + Args: + prediction_task(task): task label, must be a HAIM prediction task + sources_predictors(List[str]): predictors to use for prediction, each source has one or more predictors + sources_modalities(List[str]): the modalities of the sources used for prediction + dataset(Optional[DataFrame]): HAIM dataframe + evaluation_name(Optional[str]): name of the experiment """ + + # Set up the folder path specific to the prediction task + task_folder = f"experiments/{prediction_task}" + if not os.path.exists(task_folder): + os.makedirs(task_folder) dataset = read_csv(constants.FILE_DF, nrows=constants.N_DATA) if dataset is None else dataset # Create the HAIMDataset @@ -69,47 +71,29 @@ def run_single_experiment(prediction_task: str, constants.IMG_ID, constants.GLOBAL_ID) - # Sample the dataset using a 5-folds cross-validation method - sampler = Sampler(dataset, constants.GLOBAL_ID, 5) - _, masks = sampler() - - # Initialization of the list containing the evaluation metrics - evaluation_metrics = [BinaryAccuracy(), - BinaryBalancedAccuracy(), - BinaryBalancedAccuracy(Reduction.GEO_MEAN), - Sensitivity(), - Specificity(), - AUC(), - BrierScore(), - BinaryCrossEntropy()] - # Define the grid of hyper-parameters for the tuning grid_hps = {'max_depth': [5, 6, 7, 8], 'n_estimators': [200, 300], - 'learning_rate': [0.3, 0.1, 0.05], - } - - # Save the fixed parameters of the model - fixed_params = {'seed': 42, - 'eval_metric': 'logloss', - 'verbosity': 1 - } - - # Launch the evaluation - evaluation = Evaluator(dataset=dataset, - masks=masks, - metrics=evaluation_metrics, - model=XGBClassifier, - tuner=SklearnTuner, - tuning_metric=AUC(), - hps=grid_hps, - n_tuning_splits=5, - fixed_params=fixed_params, - filepath=constants.EXPERIMENT_PATH, - weight='scale_pos_weight', - evaluation_name=evaluation_name - ) - evaluation.evaluate() + 'learning_rate': [0.3, 0.1, 0.05]} + + # Initialize the PyCaret Evaluator + evaluator = PyCaretEvaluator(dataset=dataset, + target=prediction_task, + experiment_name=evaluation_name, + filepath=task_folder) + + # Model training and results evaluation + evaluator.run_experiment( + train_size=0.8, + fold=5, + fold_strategy='stratifiedkfold', + outer_fold=5, + outer_strategy='stratifiedkfold', + session_id=42, + model='xgboost', + optimize='AUC', + custom_grid=grid_hps + ) if __name__ == '__main__': @@ -122,25 +106,34 @@ def run_single_experiment(prediction_task: str, # Load the dataframe from disk df = read_csv(constants.FILE_DF, nrows=constants.N_DATA) - all_tasks = Task() if args.task is None else [args.task] + # Handle all tasks if none specified + all_tasks = [args.task] if args.task else [constants.FRACTURE, constants.PNEUMOTHORAX, constants.PNEUMONIA, + constants.LUNG_OPACITY, constants.LUNG_LESION, constants.ENLARGED_CARDIOMEDIASTINUM, + constants.EDEMA, constants.CONSOLIDATION, constants.CARDIOMEGALY, + constants.ATELECTASIS, constants.LOS, constants.MORTALITY] + for task in all_tasks: - print("#"*23, f"{task} experiment", "#"*23) + print("#" * 23, f"{task} experiment", "#" * 23) + # Get all possible combinations of sources for the current task sources_comb = get_all_sources_combinations(constants.SOURCES) if task in [constants.MORTALITY, constants.LOS] \ else get_all_sources_combinations(constants.CHEST_SOURCES) with tqdm(total=len(sources_comb)) as bar: for count, combination in enumerate(sources_comb): - # Get all predictors and modalities for each source predictors = [] for c in combination: - predictors = predictors + c.sources + predictors.extend(c.sources) # Collect all predictors modalities = unique([c.modality for c in combination]) - run_single_experiment(prediction_task=task, sources_predictors=predictors, sources_modalities=modalities, - dataset=df, evaluation_name=task + '_' + str(count)) + # Run the single experiment + run_single_experiment(prediction_task=task, + sources_predictors=predictors, + sources_modalities=modalities, + dataset=df, + evaluation_name=task + '_' + str(count)) bar.update() - Evaluator.get_best_of_experiments(task, constants.EXPERIMENT_PATH, count) + diff --git a/src/evaluation/evaluating.py b/src/evaluation/evaluating.py deleted file mode 100644 index a463919..0000000 --- a/src/evaluation/evaluating.py +++ /dev/null @@ -1,439 +0,0 @@ -""" -Filename: evaluating.py - -Author : Hakima Laribi - -Description: This file is used to store Evaluator object which performs different evaluations on the dataset - -Date of last modification : 2023/02/07 - -""" -import json -from os import makedirs, path -from re import search -import shutil -from time import strftime -from typing import Any, Callable, Dict, Union, List, Optional - -from src.data.dataset import HAIMDataset -from src.evaluation.tuning import SklearnHpsOptimizer - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -from sklearn.metrics import roc_curve - -from src.data import constants -from src.utils.metric_scores import Metric - - -class Evaluator: - """ - Object used to perform models evaluations - """ - - def __init__(self, - dataset: HAIMDataset, - masks: Dict[int, Dict[str, List[int]]], - metrics: List[Callable], - model: Callable, - tuner: Callable, - tuning_metric: Metric, - hps: Dict[str, Union[List[Any], Any]], - n_tuning_splits: int, - fixed_params: Dict[str, Any], - filepath: str, - model_selector: str = SklearnHpsOptimizer.GS, - parallel_tuning: bool = True, - weight: Optional[str] = None, - evaluation_name: Optional[str] = None, - stratified_sampling: bool = False): - """ - Sets protected attributes of the Evaluator - - Args: - dataset(HAIMDataset): custom HAIM dataset - masks(Dict[int, Dict[str, List[int]]]): dictionary with train, test and valid set at each split - metrics(List[Callable]): list of metrics to report at the end of the experiment - model(Callable): model to evaluate - tuner(Callable): tuner which perform hyper-parameter tuning - tuning_metric(Metric): metric to optimize (maximize/minimize) during the hyper-parameter tuning - n_tuning_splits(int): number of inner data splits, used in the tuning - hps(Dict[str, Union[List[Any], Any]]): each hyper-parameter with its search space - fixed_params(Dict[str, Any]): model's fixed parameters - filepath(str): path to the directory where to store the experiment - model_selector(str): hyper-parameter optimizer - parallel_tuning(hps): boolean to specify if the tuning would be in parallel - weight(str): weight parameter - evaluation_name(str): name of current evaluation - stratified_sampling(bool): if the sampling performed was stratified - - """ - - if evaluation_name is not None: - if path.exists(path.join(filepath, evaluation_name)): - raise ValueError("evaluation with this name already exists") - else: - makedirs(filepath, exist_ok=True) - evaluation_name = f"{strftime('%Y%m%d-%H%M%S')}" - - # Set protected attributes - self._dataset = dataset - self._masks = masks - self._metrics = metrics - self._model = model - self._tuner = tuner(tuning_metric, hps, n_tuning_splits, model_selector, parallel_tuning) - self._inner_splits = n_tuning_splits - self._fixed_params = fixed_params - self._weighted_param = weight - self._filepath = path.join(filepath, evaluation_name) - self._stratified_sampling = stratified_sampling - - # Set public attributes - self.evaluation_name = evaluation_name - - def evaluate(self) -> None: - """ - Performs nested evaluation and saves the result of each experiment in a json file - """ - # Perform the evaluation over all the splits of the dataset - for i, mask in self._masks.items(): - # Extract masks - train, test, valid = mask['train'], mask['test'], mask['valid'] - # Get data and targets for each mask - x, y = {}, {} - for (mask_, idx) in [('train', train), ('test', test), ('valid', valid)]: - if idx is not None: - x[mask_], y[mask_] = self._dataset[idx] - else: - x[mask_], y[mask_] = None, None - - if self._weighted_param is not None: - # Compute weights to assign to the positive class - positive_weight = (len(y['train']) - sum(y['train'])) / sum(y['train']) - # Update the fixed parameters of the model - self._fixed_params[self._weighted_param] = positive_weight - if self._inner_splits > 0: - # Perform the tuning to extract the model with the best hyper-pramaters - best_model = self._tuner.tune(self._model(**self._fixed_params), x['train'], y['train']) - best_hps = self._tuner.get_best_hps() - - else: - best_model = self._model(**self._fixed_params) - best_hps = self._fixed_params - - # Get probabilities predicted for each mask - y_proba = {} - for mask_ in ['train', 'test', 'valid']: - y_proba[mask_] = best_model.predict_proba(x[mask_])[:, 1] if x[mask_] is not None else None - - # Get predictions on the training set and compute the optimal threshold on the training set - threshold = self.optimize_j_statistic(y['train'], y_proba['train']) - - # Save the experiment of the current split in a json file - self.record_experiment(y_proba, y, threshold, mask, i, best_hps) - - # Summarize experiment over all the splits - self.summarize_experiment(i + 1) - - @staticmethod - def optimize_j_statistic(targets: List[int], - pred: List[float]) -> float: - """ - Finds the optimal threshold from ROC curve that separates the negative and positive classes - by optimizing the Youden's J statistics - J = TruePositiveRate – FalsePositiveRate - - Args: - targets(List[int]): ground truth labels - pred(List[float]): predicted probabilities to belong to the positive class - - Returns a float representing the optimal threshold - """ - # Calculate roc curves - fpr, tpr, thresholds = roc_curve(targets, pred, pos_label=1) - - # Get the best threshold - J = tpr - fpr - threshold = thresholds[np.argmax(J)] - - return threshold - - def record_experiment(self, - predictions: Dict[str, List[float]], - targets: Dict[str, List[int]], - threshold: float, - masks: Dict[str, List[int]], - split: int, - hps: Dict[str, Any] - ) -> None: - """ - Records the results of one single experiment on the test, train and valid sets in a json file - - Args: - predictions(Dict[str, List[float]]): probabilities predicted on all the sets - targets(Dict[str, List[int]]): ground truth labels of observation in each set - threshold(float): prediction threshold - masks(Dict[str, List[int]]): train, test and valid masks - split(int): index of the current split - hps(Dict[str, Any]): best hyper-parameters selected after the tuning - - """ - # Create the saving directory - saving_path = self._filepath + '/split_' + str(split) - makedirs(saving_path) - - # Initialize the file structure - summary = {'split': str(split), - 'sources': str(self._dataset.sources), - 'task': self._dataset.task, - 'threshold': str(threshold), - 'hyper-parameters': hps} - # Save statistics of each set - for mask, idx in masks.items(): - if idx is not None: - # Save number of elements in current set - summary['N_' + mask + 'ed'] = len(idx) - - # If observations has a global_id according to which the sampling was performed - if (self._dataset.global_ids is not None) and (not self._stratified_sampling): - summary['N_' + mask + 'ed_global_ids'] = len(self._dataset.get_global_ids(idx)) - - summary[f"proportion_positive_class_{mask}ed"] = '' - - # Initialize the metrics recorded - summary[mask + '_metrics'] = {} - - # Get metrics and prediction values for each mask - for mask, idx in masks.items(): - if idx is not None: - - # Get predicted probabilities and ground truth labels and predicted classes - y_proba, target = predictions[mask], targets[mask] - y_pred = (y_proba >= threshold).astype(float) - - # Save proportion of classes - summary[f"proportion_positive_class_{mask}ed"] = f"{round(np.sum(target) / len(target), 4) * 100} %" - - # Save metrics for current mask - for metric in self._metrics: - summary[mask + '_metrics'][metric.name] = str(metric(y_proba, target, threshold)) - - # Initialize the predictions recorded - summary[mask + '_predictions'] = {} - - # Save predictions for each element - if (self._dataset.global_ids is not None) and (not self._stratified_sampling): - # Map indexes to global ids - map_idx_global_ids = self._dataset.map_idx_to_global_ids() - - # Get the global_ids of the indexes present in the current set - global_ids = self._dataset.get_global_ids(idx) - - # Map indexes to their position in the current mask - map_idx_positions = {index: i for i, index in enumerate(idx)} - - # Map each index to its id in the dataset - map_idx_to_ids = self.reverse_map(self._dataset.map_idx_to_ids()) - - # Save predictions for each global id - for global_id in global_ids: - summary[mask + '_predictions'][str(global_id)] = {} - - # Initialize the information structure for each global_id - summary[mask + '_predictions'][str(global_id)] = {} - - # Get the indexes of the observations present in the global id - indexes = [i for i in map_idx_global_ids[global_id] if i in idx] - - # Save predictions for each observation - for index in indexes: - summary[mask + '_predictions'][str(global_id)][str(index)] = { - 'id': str(map_idx_to_ids[index]), - 'prediction': str(y_pred[map_idx_positions[index]]), - 'probability': str(y_proba[map_idx_positions[index]]), - 'target': str(target[map_idx_positions[index]]) - } - else: - # Map each index to its id in the dataset - map_idx_to_ids = self.reverse_map(self._dataset.map_idx_to_ids()) - - # Save predictions of each observation independently - for i, index in enumerate(idx): - summary[mask + '_predictions'][str(index)] = { - 'index': str(index), - 'id': str(map_idx_to_ids[index]), - 'prediction': str(y_pred[i]), - 'probability': str(y_proba[i]), - 'target': str(target[i]) - } - - # Generate ROC curve - self.plot_roc_curve(saving_path, target, y_proba, mask) - - # Generate the Json file of the split - with open(path.join(saving_path, 'records.json'), "w") as file: - json.dump(summary, file, indent=True) - - def summarize_experiment(self, - n_splits: int - ) -> None: - """ - Summarizes an experiment performed on different splits of the dataset ans saves it in a json file. - The mean, standard deviation, min and max are computed for each metric. - - Args: - n_splits(int): number of splits the model was evaluated on - """ - metrics_values = {} - # Get the folders where each split evaluation was saved - folders = [path.join(self._filepath, 'split_' + str(i)) for i in range(n_splits)] - - for folder in folders: - with open(path.join(folder, 'records.json'), "r") as read_file: - split_data = json.load(read_file) - - # Get metric values over all the splits - for section, data in split_data.items(): - # Get the sections where the metrics are saved - if search("(metric)", section): - # For each split, get the value of the metric - for metric, value in data.items(): - try: - metrics_values[section][metric].append(float(value)) - except KeyError: - try: - metrics_values[section][metric] = [float(value)] - except KeyError: - metrics_values[section] = {metric: [float(value)]} - - # Save statistics on the metrics - recap = {} - for section, data in metrics_values.items(): - recap[section] = {} - for metric, values in data.items(): - values = np.array(values) - mean_scores, std_scores = round(np.mean(values), 4), round(np.std(values), 4) - med_scores = round(np.median(values), 4) - min_scores, max_scores = round(np.min(values), 4), round(np.max(values), 4) - recap[section][metric] = { - 'info': f"{mean_scores} +- {std_scores} [{med_scores}; {min_scores}-{max_scores}]", - 'mean': mean_scores, - 'std': std_scores, - } - - # Save the file in disk - with open(path.join(self._filepath, 'recap.json'), "w") as file: - json.dump(recap, file, indent=True) - - @staticmethod - def visualize_results(file_path: str, - task: str, - recap_file: str = None, - ) -> pd.DataFrame: - """ - Saves metrics scores regrouped in the recap json file in a dataframe and prints it - - Args: - file_path(str): directory where the experiment is saved - task(str): prediction task - recap_file(str): recap json file of the experiment - """ - recap_file = 'recap.json' if recap_file is None else recap_file - with open(path.join(file_path, recap_file), "r") as file: - recap = json.load(file) - - metrics = {} - # Get the mean and std for each metric in all the sets (train, test and valid) - for _set, values in recap.items(): - for metric, stats in values.items(): - try: - metrics[metric][_set] = str(stats['mean']) + ' +- ' + str(stats['std']) - except KeyError: - metrics[metric] = {_set: str(stats['mean']) + ' +- ' + str(stats['std'])} - - for _set in ['HAIM', 'NON_HAIM']: - for metric in metrics.keys(): - if metric == 'AUC': - metrics[metric][_set] = str(constants.AUC[_set][task]) - else: - metrics[metric][_set] = '--' - - # Transform the dictionary to a dataframe - df_metrics = pd.DataFrame(metrics) - - return df_metrics - - @staticmethod - def get_best_of_experiments(task: str, - path_file: str, - n_experiments: int, - metric: str = 'AUC') -> None: - """ - Gets the experiment with the best metric value from a set of saved experiments - - Args: - task(str): file name format of the experiments - path_file(str): path to directory where the experiments are saved - n_experiments(int): number of experiments to compare - metric(str): metric name - """ - metric_values = [] - # Get the folders where each recap evaluation was saved - folders = [path.join(path_file, task + '_' + str(i)) for i in range(n_experiments)] - - for folder in folders: - with open(path.join(folder, 'recap.json'), "r") as read_file: - recap_data = json.load(read_file) - # Get AUC values of all experiments - infos = recap_data["test_metrics"][metric] - metric_values.append(float(infos['mean'])) - - best_experiment = np.argmax(np.array(metric_values)) - - # Copy the files of the best experiment to the directory file_format_best_experiment - shutil.copytree(folders[best_experiment], path.join(path_file, task + '_best_experiment')) - - @staticmethod - def reverse_map(map_: Dict[Any, Any]) -> Dict[Any, Any]: - """ - Reverses the keys and values of a dictionary - - Args: - map_(Dict[Any, Any]): dictionary - - Returns: a dictionary - """ - reversed_map = {} - for k, v in map_.items(): - for value in v: - reversed_map[value] = k - return reversed_map - - @staticmethod - def plot_roc_curve( - saving_path: str, - targets: np.array, - y_proba: np.array, - mask: str) -> None: - - """ - Plots the Area Under AUC curve and saves it - - Args: - saving_path(str): path where to save the figure - targets(np.array): ground truth labels - y_proba(np.array): probabilities predicted - mask(str): label of the current mask - """ - - fpr, tpr, _ = roc_curve(targets, y_proba, pos_label=1) - - # create ROC curve - plt.clf() - plt.plot(fpr, tpr) - plt.ylabel('True Positive Rate') - plt.xlabel('False Positive Rate') - - # Save the figure in the disk - plt.savefig(path.join(saving_path, 'roc_curve_' + mask + '.png')) diff --git a/src/evaluation/pycaret_evaluator.py b/src/evaluation/pycaret_evaluator.py new file mode 100644 index 0000000..51ba466 --- /dev/null +++ b/src/evaluation/pycaret_evaluator.py @@ -0,0 +1,235 @@ +import gc # Garbage collector to free memory after each fold +import json +import os +from time import strftime +from typing import Any, Dict, List, Optional, Union + +import numpy as np +import pandas as pd +from sklearn.model_selection import StratifiedKFold, KFold + +import ray # Import Ray +from pycaret.classification import (create_model, predict_model, pull, save_model, + setup, tune_model) + +os.environ["RAY_DEDUP_LOGS"] = "0" # Disables log deduplication in Ray + + +class PyCaretEvaluator: + """ + Class to evaluate models using PyCaret, optimized for memory management. + """ + + def __init__(self, dataset: Any, target: str, experiment_name: Optional[str], filepath: str, columns: Optional[List[str]] = None): + """ + Initialize the class parameters. + + Args: + dataset (Any): the used dataset for the task. + target (str): the target class. + experiment_name (Optional[str]): optional name for the experiment. + filepath (str): path for saving results. + columns (Optional[List[str]]): optional list of column names. + """ + # Initialize instance variables + self.dataset = dataset + self.target = target + self.experiment_name = experiment_name if experiment_name else f"experiment_{strftime('%Y%m%d-%H%M%S')}" + self.filepath = filepath + self.columns = columns + + # Create the directory if it doesn't exist + if not os.path.exists(self.filepath): + os.makedirs(self.filepath) + + def save_results(self, results: List[Dict], filename: str) -> None: + """ + Save the results in a JSON file. + + Args: + results (List[Dict]): results to save. + filename (str): file name where the results are saved. + """ + with open(os.path.join(self.filepath, filename), 'w', encoding='utf-8') as f: + json.dump(results, f, indent=4) + + @ray.remote(memory=8e9) # Mark this function to be executed in parallel by Ray + def run_fold(self, + train_index: np.ndarray, + test_index: np.ndarray, + fold_num: int, + train_size: float, + fold: int, + fold_strategy: str, + session_id: int, + model: str, + optimize: Union[str, List[str]], + custom_grid: Optional[Dict[str, List[Any]]], + search_algorithm: str, + fixed_params: Dict[str, Any]) -> Dict[str, Any]: + """ + Run a single fold in parallel using Ray. + + Args: + train_index (np.ndarray): Array of training indices for the fold. + test_index (np.ndarray): Array of testing indices for the fold. + fold_num (int): Current fold number. + train_size (float): Proportion of data to use for training within the fold. + fold (int): Number of folds for inner cross-validation. + fold_strategy (str): Strategy for inner cross-validation (e.g., 'kfold', 'stratifiedkfold'). + session_id (int): Random seed for reproducibility. + model (str): Name of the model to be created (e.g., 'xgboost', 'lightgbm'). + optimize (Union[str, List[str]]): Metric(s) to optimize during model tuning (e.g., 'AUC', 'Accuracy'). + custom_grid (Optional[Dict[str, List[Any]]]): Custom hyperparameter grid for tuning the model. + search_algorithm (str): Hyperparameter search algorithm to use ('grid' or 'random'). + fixed_params (Dict[str, Any]): Dictionary of fixed parameters for model setup (e.g., seed, eval_metric). + + Returns: + Dict[str, Any]: A dictionary containing training results, test predictions, and the best hyperparameters for the fold. + """ + print(f"Outer fold {fold_num}") + + # Extract training and testing subsets + train_data_x = self.dataset.x[train_index] + train_data_y = self.dataset.y[train_index] + test_data_x = self.dataset.x[test_index] + test_data_y = self.dataset.y[test_index] + + # Convert NumPy arrays to DataFrames for PyCaret setup + train_df = pd.DataFrame(train_data_x, columns=self.columns) + train_df[self.target] = train_data_y + test_df = pd.DataFrame(test_data_x, columns=self.columns) + test_df[self.target] = test_data_y + + print(f"Train indices: {train_index}") + print(f"Test indices: {test_index}") + + # Configure PyCaret for the current fold + exp = setup(data=train_df, + target=self.target, + train_size=train_size, + fold=fold, + fold_strategy=fold_strategy, + session_id=fixed_params['seed'], + verbose=False, + n_jobs=1) + + print(f"Configuring PyCaret for outer fold {fold_num}") + + # Create and tune the specified model + best_model = create_model(model, fold=fold) + + if custom_grid: + print(f"Tuning hyperparameters for model {model} with custom grid using {search_algorithm} search") + best_model = tune_model(best_model, custom_grid=custom_grid, fold=fold, optimize=optimize, search_algorithm=search_algorithm, verbose=False) + + # Extract the best hyperparameters after tuning + best_hyperparams = best_model.get_params() + else: + best_hyperparams = best_model.get_params() # Default parameters if no tuning + + # Get the results and predictions + model_results = pull() # Pull the results after create_model or tune_model + save_model(best_model, os.path.join(self.filepath, f"best_model_fold_{fold_num}")) + test_predictions = predict_model(best_model, data=test_df) + + # Save fold results + split_result = { + 'fold': fold_num, + 'train_results': model_results.to_dict(), + 'test_predictions': test_predictions.to_dict(), + 'best_hyperparams': best_hyperparams # Save best hyperparameters for this fold + } + + # Clean up memory after each fold (memory management) + del train_df, test_df, best_model, model_results, test_predictions, exp + gc.collect() + + return split_result + + def run_experiment(self, + train_size: float = 0.8, + fold: int = 5, + fold_strategy: str = 'stratifiedkfold', + outer_fold: int = 5, + outer_strategy: str = 'stratifiedkfold', + session_id: int = 42, + model: Optional[str] = 'xgboost', + optimize: Union[str, List[str]] = 'AUC', + custom_grid: Optional[Dict[str, List[Any]]] = None, + search_algorithm: str = 'grid', + fixed_params: Dict[str, Any] = None) -> None: + """ + Runs the entire experiment, including external cross-validation, training, and model optimization. + + Args: + train_size (float): Proportion of the dataset to include in the training split. + fold (int): Number of folds for internal cross-validation. + fold_strategy (str): Strategy for internal cross-validation ('kfold', 'stratifiedkfold'). + outer_fold (int): Number of folds for external cross-validation. + outer_strategy (str): Strategy for external cross-validation ('kfold', 'stratifiedkfold'). + session_id (int): Session ID for reproducibility. + model (Optional[str]): Specific model to use. + optimize (Union[str, List[str]]): The metric to optimize. + custom_grid (Optional[Dict[str, List[Any]]]): Custom grid of parameters for tuning. + search_algorithm (str): Algorithm to use for hyperparameter tuning ('grid' or 'random'). + fixed_params (Dict[str, Any]): Fixed parameters such as seed and eval_metric. + """ + # Params fixed by the original study + if fixed_params is None: + fixed_params = {'seed': 42, 'eval_metric': 'logloss', 'verbosity': 0} + + # Define the outer cross-validation strategy + if outer_strategy == 'stratifiedkfold': + outer_cv = StratifiedKFold(n_splits=outer_fold, shuffle=True, random_state=session_id) + elif outer_strategy == 'kfold': + outer_cv = KFold(n_splits=outer_fold, shuffle=True, random_state=session_id) + else: + raise ValueError(f"Unknown outer_strategy: {outer_strategy}") + + ray.init(ignore_reinit_error=True, num_cpus=os.cpu_count()) # Initialize Ray with available CPUs + ray_tasks = [] # List to store Ray tasks + + # Generate Ray tasks for each fold + for i, (train_index, test_index) in enumerate(outer_cv.split(self.dataset.x, self.dataset.y)): + ray_task = self.run_fold.remote(self, train_index, test_index, i + 1, train_size, fold, fold_strategy, session_id, model, + optimize, custom_grid, search_algorithm, fixed_params) + ray_tasks.append(ray_task) + + # Execute and collect results of Ray tasks + results = ray.get(ray_tasks) + self.save_results(results, f"{self.experiment_name}_results.json") + + # Collect and compute final metrics after training + fold_metrics_list = [] + best_hyperparams_list = [] + + for result in results: + train_results = result.get('train_results', {}) + best_hyperparams = result.get('best_hyperparams', {}) + if isinstance(train_results, dict): + metrics_df = pd.DataFrame(train_results, index=[0]) + fold_metrics_list.append(metrics_df) + if best_hyperparams: + best_hyperparams_list.append(best_hyperparams) + + # Calculate and save the mean and standard deviation of metrics + if fold_metrics_list: + all_fold_metrics = pd.concat(fold_metrics_list, ignore_index=True) + final_metrics_mean = all_fold_metrics.mean() + final_metrics_std = all_fold_metrics.std() + + metrics_table = pd.DataFrame({'Metric': final_metrics_mean.index, 'Mean': final_metrics_mean.values, + 'Std Dev': final_metrics_std.values}) + print("Final metrics table:") + print(metrics_table) + + metrics_table.to_csv(os.path.join(self.filepath, f"{self.experiment_name}_final_metrics.csv"), index=False) + + # Determine the most common hyperparameters across folds + if best_hyperparams_list: + best_hyperparams_df = pd.DataFrame(best_hyperparams_list) + most_common_hyperparams = best_hyperparams_df.mode().iloc[0] # Most frequent hyperparameters across folds + print(f"Best hyperparameters across all folds: {most_common_hyperparams}") + + ray.shutdown() diff --git a/src/evaluation/tuning.py b/src/evaluation/tuning.py deleted file mode 100644 index 6851b48..0000000 --- a/src/evaluation/tuning.py +++ /dev/null @@ -1,159 +0,0 @@ -""" -Filename: Tuning.py - -Author : Hakima Laribi - -Description: This file is used to store objects used for tuning - -Date of last modification : 2023/02/06 - -""" -from abc import abstractmethod, ABC -from typing import Callable, Dict, List, Any, Union - -from numpy import array -from sklearn.base import BaseEstimator -from sklearn.metrics import make_scorer -from sklearn.model_selection import GridSearchCV, RandomizedSearchCV - -from src.utils.metric_scores import Metric - - -class SklearnHpsOptimizer: - """ - Object used to store Scikit-learn hyper-parameter optimizers labels - """ - - GS = 'grid_search' - RS = 'random_search' - - def __iter__(self): - return iter([self.GS, self.RS]) - - def __getitem__(self, item: str) -> Union[Callable, None]: - if item == self.GS: - return GridSearchCV - elif item == self.RS: - return RandomizedSearchCV - else: - raise ValueError(f"{item}: hyper-parameters optimizer not supported") - - -class Tuner(ABC): - """ - Abstract class used to define tuner skeleton - """ - - def __init__(self, - metric: Metric, - hps: Dict[str, List[Any]], - n_splits: int, - parallel: bool = True): - """ - Sets protected attributes - - Args: - metric(Metric): callable function to optimize - hps(Dict[str, List[Any]]): dictionary with the hyper-parameters to optimize and the corresponding values - to explore - n_splits(int): number of inner splits on which test each combination of hyper-parameters - parallel(bool): whether to run the tuning in parallel or not - """ - # Set protected attributes - self._metric = metric - self._hps = hps - self._n_splits = n_splits - self._n_cpus = -1 if parallel else 1 # Use all cpus for parallel tuning or a single one only - - @abstractmethod - def tune(self, - model: Any, - x: array, - y: array): - """ - Performs the tuning of a model on specific data - - Args: - model(Any): the model to which find the best combination of hyper-parameters - x(array): (N, D) array where N is the number of observations and D the number of predictors - y(array): (N, 1) ground truth labels - """ - raise NotImplementedError - - @abstractmethod - def get_best_hps(self): - """ - Returns the combination of hyper-parameters which optimized the metric value - """ - raise NotImplementedError - - -class SklearnTuner(Tuner): - """ - Object used to perform hyper-parameter tuning using Scikit-learn optimizers - """ - - def __init__(self, - metric: Metric, - hps: Dict[str, List[Any]], - n_splits: int, - model_selector: str = SklearnHpsOptimizer.GS, - parallel: bool = True): - """ - Sets protected attributes - - Args: - metric(Metric): callable function to optimize - hps(Dict[str, List[Any]]): dictionary with the hyper-parameters to optimize and the corresponding values - to explore - n_splits(int): number of inner splits on which test each combination of hyper-parameters - model_selector(str): sckit-learn hyper-parameters optimizer - parallel(bool): whether to run the tuning in parallel or not - - """ - # Validation of inputs - if model_selector not in SklearnHpsOptimizer(): - raise ValueError(f"{model_selector}: unsupported hyper-parameters optimizer in Scikit-Learn") - - # Create a custom scikit-learn score metric - metric = make_scorer(metric, greater_is_better=(metric.direction == 'maximize')) - - # Call parent constructor - super().__init__(metric, hps, n_splits, parallel) - - # Get the Hyper-parameter optimizer in Scikit-learn - self._hps_optimizer = SklearnHpsOptimizer()[model_selector] - self._optimizer = None - - def tune(self, - model: BaseEstimator, - x: array, - y: array) -> BaseEstimator: - """ - Performs the tuning of a model on specific data using a Scikit-learn Hyper-parameters optimizer - - Args: - model(Any): the model to which find the best combination of hyper-parameters - x(array): (N, D) array where N is the number of observations and D the number of predictors - y(array): (N, 1) ground truth labels - - Returns: - Scikit-Learn optimized model - """ - if not isinstance(model, BaseEstimator): - raise ValueError(f"{model} is not a Scikit-Learn estimator, cannot perform tuning with SKlearnTuner") - - # instantiate the hyper-parameter Sklearn optimizer - self._optimizer = self._hps_optimizer(model, self._hps, scoring=self._metric, cv=self._n_splits, - n_jobs=self._n_cpus, refit=True, verbose=0) - # Launch the hyper-parameter optimization - self._optimizer.fit(x, y) - - # return the model with best hyper-parameters - return self._optimizer.best_estimator_ - - def get_best_hps(self) -> Dict[str, Any]: - """ - Returns the combination of hyper-parameters which optimized the metric value - """ - return self._optimizer.best_params_