add DeepExtreme dataset

.gitignore

@@ -152,3 +152,4 @@ checkpoints/*
 *.png
 *.nc
 *.zarr
+*.slurm

README.md

@@ -23,7 +23,7 @@ We recommend using Anaconda for managing dependencies of this library. The follo
 conda create -n emp -c conda-forge python 
 conda activate emp
 pip install torch torchvision --index-url https://download.pytorch.org/whl/cu126
-pip install numpy matplotlib pillow xarray zarr netcdf4 scipy imageio networkx scikit-image s3fs pyproj pyarrow seaborn cartopy tensorboard dask pytorch-lightning torchmetrics statsmodels earthnet earthnet-minicuber segmentation-models-pytorch albumentations ipykernel
+pip install numpy matplotlib pillow xarray zarr netcdf4 scipy imageio networkx scikit-image s3fs pyproj pyarrow seaborn cartopy tensorboard dask pytorch-lightning torchmetrics statsmodels earthnet earthnet-minicuber segmentation-models-pytorch albumentations ipykernel geopandas
 ```
 
 ## Installation

configs/de23/convlstm_ae/all_variables.yaml

@@ -0,0 +1,84 @@
+Architecture: "convlstm_ae"
+
+Seed: 42
+
+Setting: "de23"
+
+Logger:
+    save_dir: "experiments/"
+
+
+Checkpointer:
+    save_top_k: 1
+    save_last: True
+    every_n_epochs: 1
+
+Trainer:
+    gpus: 1
+    strategy: 'ddp_find_unused_parameters_false'
+    #deterministic: True
+    log_every_n_steps: 32
+    profiler: 'simple'
+    accumulate_grad_batches: 8
+    #fast_dev_run: True
+    #log_gpu_memory: 'all'
+    #weights_summary: 'full'
+    max_epochs: 30
+    #limit_train_batches: 32
+    #limit_val_batches: 32
+    gradient_clip_val: 1
+    # check_val_every_n_epoch: 1 # default setting
+    # Whether to use torch.inference_mode() or torch.no_grad() mode during evaluation
+    # inference_mode: True # if False use 'torch.no_grad' instead. 
+    val_check_interval: 0.25 #0.005 # 
+
+Data:
+    base_dir: "/Net/Groups/BGI/tscratch/mweynants/dx-minicubes"  # "/Net/Groups/BGI/work_2/scratch/DeepExtremes/dx-minicubes" # 
+    test_fold: 10
+    val_fold: 9
+    fold_path: "/Net/Groups/BGI/work_2/scratch/DeepExtremes/mc_earthnet.csv"
+    test_track: "iid"
+    target: "ndvi"
+    train_batch_size: 4
+    val_batch_size: 4
+    test_batch_size: 4
+    num_workers: 4
+
+
+Task:
+    loss:
+        name: "MaskedPixelwiseLoss"
+        # lc_min: 40
+        # lc_max: 90
+        ndvi_pred_idx: 0 # index of the NDVI band
+        ndvi_targ_idx: 0 # index of the NDVI band
+        pred_mask_value: -1
+        scale_by_std: False
+    context_length: 73
+    target_length: 17
+    metric: "RMSE"
+    n_stochastic_preds: 2
+    optimization:
+        optimizer:
+            - 
+                name: AdamW
+                args: 
+                    betas: [0.9, 0.999]
+                lr_per_sample: 0.0001
+        lr_shedule:
+            -
+                name: MultiStepLR
+                args:
+                    milestones: [3, 6] #[2, 20, 50, 90]
+                    gamma: 0.1
+    n_log_batches: 2
+    compute_metric_on_test: True
+
+Model:
+    hidden_dim: [64, 64, 64, 64]
+    kernel_size: 3
+    bias: True
+    skip_connections: True
+    num_inputs: 33 # 1 (target) + 7 (s2) + 24 (meteo) + 1 (topo)
+    num_outputs: 1 
+    teacher_forcing: True

configs/de23/convlstm_ae/lr_1e-3.yaml

@@ -0,0 +1,84 @@
+Architecture: "convlstm_ae"
+
+Seed: 42
+
+Setting: "de23"
+
+Logger:
+    save_dir: "experiments/"
+
+
+Checkpointer:
+    save_top_k: 1
+    save_last: True
+    every_n_epochs: 1
+
+Trainer:
+    gpus: 1
+    strategy: 'ddp_find_unused_parameters_false'
+    #deterministic: True
+    log_every_n_steps: 32
+    #profiler: 'simple'
+    accumulate_grad_batches: 8
+    #fast_dev_run: True
+    #log_gpu_memory: 'all'
+    #weights_summary: 'full'
+    max_epochs: 30
+    #limit_train_batches: 32
+    #limit_val_batches: 32
+    gradient_clip_val: 1
+    # check_val_every_n_epoch: 1 # default setting
+    # Whether to use torch.inference_mode() or torch.no_grad() mode during evaluation
+    # inference_mode: True # if False use 'torch.no_grad' instead. 
+    val_check_interval: 0.25 #0.005 # 
+
+Data:
+    base_dir: "/Net/Groups/BGI/work_2/scratch/DeepExtremes/dx-minicubes" # 
+    test_fold: 10
+    val_fold: 9
+    fold_path: "/Net/Groups/BGI/work_2/scratch/DeepExtremes/mc_earthnet.csv"
+    test_track: "iid"
+    target: "ndvi"
+    train_batch_size: 4
+    val_batch_size: 4
+    test_batch_size: 4
+    num_workers: 4
+
+
+Task:
+    loss:
+        name: "MaskedPixelwiseLoss"
+        # lc_min: 40
+        # lc_max: 90
+        ndvi_pred_idx: 0 # index of the NDVI band
+        ndvi_targ_idx: 0 # index of the NDVI band
+        pred_mask_value: -1
+        scale_by_std: False
+    context_length: 72
+    target_length: 18
+    metric: "RMSE"
+    n_stochastic_preds: 2
+    optimization:
+        optimizer:
+            - 
+                name: AdamW
+                args: 
+                    betas: [0.9, 0.999]
+                lr_per_sample: 0.001
+        lr_shedule:
+            -
+                name: MultiStepLR
+                args:
+                    milestones: [50,90] #[3, 6] #[2, 20, 50, 90]
+                    gamma: 0.1
+    n_log_batches: 2
+    compute_metric_on_test: True
+
+Model:
+    hidden_dim: [64, 64, 64, 64]
+    kernel_size: 3
+    bias: True
+    skip_connections: True
+    num_inputs: 33 # 1 (target) + 7 (s2) + 24 (meteo) + 1 (topo)
+    num_outputs: 1 
+    teacher_forcing: True

configs/de23/convlstm_ae/lr_1e-4_continent_1.yaml

@@ -0,0 +1,85 @@
+Architecture: "convlstm_ae"
+
+Seed: 42
+
+Setting: "de23"
+
+Logger:
+    save_dir: "experiments/"
+
+
+Checkpointer:
+    save_top_k: 1
+    save_last: True
+    every_n_epochs: 1
+
+Trainer:
+    gpus: 1
+    strategy: 'ddp_find_unused_parameters_false'
+    #deterministic: True
+    log_every_n_steps: 32
+    profiler: 'simple'
+    accumulate_grad_batches: 8
+    #fast_dev_run: True
+    #log_gpu_memory: 'all'
+    #weights_summary: 'full'
+    max_epochs: 10
+    #limit_train_batches: 32
+    #limit_val_batches: 32
+    gradient_clip_val: 1
+    # check_val_every_n_epoch: 1 # default setting
+    # Whether to use torch.inference_mode() or torch.no_grad() mode during evaluation
+    # inference_mode: True # if False use 'torch.no_grad' instead. 
+    val_check_interval: 0.25 #0.005 # 
+
+Data:
+    base_dir: "/Net/Groups/BGI/tscratch/crobin/dx-minicubes_interpolated"  # "/Net/Groups/BGI/work_2/scratch/DeepExtremes/dx-minicubes" # 
+    test_fold: 10
+    val_fold: 9
+    fold_path: "/Net/Groups/BGI/work_2/scratch/DeepExtremes/mc_earthnet.csv"
+    test_track: "iid"
+    target: "ndvi"
+    continent_split: True
+    train_batch_size: 4
+    val_batch_size: 4
+    test_batch_size: 4
+    num_workers: 4
+
+
+Task:
+    loss:
+        name: "MaskedPixelwiseLoss"
+        # lc_min: 40
+        # lc_max: 90
+        ndvi_pred_idx: 0 # index of the NDVI band
+        ndvi_targ_idx: 0 # index of the NDVI band
+        pred_mask_value: -1
+        scale_by_std: False
+    context_length: 73
+    target_length: 17
+    metric: "RMSE"
+    n_stochastic_preds: 2
+    optimization:
+        optimizer:
+            - 
+                name: AdamW
+                args: 
+                    betas: [0.9, 0.999]
+                lr_per_sample: 0.0001
+        lr_shedule:
+            -
+                name: MultiStepLR
+                args:
+                    milestones: [1, 7] #[3, 6] #[2, 20, 50, 90]
+                    gamma: 0.1
+    n_log_batches: 2
+    compute_metric_on_test: True
+
+Model:
+    hidden_dim: [64, 64, 64, 64]
+    kernel_size: 3
+    bias: True
+    skip_connections: True
+    num_inputs: 33 # 1 (target) + 7 (s2) + 24 (meteo) + 1 (topo)
+    num_outputs: 1 
+    teacher_forcing: True

configs/de23/convlstm_ae/lr_1e-4_continent_2.yaml

@@ -0,0 +1,85 @@
+Architecture: "convlstm_ae"
+
+Seed: 42
+
+Setting: "de23"
+
+Logger:
+    save_dir: "experiments/"
+
+
+Checkpointer:
+    save_top_k: 1
+    save_last: True
+    every_n_epochs: 1
+
+Trainer:
+    gpus: 1
+    strategy: 'ddp_find_unused_parameters_false'
+    #deterministic: True
+    log_every_n_steps: 32
+    profiler: 'simple'
+    accumulate_grad_batches: 8
+    #fast_dev_run: True
+    #log_gpu_memory: 'all'
+    #weights_summary: 'full'
+    max_epochs: 10
+    #limit_train_batches: 32
+    #limit_val_batches: 32
+    gradient_clip_val: 1
+    # check_val_every_n_epoch: 1 # default setting
+    # Whether to use torch.inference_mode() or torch.no_grad() mode during evaluation
+    # inference_mode: True # if False use 'torch.no_grad' instead. 
+    val_check_interval: 0.25 #0.005 # 
+
+Data:
+    base_dir: "/Net/Groups/BGI/tscratch/crobin/dx-minicubes_interpolated"  # "/Net/Groups/BGI/work_2/scratch/DeepExtremes/dx-minicubes" # 
+    test_fold: 8
+    val_fold: 7
+    fold_path: "/Net/Groups/BGI/work_2/scratch/DeepExtremes/mc_earthnet.csv"
+    test_track: "iid"
+    target: "ndvi"
+    continent_split: True
+    train_batch_size: 4
+    val_batch_size: 4
+    test_batch_size: 4
+    num_workers: 4
+
+
+Task:
+    loss:
+        name: "MaskedPixelwiseLoss"
+        # lc_min: 40
+        # lc_max: 90
+        ndvi_pred_idx: 0 # index of the NDVI band
+        ndvi_targ_idx: 0 # index of the NDVI band
+        pred_mask_value: -1
+        scale_by_std: False
+    context_length: 73
+    target_length: 17
+    metric: "RMSE"
+    n_stochastic_preds: 2
+    optimization:
+        optimizer:
+            - 
+                name: AdamW
+                args: 
+                    betas: [0.9, 0.999]
+                lr_per_sample: 0.0001
+        lr_shedule:
+            -
+                name: MultiStepLR
+                args:
+                    milestones: [1, 7] #[3, 6] #[2, 20, 50, 90]
+                    gamma: 0.1
+    n_log_batches: 2
+    compute_metric_on_test: True
+
+Model:
+    hidden_dim: [64, 64, 64, 64]
+    kernel_size: 3
+    bias: True
+    skip_connections: True
+    num_inputs: 33 # 1 (target) + 7 (s2) + 24 (meteo) + 1 (topo)
+    num_outputs: 1 
+    teacher_forcing: True