Ts embedding

.gitignore

@@ -237,3 +237,5 @@ dl/notebooks/**/*.ipynb
 !dl/notebooks/**/tree_darts.ipynb
 dl/notebooks/lightning_logs/*
 dl/notebooks/tmp/*
+
+dl/notebooks/data/predictive_coding/

dl/notebooks/configs/predictive_coding/config.binaryheartbeat.yaml

@@ -0,0 +1,42 @@
+data:
+  dataset_name: BinaryHeartbeat
+  data_dir: data/predictive_coding
+
+dataloader:
+  batch_size: 32
+  drop_last: false
+  num_workers: 0
+  val_split: 0.1
+
+encoder:
+  hidden_size: 64
+  norm_mode: layerNorm
+  conv_layers:
+    - {kernel_size: 10, stride: 5, padding: 3}
+    - {kernel_size: 8,  stride: 4, padding: 2}
+    - {kernel_size: 4,  stride: 2, padding: 1}
+    - {kernel_size: 4,  stride: 2, padding: 1}
+
+ar:
+  mode: GRU
+  hidden_size: 200
+  n_levels: 1
+  reverse: true
+
+loss:
+  n_predictions: 8
+  negative_sampling_ext: 128
+  temperature: 0.07
+
+optimizer:
+  learning_rate: 0.0002
+  weight_decay: 0.000001
+
+trainer:
+  epochs: 300
+  output_dir: lightning_logs/binaryheartbeat_cpc
+
+runtime:
+  seed: 137
+  accelerator: auto
+  devices: 1

dl/notebooks/configs/predictive_coding/config.ecg200.yaml

@@ -0,0 +1,51 @@
+data:
+  dataset_name: ECG200
+  data_dir: data/predictive_coding
+
+dataloader:
+  batch_size: 32
+  drop_last: false
+  num_workers: 0
+  val_split: 0.1
+
+encoder:
+  hidden_size: 50
+  norm_mode: layerNorm
+  conv_layers:
+    - {kernel_size: 10, stride: 5, padding: 3}
+    - {kernel_size: 8,  stride: 4, padding: 2}
+    - {kernel_size: 3,  stride: 1, padding: 1}
+    # - {kernel_size: 4,  stride: 2, padding: 1}
+
+ar:
+  # mode: LSTM
+  # mode: transformer
+  mode: GRU
+  # mode: no_ar
+  hidden_size: 150
+  n_levels: 1
+  reverse: true
+  # transformer:
+  #   layers: 1
+  #   heads: 4
+  #   dropout: 0.1
+  #   max_seq_len: 256
+  #   abspos: false
+
+loss:
+  n_predictions: 2
+  negative_sampling_ext: 64
+  temperature: 0.07
+
+optimizer:
+  learning_rate: 0.0002
+  weight_decay: 0.000001
+
+trainer:
+  epochs: 400
+  output_dir: lightning_logs/ecg200_cpc
+
+runtime:
+  seed: 137
+  accelerator: auto
+  devices: 1

dl/notebooks/configs/predictive_coding/config.ecg5000.yaml

@@ -0,0 +1,41 @@
+data:
+  dataset_name: ECG5000
+  data_dir: data/predictive_coding
+
+dataloader:
+  batch_size: 64
+  drop_last: false
+  num_workers: 0
+  val_split: 0.1
+
+encoder:
+  hidden_size: 64
+  norm_mode: layerNorm
+  conv_layers:
+    - {kernel_size: 10, stride: 5, padding: 3}
+    - {kernel_size: 8,  stride: 4, padding: 2}
+    - {kernel_size: 4,  stride: 1, padding: 1}
+
+ar:
+  mode: GRU
+  hidden_size: 200
+  n_levels: 2
+  reverse: true
+
+loss:
+  n_predictions: 1
+  negative_sampling_ext: 128
+  temperature: 0.07
+
+optimizer:
+  learning_rate: 0.0002
+  weight_decay: 0.000001
+
+trainer:
+  epochs: 500
+  output_dir: lightning_logs/ecg5000_cpc
+
+runtime:
+  seed: 137
+  accelerator: auto
+  devices: 1

dl/notebooks/configs/predictive_coding/config.forda.yaml

@@ -0,0 +1,40 @@
+data:
+  dataset_name: FordA
+  data_dir: data/predictive_coding
+
+dataloader:
+  batch_size: 128
+  drop_last: false
+  num_workers: 0
+  val_split: 0.1
+
+encoder:
+  hidden_size: 128
+  norm_mode: layerNorm
+  conv_layers:
+    - {kernel_size: 10, stride: 5, padding: 3}
+    - {kernel_size: 8,  stride: 4, padding: 2}
+
+ar:
+  mode: GRU
+  hidden_size: 256
+  n_levels: 1
+  reverse: true
+
+loss:
+  n_predictions: 6
+  negative_sampling_ext: 256
+  temperature: 0.07
+
+optimizer:
+  learning_rate: 0.0002
+  weight_decay: 0.000001
+
+trainer:
+  epochs: 250
+  output_dir: lightning_logs/forda_cpc
+
+runtime:
+  seed: 137
+  accelerator: auto
+  devices: 1

dl/notebooks/configs/predictive_coding/config.sleep.yaml

@@ -0,0 +1,41 @@
+data:
+  dataset_name: Sleep
+  data_dir: data/predictive_coding
+
+dataloader:
+  batch_size: 256
+  drop_last: false
+  num_workers: 0
+  val_split: 0.02
+
+encoder:
+  hidden_size: 128
+  norm_mode: layerNorm
+  conv_layers:
+    - {kernel_size: 10, stride: 5, padding: 3}
+    - {kernel_size: 8,  stride: 4, padding: 2}
+    - {kernel_size: 3,  stride: 1, padding: 1}
+
+ar:
+  mode: GRU
+  hidden_size: 256
+  n_levels: 1
+  reverse: true
+
+loss:
+  n_predictions: 4
+  negative_sampling_ext: 256
+  temperature: 0.07
+
+optimizer:
+  learning_rate: 0.0002
+  weight_decay: 0.000001
+
+trainer:
+  epochs: 300
+  output_dir: lightning_logs/sleep_cpc
+
+runtime:
+  seed: 137
+  accelerator: auto
+  devices: 1

dl/notebooks/configs/predictive_coding/config.synth.yaml

@@ -0,0 +1,41 @@
+encoder:
+  hidden_size: 50
+  norm_mode: layerNorm
+  conv_layers:
+    - {kernel_size: 10, stride: 5, padding: 3}
+    - {kernel_size: 8,  stride: 4, padding: 2}
+    - {kernel_size: 3,  stride: 1, padding: 1}
+    # - {kernel_size: 4,  stride: 2, padding: 1}
+
+ar:
+  # mode: LSTM
+  # mode: transformer
+  mode: GRU
+  # mode: no_ar
+  hidden_size: 150
+  n_levels: 1
+  reverse: true
+  # transformer:
+  #   layers: 1
+  #   heads: 4
+  #   dropout: 0.1
+  #   max_seq_len: 256
+  #   abspos: false
+
+loss:
+  n_predictions: 2
+  negative_sampling_ext: 64
+  temperature: 0.07
+
+optimizer:
+  learning_rate: 0.0002
+  weight_decay: 0.000001
+
+trainer:
+  epochs: 250
+  output_dir: lightning_logs/synth_cpc
+
+runtime:
+  seed: 137
+  accelerator: auto
+  devices: 1

dl/notebooks/creating_time_series_datasets.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: deep-learning
 #     language: python

dl/notebooks/diffusion_model.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: .venv
 #     language: python

dl/notebooks/diffusion_model_timegrad.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: .venv
 #     language: python

dl/notebooks/diffusion_process.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 # ---
 
 # # Diffusion Process

dl/notebooks/feedforward_neural_netwroks_timeseries.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: deep-learning
 #     language: python

dl/notebooks/hierarchical_forecasting_mint.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: Python 3
 #     language: python

dl/notebooks/lstm_properties.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: .venv
 #     language: python

dl/notebooks/neuralode_timeseries.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: deep-learning
 #     language: python

dl/notebooks/pendulum_dataset.py

@@ -6,7 +6,7 @@
 #       extension: .py
 #       format_name: light
 #       format_version: '1.5'
-#       jupytext_version: 1.15.2
+#       jupytext_version: 1.19.1
 #   kernelspec:
 #     display_name: deep-learning
 #     language: python