DAVID_XP_boston.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hMmdCEWr6kmD"
   },
   "source": [
    "# IMPORT MODULES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 1516,
     "status": "ok",
     "timestamp": 1703936222702,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "DJ1Cd1IKvZLi"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "# from torchmetrics.regression import SpearmanCorrCoef, KendallRankCorrCoef\n",
    "torch.manual_seed(42)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed(42)\n",
    "    torch.cuda.manual_seed_all(42)\n",
    "import copy\n",
    "\n",
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch.utils.data\n",
    "from torch import nn, optim\n",
    "from torchvision import datasets, transforms\n",
    "from torchvision.utils import save_image\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 976,
     "status": "ok",
     "timestamp": 1703936223677,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "nEepsaOvr2p5",
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import warnings\n",
    "# import dill\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler,MinMaxScaler,OneHotEncoder\n",
    "from scipy.stats import gaussian_kde, spearmanr, pearsonr\n",
    "from statsmodels.distributions.empirical_distribution import ECDF\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.decomposition import PCA, KernelPCA\n",
    "import seaborn as sns\n",
    "from matplotlib.pylab import rcParams\n",
    "rcParams['figure.figsize'] = 3, 3\n",
    "sns.set(rc={'figure.figsize':(3,3)})\n",
    "from sklearn.metrics import mean_squared_error,mean_absolute_percentage_error,median_absolute_error, balanced_accuracy_score\n",
    "import os\n",
    "# import chime\n",
    "# chime.theme('mario')\n",
    "np.random.seed(42)\n",
    "from tqdm import trange\n",
    "from IPython.display import Image\n",
    "from scipy.special import kl_div\n",
    "import h2o\n",
    "from h2o.automl import H2OAutoML\n",
    "import prince"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sdv.single_table import TVAESynthesizer, CTGANSynthesizer, CopulaGANSynthesizer\n",
    "from sdv.metadata import SingleTableMetadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# pip install ImbalancedLearningRegression-master.zip\n",
    "from ImbalancedLearningRegression import smote, ro, gn, adasyn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 1849,
     "status": "ok",
     "timestamp": 1703936225522,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "8i3CLJfcM0DZ",
    "outputId": "2e3d1002-2b65-4480-883b-b9ae79d1513b",
    "tags": []
   },
   "outputs": [],
   "source": [
    "rep = 'C:/Users/samgo/OneDrive/Perso/Thèse/Travaux/VAKIR/XP4/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SGxSNRt77zk6"
   },
   "source": [
    "# DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"C:/Users/samgo/OneDrive/Perso/Thèse/Travaux/DataSets-IR/boston.csv\")\n",
    "context=\"supervised\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "lab_Y = \"HousValue\"\n",
    "data.dropna(axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.dropna(axis=0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.histplot(data[lab_Y])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "sns.pairplot(data,diag_kind='kde')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# X_df = pd.read_csv(rep+\"/X_df.csv\")\n",
    "# data = X_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 394
    },
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1703936225523,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "1d0uVZsRu9p3",
    "outputId": "3e7bdf21-d0d3-4ea0-971b-e9484648234f",
    "tags": []
   },
   "outputs": [],
   "source": [
    "kdem = gaussian_kde(np.array(data.T),bw_method=\"scott\")\n",
    "pd.DataFrame(kdem.covariance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1703936225523,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "iXH9DTjAFf4I",
    "tags": []
   },
   "outputs": [],
   "source": [
    "def transformer(inputs, mode_quanti=\"MinMax\", mode_quali=\"OHE\"):\n",
    "    quanti = inputs.select_dtypes(include=['number'])\n",
    "    quali = inputs.select_dtypes(include=['object', 'category'])\n",
    "    name_quanti = quanti.columns\n",
    "    if mode_quanti==\"MinMax\" :\n",
    "      scaler = MinMaxScaler()\n",
    "      quanti_t = pd.DataFrame(scaler.fit_transform(quanti))\n",
    "    else:\n",
    "      scaler = StandardScaler()\n",
    "      quanti_t = pd.DataFrame(scaler.fit_transform(quanti))\n",
    "    quanti_t.columns = name_quanti\n",
    "    if quali.shape[1]==0:\n",
    "        return quanti_t\n",
    "    else:\n",
    "        quali_t = pd.get_dummies(quali)\n",
    "        if mode_quali==\"AFDM\" :\n",
    "          quali_t=quali_t/np.sqrt(quali_t.mean())\n",
    "        return pd.concat([quanti_t, quali_t], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def inv_transformer(inputs, outputs, mode_quanti=\"MinMax\", mode_quali=\"OHE\"):\n",
    "  quanti = inputs.select_dtypes(include=['number'])\n",
    "  quali = inputs.select_dtypes(include=['object', 'category'])\n",
    "  name_quanti = quanti.columns\n",
    "  if quanti.shape[1] > 0 :\n",
    "    if mode_quanti==\"MinMax\" :\n",
    "      scaler = MinMaxScaler()\n",
    "      scaler.fit(quanti)\n",
    "      res_quanti =  pd.DataFrame(scaler.inverse_transform(outputs[name_quanti]))\n",
    "    elif mode_quanti==\"SC\" :\n",
    "      scaler = StandardScaler()\n",
    "      scaler.fit(quanti)\n",
    "      res_quanti =  pd.DataFrame(scaler.inverse_transform(outputs[name_quanti]))\n",
    "    else:\n",
    "      res_quanti =  pd.DataFrame(outputs[name_quanti])\n",
    "    res_quanti.columns = name_quanti\n",
    "    res_quanti.index = outputs.index\n",
    "  if quali.shape[1] > 0 :\n",
    "    quali_t = pd.get_dummies(quali)\n",
    "    name_quali_t = quali_t.columns\n",
    "    res_quali = outputs.copy()\n",
    "    res_quali.drop(name_quanti, axis=1, inplace=True)\n",
    "    res_quali.index = quali_t.index\n",
    "    res_quali = np.round(res_quali,0)\n",
    "    if quanti.shape[1] > 0 :\n",
    "      return pd.concat([res_quanti, res_quali], axis=1)\n",
    "    else:\n",
    "      return res_quali\n",
    "  else:\n",
    "    return res_quanti"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "colY = np.where(data.columns==lab_Y)[0][0] ; colY"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pondération IR "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def IR_weighting(Y, plot=False, alpha=1/2):\n",
    "    if alpha is None:\n",
    "        w = Y*0+1\n",
    "    else:\n",
    "        w= 1/gaussian_kde(Y).evaluate(Y)**(alpha)\n",
    "    w=w/sum(w)\n",
    "    if plot==True:\n",
    "        sns.set(rc={'figure.figsize':(3,3)})\n",
    "        sns.histplot(Y)\n",
    "        plt.show()\n",
    "        sns.scatterplot(x=Y,y=w)\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "alfa=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "w_Y=IR_weighting(data[lab_Y], plot=True, alpha=alfa)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_size=0.4\n",
    "train_size=0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def trainTest(data, test_size=0.3,np_seed=None,w=None, train_size=None):\n",
    "    if np_seed is None:\n",
    "        np.random.seed()\n",
    "    else:\n",
    "        np.random.seed(np_seed)\n",
    "    n = data.shape[0]\n",
    "    if w is None:w=np.repeat(1,n)/n\n",
    "    n_test=round(test_size*n)\n",
    "    id_test = np.random.choice(n, size=n_test, p=w , replace=False)\n",
    "    X_test = data.iloc[id_test,:]\n",
    "    X_train = data.drop(index=id_test)\n",
    "    if train_size is not None:X_train=X_train.sample(round(train_size*n))\n",
    "    \n",
    "    return{'X_train':X_train,'X_test':X_test}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "seed_sample = np.random.randint(1000);seed_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "tags": []
   },
   "source": [
    "Tirage uniforme"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "split = trainTest(data,test_size,seed_sample,w=w_Y,train_size=train_size)\n",
    "X_train = split['X_train']\n",
    "X_test = split['X_test']\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "train_df = pd.DataFrame(X_train[[lab_Y]])\n",
    "train_df['sample']='train'\n",
    "test_df = pd.DataFrame(X_test[[lab_Y]])\n",
    "test_df['sample']='test'\n",
    "sns.histplot(data=pd.concat([train_df,test_df],axis=0),x=lab_Y,hue='sample',kde=True, stat=\"density\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(X_train[[lab_Y]].describe())\n",
    "sns.histplot(X_train[[lab_Y]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(X_test[[lab_Y]].describe())\n",
    "sns.histplot(X_test[[lab_Y]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "Xy_train=X_train.copy()\n",
    "if context==\"supervised\":\n",
    "    y_train =  X_train[[lab_Y]]\n",
    "    X_train.drop(data.columns[colY], axis=1, inplace=True)\n",
    "    w_Y=IR_weighting(y_train[lab_Y], alpha=alfa)\n",
    "    sns.scatterplot(x=y_train[lab_Y],y=w_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "executionInfo": {
     "elapsed": 73307,
     "status": "ok",
     "timestamp": 1703936299076,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "4ICe5zkJsMDD",
    "outputId": "561cf313-4324-4dce-bd71-5bfb62499373",
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "sns.pairplot(data,diag_kind='kde')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 724
    },
    "executionInfo": {
     "elapsed": 51,
     "status": "ok",
     "timestamp": 1703936299077,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "BbAqrRfWJMh4",
    "outputId": "cb0a5fbc-9502-404a-8d73-f9100d88e506",
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "sns.heatmap(np.round(data.corr(method=\"pearson\"),1),annot=True, cmap=\"Reds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 724
    },
    "executionInfo": {
     "elapsed": 32,
     "status": "ok",
     "timestamp": 1703936299077,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "6Of5nIFvIB4e",
    "outputId": "858496bd-8c82-47aa-c13b-d8d832a61aef",
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "sns.heatmap(np.round(data.corr(method=\"spearman\"),1),annot=True, cmap=\"RdYlGn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 28,
     "status": "ok",
     "timestamp": 1703936299077,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "3CnqP-jyNLe6"
   },
   "outputs": [],
   "source": [
    "# X_df.to_csv(rep+'X_df.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ubVrZsfn3tIt"
   },
   "source": [
    "# PARAMETRES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 28,
     "status": "ok",
     "timestamp": 1703936299077,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "jlIWYmCzt2Cq"
   },
   "outputs": [],
   "source": [
    "if torch.cuda.is_available():\n",
    "    device = torch.device(\"cuda\")\n",
    "else:\n",
    "    device = torch.device(\"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 24,
     "status": "ok",
     "timestamp": 1703936299078,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "07aM0S7n3xf4"
   },
   "outputs": [],
   "source": [
    "encDim_S = 5\n",
    "encDim_M = 10\n",
    "encDim_L = 15\n",
    "\n",
    "hp_fae_w_corr = None\n",
    "hp_fae_sparsity_factor = None\n",
    "hp_fae_sparsity_target=0.05\n",
    "hp_fae_sparsity_mode=None\n",
    "hp_fae_wNorm=False\n",
    "hp_fae_mirror=False\n",
    "hp_fae_encDim=1\n",
    "hp_fae_epochs=1000\n",
    "hp_fae_batch_size=64\n",
    "hp_fae_lr=10e-4\n",
    "#, BN_thr=0.05\n",
    "hp_fae_lossfunc=\"wMSE\"\n",
    "hp_fae_dimHL=5\n",
    "hp_fae_w_Y = None\n",
    "hp_dimLat = 10 \n",
    "hp_betaKLD=1e-6 \n",
    "hp_betaX=1 \n",
    "hp_betaY=10\n",
    "\n",
    "hp_power=1\n",
    "hp_penalRank=1\n",
    "\n",
    "plotModel=False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "JKGnlzREKuum"
   },
   "source": [
    "# ---------------------------- INDUSTRIALISATION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Definition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VanillaAEy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AEy(nn.Module):\n",
    "    def __init__(self, p,dimHL=hp_fae_dimHL, power = hp_power,penal_rank=hp_penalRank, dimLat=hp_dimLat,connect=True, betaKLD=None):\n",
    "        super().__init__()\n",
    "        self.p = p\n",
    "        self.dimLat = dimLat\n",
    "        q = int(p/10)+1\n",
    "        self.encoders = nn.Sequential(\n",
    "                    nn.Linear(p, 2*p),\n",
    "                    nn.Tanh(),\n",
    "                    nn.Linear(2*p, p-1*q),\n",
    "                    nn.Tanh(),\n",
    "                    nn.Linear(p-1*q, p-2*q),\n",
    "                    nn.Tanh(),\n",
    "                    nn.Linear(p-2*q, p-3*q))\n",
    "\n",
    "        self.decoders = nn.Sequential(\n",
    "                    nn.Linear(p-3*q, p-2*q),\n",
    "                    nn.Tanh(),\n",
    "                    nn.Linear(p-2*q, p-1*q),\n",
    "                    nn.Tanh(),\n",
    "                    nn.Linear(p-1*q, 2*p),\n",
    "                    nn.Tanh(),\n",
    "                    nn.Linear(2*p, p))\n",
    "        \n",
    "\n",
    "    def encode(self, x) :\n",
    "        encoded=self.encoders(x)\n",
    "        return encoded\n",
    "    \n",
    "    def decode(self, z):\n",
    "        decoded = self.decoders(z)\n",
    "        return decoded[:,:-1],decoded[:,[-1]]\n",
    "    \n",
    "    \n",
    "    def forward(self, x,y):\n",
    "        encoded = self.encode(torch.cat((x,y),dim=1))  \n",
    "        decoded_x, decode_y = self.decode(encoded)\n",
    "        return decoded_x, decode_y, encoded, encoded*0, encoded*0 \n",
    "\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VanillaVAEy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class VAEy(nn.Module):\n",
    "    def __init__(self, p,dimHL=hp_fae_dimHL,sparsity_factor=None, power = hp_power,penal_rank=hp_penalRank, dimLat=hp_dimLat,connect=True, betaKLD=None):\n",
    "        super().__init__()\n",
    "        self.p = p\n",
    "        self.dimLat = p-3\n",
    "        self.betaKLD = betaKLD\n",
    "        q = int(p/10)+1\n",
    "        self.fc1 = nn.Linear(p+1, 2*p+1)\n",
    "        self.fc2 = nn.Linear(2*p+1, p-1*q) \n",
    "        self.fc3 = nn.Linear(p-1*q, p-2*q) \n",
    "        self.fc41 = nn.Linear(p-2*q, p-3*q)\n",
    "        self.fc42 = nn.Linear(p-2*q,p-3*q) \n",
    "\n",
    "        self.fc5 = nn.Linear(p-3*q, p-2*q)\n",
    "        self.fc6 = nn.Linear(p-2*q, p-1*q) \n",
    "        self.fc7 = nn.Linear(p-1*q, 2*p+1) \n",
    "        self.fc81 = nn.Linear(2*p+1, p)\n",
    "        self.fc82 = nn.Linear(2*p+1, 1) \n",
    "               \n",
    "    def encode(self, x, a) :\n",
    "        h1 = F.tanh(self.fc1(torch.cat([x.view(-1, x.shape[1]),a.view(-1, 1)],1)))  # on utilise une simple ReLu pour la premiere couche de 400 neurones et ID pour la deuxieme couche pour u et std\n",
    "        h2 = F.tanh(self.fc2(h1))\n",
    "        h3 = F.tanh(self.fc3(h2))\n",
    "        mu = self.fc41(h3)\n",
    "        logvar = self.fc42(h3)\n",
    "        return mu, logvar\n",
    "              \n",
    "\n",
    "    def reparameterize(self, mu, logvar):\n",
    "        std = torch.exp(0.5*logvar)  # racine de la variance ==> std\n",
    "        eps = torch.randn_like(std)   # une loi normal entre 0 et 1 avec une taille de std soit de 20 !!\n",
    "        return mu + eps*std  # z = u + sigm * eps\n",
    "  \n",
    "    def decode(self, z):\n",
    "        h5 = F.tanh(self.fc5(z))\n",
    "        h6 = F.tanh(self.fc6(h5))\n",
    "        h7 = F.tanh(self.fc7(h6))\n",
    "        decoded_x = self.fc81(h7)\n",
    "        decoded_y = self.fc82(h7)\n",
    "        return decoded_x, decoded_y\n",
    "    \n",
    "    \n",
    "    def forward(self, x, a):\n",
    "        mu,logvar = self.encode(x.view(-1, x.shape[1]), a) \n",
    "        encoded = self.reparameterize(mu, logvar)\n",
    "        decoded_x, decode_y = self.decode(encoded)\n",
    "        return decoded_x, decode_y, encoded, mu, logvar \n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def balanced_mse_loss(recon_y, y, w):\n",
    "    # print(\"torch.nn.functional.mse_loss(recon_y, y, reduction='sum')\",torch.nn.functional.mse_loss(recon_y, y, reduction='sum'))\n",
    "    # print(\"torch.sum(w*(recon_y-y).T**2\",torch.sum(w*(recon_y-y).T**2))\n",
    "    return torch.sum(w*(recon_y-y).T**2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def loss_function_VAE(recon_x, z, recon_y, y, x, mu, logvar, epoch,i, betaX = 1,betaY = 0,  betaKLD = 0.5, w_Y=None):\n",
    "    BCE_X = torch.nn.functional.mse_loss(recon_x, x, reduction='sum')\n",
    "    if w_Y is None:\n",
    "        BCE_Y = torch.nn.functional.mse_loss(recon_y, y, reduction='sum')\n",
    "    else:\n",
    "        BCE_Y = balanced_mse_loss(recon_y, y, w=w_Y)\n",
    "    KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) # ici c'est la KL pour une loi normal 0 & 1 uniquement\n",
    "    return betaX*BCE_X + betaKLD*KLD + betaY*BCE_Y , betaY*BCE_Y , betaX*BCE_X  , betaKLD*KLD  #+ mmd  # ici on additionne tout simplement les deux pour la loss function avec BCE pour retrouver X et KLD pour ne pas s'éloigner d'une loi normale\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "a0xo5OFDs5QJ"
   },
   "source": [
    "## Model Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AE-X & VAE-X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_VAE(X_train,y_train, seed=seed_sample, lr=hp_fae_lr, batch_size=hp_fae_batch_size, epochs=hp_fae_epochs, dimHL=hp_fae_dimHL,dimLat=hp_dimLat,stacked=True, \n",
    "              power=hp_power,penal_rank=hp_penalRank, betaX = hp_betaX,betaY = 0,  betaKLD=hp_betaKLD, connect=False,type_model=\"VAE\"):\n",
    "        \n",
    "        print(\"betaX\",betaX)\n",
    "        print(\"betaKLD\",betaKLD)\n",
    "        X_train = transformer(X_train)\n",
    "        X_train = np.array(X_train)\n",
    "        y_train = transformer(y_train)\n",
    "        y_train = np.array(y_train)\n",
    "        \n",
    "        inputsX=torch.FloatTensor(X_train).to(device)\n",
    "        inputsY=torch.FloatTensor(y_train).to(device)\n",
    "        p = X_train.shape[1]\n",
    "        if type_model==\"VAE\":\n",
    "            print(\"type model : VAE\")\n",
    "            model = VAE(p,dimHL=dimHL, power = power,penal_rank=penal_rank, dimLat=dimLat, connect=connect).to(device)\n",
    "        elif type_model==\"AE\":\n",
    "            print(\"type model : AE\")\n",
    "            model = AE(p,dimHL=dimHL, power = power,penal_rank=penal_rank, dimLat=dimLat, connect=connect).to(device)\n",
    "\n",
    "        \n",
    "        optimizer = torch.optim.AdamW(model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.0001)\n",
    "        batch_idx=0\n",
    "        # log_interval=50\n",
    "        batch_no = len(X_train) // batch_size\n",
    "\n",
    "        decod=[]\n",
    "        losses=[]\n",
    "        losses_VAE=[]\n",
    "        losses_X=[]\n",
    "        losses_Y=[]\n",
    "        losses_KL=[]\n",
    "        \n",
    "        for epoch in trange(epochs, desc=\"Proving P=NP\", unit=\"carrots\"):\n",
    "          x_train,  ytrain = shuffle(X_train, np.expand_dims(y_train,axis = 1))\n",
    "            # Mini batch learning\n",
    "          for i in range(batch_no):\n",
    "            start = i * batch_size\n",
    "            end = start + batch_size\n",
    "            x_var = Variable(torch.FloatTensor(x_train[start:end])).to(device)\n",
    "            y_var = Variable(torch.FloatTensor(ytrain[start:end])).to(device)\n",
    "\n",
    "            # model.train()\n",
    "            train_loss = 0\n",
    "            optimizer.zero_grad()\n",
    "            decoded, encoded, mu, logvar = model(x_var)\n",
    "            loss_VAE, loss_Y, loss_X, loss_KL = loss_function_VAE(decoded, encoded,y_var, y_var, x_var, mu, logvar,epoch,i, betaX = betaX,betaY = betaY,  betaKLD = betaKLD)\n",
    "            if type_model==\"AE\":\n",
    "                loss = loss_X\n",
    "            else:\n",
    "                loss = loss_VAE\n",
    "            loss.backward()\n",
    "            train_loss += loss.item()\n",
    "            losses.append(loss.cpu().detach().numpy()+0)\n",
    "            losses_VAE.append(loss_VAE.cpu().detach().numpy()+0)\n",
    "            losses_X.append(loss_X.cpu().detach().numpy()+0)\n",
    "            # losses_Y.append(loss_Y.cpu().detach().numpy()+0)\n",
    "            losses_KL.append(loss_KL.cpu().detach().numpy()+0)\n",
    "            optimizer.step()\n",
    "          decoded, encoded, mu, logvar = model(inputsX)\n",
    "          if type_model==\"AE\":\n",
    "                mu=encoded\n",
    "        return {'model':model, \"decoded\":decoded, \"encoded\":encoded, \"mu\":mu, \"logvar\":logvar,\n",
    "                'losses':losses, 'losses_VAE':losses_VAE, 'losses_X':losses_X, 'losses_KL':losses_KL}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AE-Xy & VAE-Xy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_VAEy(X_train,y_train, seed=seed_sample, lr=hp_fae_lr, batch_size=hp_fae_batch_size, epochs=hp_fae_epochs, dimHL=hp_fae_dimHL,dimLat=hp_dimLat,stacked=True, \n",
    "              power=hp_power,penal_rank=hp_penalRank, betaX = hp_betaX,betaY = hp_betaY,  betaKLD=hp_betaKLD, connect=False,type_model=\"VAE\", w_Y=None):\n",
    "        \n",
    "        torch.manual_seed(seed)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        if torch.cuda.is_available():\n",
    "            torch.cuda.manual_seed(seed)\n",
    "            torch.cuda.manual_seed_all(seed)\n",
    "        \n",
    "        print(\"betaX\",betaX)\n",
    "        print(\"betaKLD\",betaKLD)\n",
    "        print(\"betaY\",betaY)\n",
    "        X_train = transformer(X_train)\n",
    "        y_train = transformer(y_train)\n",
    "        inputsX=torch.FloatTensor(X_train.to_numpy()).to(device)\n",
    "        inputsY=torch.FloatTensor(y_train.to_numpy()).to(device)\n",
    "        if w_Y is not None:\n",
    "            w_Y=torch.FloatTensor(w_Y)*y_train.shape[0]\n",
    "        else:\n",
    "            w_Y=torch.FloatTensor(np.repeat(1,y_train.shape[0]))\n",
    "        p = X_train.shape[1]\n",
    "        if type_model==\"VAE\":\n",
    "            print(\"type model : VAEy\")\n",
    "            model = VAEy(p,dimHL=dimHL, power = power,penal_rank=penal_rank, dimLat=dimLat, connect=connect).to(device)\n",
    "        elif type_model==\"AE\":\n",
    "            print(\"type model : AEy\")\n",
    "            model = AEy(p+1,dimHL=dimHL, power = power,penal_rank=penal_rank, dimLat=dimLat, connect=connect).to(device)\n",
    "            \n",
    "        optimizer = torch.optim.AdamW(model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=0.0001)\n",
    "        batch_idx=0\n",
    "        # log_interval=50\n",
    "        batch_no = len(X_train) // batch_size\n",
    "\n",
    "        decod=[]\n",
    "        losses=[]\n",
    "        losses_VAE=[]\n",
    "        losses_X=[]\n",
    "        losses_Y=[]\n",
    "        losses_KL=[]\n",
    "        \n",
    "        for epoch in trange(epochs, desc=\"Proving P=NP\", unit=\"carrots\"):\n",
    "          # x_train,  ytrain = shuffle(X_train, np.expand_dims(y_train,axis = 1))\n",
    "          xtrain, ytrain, wtrain = shuffle(inputsX, inputsY, w_Y)\n",
    "          # Mini batch learning\n",
    "          for i in range(batch_no):\n",
    "            start = i * batch_size\n",
    "            end = start + batch_size\n",
    "            x_var = Variable(xtrain[start:end]).to(device)\n",
    "            y_var = Variable(ytrain[start:end]).to(device)\n",
    "            w_var = Variable(wtrain[start:end]).to(device)\n",
    "            # plt.scatter(y_var.cpu().detach().numpy(),w_var.cpu().detach().numpy())\n",
    "            # plt.show()\n",
    "\n",
    "            # model.train()\n",
    "            train_loss = 0\n",
    "            optimizer.zero_grad()\n",
    "            decoded_x, decoded_y, encoded, mu, logvar = model(x_var, y_var)\n",
    "            loss_VAE, loss_Y, loss_X, loss_KL = loss_function_VAE(decoded_x, encoded, decoded_y, y_var, x_var, mu, logvar,epoch,i, betaX = betaX,betaY = betaY,  \n",
    "                                                                  betaKLD = betaKLD, w_Y=w_var)\n",
    "            loss = loss_VAE\n",
    "            loss.backward()\n",
    "            train_loss += loss.item()\n",
    "            losses.append(loss.cpu().detach().numpy()+0)\n",
    "            losses_VAE.append(loss_VAE.cpu().detach().numpy()+0)\n",
    "            losses_X.append(loss_X.cpu().detach().numpy()+0)\n",
    "            losses_Y.append(loss_Y.cpu().detach().numpy()+0)\n",
    "            losses_KL.append(loss_KL.cpu().detach().numpy()+0)\n",
    "            optimizer.step()\n",
    "          decoded_x, decoded_y, encoded, mu, logvar = model(inputsX, inputsY)\n",
    "        if type_model==\"AE\":\n",
    "                mu=encoded\n",
    "        return {'model':model, \"decoded_x\":decoded_x, \"decoded_y\":decoded_y, \"encoded\":encoded, \"mu\":mu, \"logvar\":logvar,\n",
    "                'losses':losses, 'losses_VAE':losses_VAE, 'losses_X':losses_X, 'losses_KL':losses_KL, 'losses_Y':losses_Y}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def KL_data(r,s):\n",
    "    P = gaussian_kde(r).evaluate(s)\n",
    "    Q = gaussian_kde(s).evaluate(s)\n",
    "    return kl_div(P,Q).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def KL_dataset(Synth):\n",
    "    res = 0\n",
    "    for col in Synth.columns:\n",
    "        if col != 'Type':\n",
    "            res = res + KL_data(Synth[Synth.Type=='Real'][col],Synth[Synth.Type=='Recons'][col])\n",
    "    return res\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "db4RDjjGADUA"
   },
   "source": [
    "------------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ySVp_767AUx2"
   },
   "source": [
    "------------------------------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Automatisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def PCA_NL(X_train, y_train=None,betaKLD=hp_betaKLD, stacked=False, epochs=hp_fae_epochs,seed=np.random.randint(1000), connect=False, type_model=\"VAE\"):\n",
    "    if y_train is None: np.repaat(1, train.shape[O])\n",
    "    # Linéarisation\n",
    "    res_LSFVAE = train_VAE(X_train=X_train,y_train=y_train, seed=seed_sample, epochs=epochs, stacked=stacked, betaKLD=betaKLD, connect=connect, type_model=type_model)\n",
    "    \n",
    "    # Sorties\n",
    "    decoded = res_LSFVAE['decoded']\n",
    "    encoded = pd.DataFrame(res_LSFVAE['encoded'].cpu().detach().numpy())\n",
    "    \n",
    "    return {'res_LSFVAE':res_LSFVAE, 'encoded':encoded}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def PCA_NL_Analyze(res, X_train, plot_Model=False, MC_ref=None):\n",
    "    sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "    train=X_train.copy()\n",
    "    decoded = pd.DataFrame(res['res_LSFVAE']['decoded'].cpu().detach().numpy())\n",
    "    encoded = res['encoded']\n",
    "    decoded.columns=X_train.columns\n",
    "    RMSE_X = np.sqrt(np.mean((decoded-transformer(train))**2))\n",
    "    decoded = inv_transformer(X_train, decoded)\n",
    "    decoded['Type'] = 'Recons'\n",
    "    train['Type'] = 'Real'\n",
    "    reconstruct  = pd.concat([train,decoded])\n",
    "    KL_R = 0#KL_dataset(reconstruct)\n",
    "    if plot_Model:\n",
    "        losses=res['res_LSFVAE']['losses']\n",
    "        losses_VAE=res['res_LSFVAE']['losses_VAE']\n",
    "        losses_X=res['res_LSFVAE']['losses_X']\n",
    "        losses_KL=res['res_LSFVAE']['losses_KL']\n",
    "        plt.clf()\n",
    "        m=int(len(losses)/50)\n",
    "        sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "        sns.lineplot(pd.DataFrame({\"losses\":losses,\"losses_VAE\":losses_VAE,\"losses_X\":losses_X,\"losses_KL\":losses_KL}))\n",
    "        plt.suptitle(\"Losses Analysis\", y=1.02)\n",
    "        plt.show()\n",
    "        sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "        sns.lineplot(pd.DataFrame({\"losses\":losses,\"losses_VAE\":losses_VAE,\"losses_X\":losses_X,\"losses_KL\":losses_KL}).iloc[m:,:])\n",
    "        plt.suptitle(\"Losses Analysis-Focus\", y=1.02)\n",
    "        plt.show()\n",
    "        print(\"losses_min:\",np.min(losses))\n",
    "        print(\"RMSE_X:\",RMSE_X)\n",
    "        print(\"MSE_X:\",RMSE_X**2)\n",
    "    if  plot_Model:\n",
    "        sns.pairplot(encoded,diag_kind='kde')\n",
    "        plt.show()\n",
    "    # if plot_Model:\n",
    "    #     sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "    #     sns.heatmap(np.round(pd.DataFrame(encoded.corr(method=\"spearman\")),1),annot=True)\n",
    "    #     sns.heatmap(np.round(pd.DataFrame(encoded.corr(method=\"pearson\")),1),annot=True)\n",
    "    #     plt.show()\n",
    "    # if MC_ref is not None:\n",
    "    #     MC_dif = (np.abs(MC_ref - np.abs(pd.DataFrame(encoded.corr(method=\"spearman\")))).sum().sum())/2\n",
    "    #     MC_filtre = MC_ref - np.eye(MC_ref.shape[1])\n",
    "    #     MC_corr=np.abs(np.abs(pd.DataFrame(encoded.corr(method=\"spearman\"))) * MC_filtre - MC_filtre).sum().sum()/2 \n",
    "    if plot_Model:  \n",
    "        sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "        sns.pairplot(decoded, diag_kind='kde')\n",
    "        sns.pairplot(reconstruct, diag_kind='kde', hue='Type')\n",
    "        \n",
    "    if MC_ref is not None:\n",
    "        return {'RMSE_X':RMSE_X,'MSE_X':RMSE_X**2, 'MC_dif':MC_dif,'MC_corr':MC_corr,\"KL_R\":KL_R}\n",
    "    else:\n",
    "        return {'RMSE_X':RMSE_X, 'MSE_X':RMSE_X**2, \"KL_R\":KL_R}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def PCA_NL_Analyze_y(res, X_train, y_train, plot_Model=False, MC_ref=None, w_Y=None):\n",
    "    sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "    train=X_train.copy()\n",
    "    ytrain = pd.DataFrame(y_train.copy())\n",
    "    ytrain.columns = [lab_Y]\n",
    "    decoded_x = pd.DataFrame(res['decoded_x'].cpu().detach().numpy())\n",
    "    decoded_y = pd.DataFrame(res['decoded_y'].cpu().detach().numpy())\n",
    "    encoded = pd.DataFrame(res['encoded'].cpu().detach())\n",
    "    decoded_x.columns=X_train.columns\n",
    "    decoded_y.columns=ytrain.columns\n",
    "    RMSE_X = np.sqrt(np.mean((decoded_x-transformer(train))**2))\n",
    "    RMSE_y = np.sqrt(np.mean((decoded_y-transformer(ytrain))**2))\n",
    "    wRMSE_y = np.sqrt(np.mean(((decoded_y-transformer(y_train))**2)[lab_Y]*w_Y))\n",
    "    decoded_x = inv_transformer(X_train, decoded_x)\n",
    "    decoded_x['Type'] = 'Recons'\n",
    "    train['Type'] = 'Real'\n",
    "    reconstruct_x  = pd.concat([train,decoded_x])\n",
    "    \n",
    "    decoded_y = inv_transformer(ytrain, decoded_y)\n",
    "    decoded_y['Type'] = 'Recons'\n",
    "    ytrain['Type'] = 'Real'\n",
    "    reconstruct_y  = pd.concat([ytrain,decoded_y])\n",
    "    \n",
    "    KL_R = 0#KL_dataset(reconstruct)\n",
    "    if plot_Model:\n",
    "        losses=res['losses']\n",
    "        losses_VAE=res['losses_VAE']\n",
    "        losses_X=res['losses_X']\n",
    "        losses_Y=res['losses_Y']\n",
    "        losses_KL=res['losses_KL']\n",
    "        plt.clf()\n",
    "        m=int(len(losses)/50)\n",
    "        sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "        sns.lineplot(pd.DataFrame({\"losses\":losses,\"losses_VAE\":losses_VAE,\"losses_X\":losses_X,\"losses_Y\":losses_Y,\"losses_KL\":losses_KL}))\n",
    "        plt.suptitle(\"Losses Analysis\", y=1.02)\n",
    "        plt.show()\n",
    "        sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "        sns.lineplot(pd.DataFrame({\"losses\":losses,\"losses_VAE\":losses_VAE,\"losses_X\":losses_X,\"losses_Y\":losses_Y,\"losses_KL\":losses_KL}).iloc[m:,:])\n",
    "        plt.suptitle(\"Losses Analysis-Focus\", y=1.02)\n",
    "        plt.show()\n",
    "        print(\"losses_min:\",np.min(losses))\n",
    "        print(\"RMSE_X:\",RMSE_X)\n",
    "        print(\"RMSE_y\",RMSE_y)\n",
    "        print('wRMSE_y',wRMSE_y)\n",
    "        print(\"MSE_X:\",RMSE_X**2)\n",
    "    if  plot_Model:\n",
    "        sns.pairplot(encoded,diag_kind='kde')\n",
    "        plt.show()\n",
    "    # if plot_Model:\n",
    "    #     sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "    #     sns.heatmap(np.round(pd.DataFrame(encoded.corr(method=\"spearman\")),1),annot=True)\n",
    "    #     sns.heatmap(np.round(pd.DataFrame(encoded.corr(method=\"pearson\")),1),annot=True)\n",
    "    #     plt.show()\n",
    "    # if MC_ref is not None:\n",
    "    #     MC_dif = (np.abs(MC_ref - np.abs(pd.DataFrame(encoded.corr(method=\"spearman\")))).sum().sum())/2\n",
    "    #     MC_filtre = MC_ref - np.eye(MC_ref.shape[1])\n",
    "    #     MC_corr=np.abs(np.abs(pd.DataFrame(encoded.corr(method=\"spearman\"))) * MC_filtre - MC_filtre).sum().sum()/2 \n",
    "    if plot_Model:  \n",
    "        sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "        sns.pairplot(decoded_x, diag_kind='kde')\n",
    "        sns.pairplot(reconstruct_x, diag_kind='kde', hue='Type')\n",
    "        sns.pairplot(decoded_y, diag_kind='kde')\n",
    "        sns.pairplot(reconstruct_y, diag_kind='kde', hue='Type')\n",
    "        \n",
    "    if MC_ref is not None:\n",
    "        return {'RMSE_X':RMSE_X,'RMSE_y':RMSE_y,'wRMSE_y':wRMSE_y,'MSE_X':RMSE_X**2, 'MC_dif':MC_dif,'MC_corr':MC_corr,\"KL_R\":KL_R}\n",
    "    else:\n",
    "        return {'RMSE_X':RMSE_X,'RMSE_y':RMSE_y,'wRMSE_y':wRMSE_y, 'MSE_X':RMSE_X**2, \"KL_R\":KL_R}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def generationXy(res, X, y, w=None, N=None, seed=None, mode=\"VAE\", hmult=0.1):\n",
    "    n = X.shape[0]\n",
    "    if N is None:N=n\n",
    "    if w is None:w=np.repeat(1,n)/n\n",
    "    if seed is None: seed = np.random.choice(n, size=N, replace=True)\n",
    "    inputsX0=X.copy()\n",
    "    inputsy0=y.copy()\n",
    "    if res is not None:\n",
    "        inputsX=transformer(X)\n",
    "        inputsy=transformer(y)\n",
    "        if mode == \"VAE\":\n",
    "            inputsX = np.array(inputsX)[seed,:]\n",
    "            inputsy = np.array(inputsy)[seed,:]\n",
    "            decoded_x, decoded_y, encoded, mu, logvar = res['model'](torch.FloatTensor(inputsX).to(device),torch.FloatTensor(inputsy).to(device))\n",
    "            outputs_x=pd.DataFrame(decoded_x.cpu().detach())\n",
    "            outputs_y=pd.DataFrame(decoded_y.cpu().detach())\n",
    "            outputs_x.columns=X.columns\n",
    "            outputs_y.columns=y.columns\n",
    "        elif mode in [\"kVAE\",\"kAE\"]:\n",
    "            mu=res['mu']\n",
    "            kde = gaussian_kde(mu.cpu().detach().numpy().T, bw_method = \"silverman\")\n",
    "            H = kde.factor**2 * kde.covariance * hmult\n",
    "            sim = np.random.multivariate_normal(mean=np.repeat(0,mu.shape[1]),cov=H,size=N)\n",
    "            encoded = mu[seed,:] + torch.FloatTensor(sim).to(device)\n",
    "            decoded_x, decoded_y = res['model'].decode(encoded)\n",
    "            outputs_x=pd.DataFrame(decoded_x.cpu().detach())\n",
    "            outputs_y=pd.DataFrame(decoded_y.cpu().detach())\n",
    "            outputs_x.columns=X.columns\n",
    "            outputs_y.columns=y.columns\n",
    "        outputs_x = inv_transformer(X,outputs_x)\n",
    "        outputs_y = inv_transformer(y,outputs_y)\n",
    "    else:\n",
    "        if mode == \"kPCA\":\n",
    "            Xytrain=pd.concat([y,X],axis=1)\n",
    "            pca = PCA(n_components=Xytrain.shape[1])\n",
    "            scaler = StandardScaler()\n",
    "            Xytrain_sc = scaler.fit_transform(Xytrain)\n",
    "            Xyfact = pca.fit_transform(Xytrain_sc)\n",
    "            inputs = Xyfact[graine,:]\n",
    "            kde = gaussian_kde(Xyfact.T, bw_method = \"silverman\")\n",
    "            H = kde.factor**2 * kde.covariance * hmult\n",
    "            sim = np.random.multivariate_normal(mean=np.repeat(0,inputs.shape[1]),cov=H,size=N)\n",
    "            synth = pd.DataFrame(inputs + sim)\n",
    "            synth = pca.inverse_transform(synth)\n",
    "            synth = pd.DataFrame(scaler.inverse_transform(synth))\n",
    "            synth.columns=Xytrain.columns\n",
    "            synth.set_index(Xytrain.iloc[graine,:].index,inplace=True)\n",
    "            outputs_x = synth[X.columns]\n",
    "            outputs_y = synth[y.columns]\n",
    "        elif mode == \"kKPCA\":\n",
    "            Xytrain=pd.concat([y,X],axis=1)\n",
    "            pca = KernelPCA(n_components=Xytrain.shape[1], kernel='linear', fit_inverse_transform=True)\n",
    "            scaler = StandardScaler()\n",
    "            Xytrain_sc = scaler.fit_transform(Xytrain)\n",
    "            Xyfact = pca.fit_transform(Xytrain_sc)\n",
    "            inputs = Xyfact[graine,:]\n",
    "            kde = gaussian_kde(Xyfact.T, bw_method = \"silverman\")\n",
    "            H = kde.factor**2 * kde.covariance * hmult\n",
    "            sim = np.random.multivariate_normal(mean=np.repeat(0,inputs.shape[1]),cov=H,size=N)\n",
    "            synth = pd.DataFrame(inputs + sim)\n",
    "            synth = pca.inverse_transform(synth)\n",
    "            synth = pd.DataFrame(scaler.inverse_transform(synth))\n",
    "            synth.columns=Xytrain.columns\n",
    "            synth.set_index(Xytrain.iloc[graine,:].index,inplace=True)\n",
    "            outputs_x = synth[X.columns]\n",
    "            outputs_y = synth[y.columns]\n",
    "        else:\n",
    "            inputs0=pd.concat([inputsX0,inputsy0],axis=1).to_numpy()\n",
    "            inputs=pd.concat([inputsX0.iloc[seed,:],inputsy0.iloc[seed,:]],axis=1).to_numpy()\n",
    "            kde = gaussian_kde(inputs0.T, bw_method = \"silverman\")\n",
    "            H = kde.factor**2 * kde.covariance * hmult\n",
    "            sim = np.random.multivariate_normal(mean=np.repeat(0,inputs.shape[1]),cov=H,size=N)\n",
    "            outputs = pd.DataFrame(inputs + sim)\n",
    "            outputs_x = outputs.iloc[:,:-1]\n",
    "            outputs_y = pd.DataFrame(outputs.iloc[:,-1])\n",
    "            outputs_x.columns=X.columns\n",
    "            outputs_y.columns=y.columns\n",
    "    outputs_x.index=X.iloc[graine,:].index\n",
    "    outputs_y.index=y.iloc[graine,:].index\n",
    "    # outputs_x['seed']=seed\n",
    "    return outputs_x,outputs_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hIqekueKwKYu",
    "tags": []
   },
   "source": [
    "# ILLUSTRATION"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "executionInfo": {
     "elapsed": 19,
     "status": "ok",
     "timestamp": 1703936299079,
     "user": {
      "displayName": "Samuel Actunéo",
      "userId": "04483671778344655050"
     },
     "user_tz": -60
    },
    "id": "bd2xjdrDEmE4",
    "tags": []
   },
   "outputs": [],
   "source": [
    "epoch_simu = 2000\n",
    "plotModel=False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Vanilla Xy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-wE60206EmE3",
    "tags": []
   },
   "source": [
    "##### AE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_PCA_NL = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, betaKLD=0, epochs=epoch_simu, type_model=\"AE\")\n",
    "res_AE = res_PCA_NL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = PCA_NL_Analyze_y(res_PCA_NL, X_train, y_train, plot_Model=plotModel,w_Y=w_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-wE60206EmE3",
    "tags": []
   },
   "source": [
    "##### $0-$VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_PCA_NL = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, epochs=epoch_simu, betaKLD=0,type_model=\"VAE\")\n",
    "res_0VAE = res_PCA_NL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res = PCA_NL_Analyze_y(res_PCA_NL, X_train,y_train, plot_Model=plotModel,w_Y=w_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(pd.DataFrame(res_PCA_NL['mu'].cpu().detach().numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-wE60206EmE3"
   },
   "source": [
    "##### VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# res_PCA_NL = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, epochs=epoch_simu, betaKLD=1,type_model=\"VAE\", betaY=1)\n",
    "# res_VAE=res_PCA_NL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# res = PCA_NL_Analyze_y(res_PCA_NL, X_train,y_train, plot_Model=plotModel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# if plotModel:sns.pairplot(pd.DataFrame(res_PCA_NL['mu'].cpu().detach().numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-wE60206EmE3"
   },
   "source": [
    "##### $\\beta-$VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_PCA_NL = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, epochs=epoch_simu,type_model=\"VAE\")\n",
    "res_BVAE = res_PCA_NL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_BVAE['encoded'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res = PCA_NL_Analyze_y(res_PCA_NL, X_train,y_train, plot_Model=plotModel,w_Y=w_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(pd.DataFrame(res_PCA_NL['mu'].cpu().detach().numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-wE60206EmE3"
   },
   "source": [
    "##### $\\beta-$VAE - wMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_PCA_NL = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, epochs=epoch_simu,type_model=\"VAE\",w_Y=w_Y)\n",
    "res_BVAEw = res_PCA_NL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res = PCA_NL_Analyze_y(res_PCA_NL, X_train,y_train, plot_Model=plotModel,w_Y=w_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(pd.DataFrame(res_PCA_NL['mu'].cpu().detach().numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "j=0\n",
    "for i in range(X_train.shape[1]):\n",
    "    sns.scatterplot(x=X_train.iloc[:,i],y=res_PCA_NL['mu'].cpu().detach()[:,j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generation-Xy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plotModel=False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "N = X_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "graine = np.random.choice(X_train.shape[0], size=N, p=w_Y , replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "hmult=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_grain = y_train.iloc[graine,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(5,5)})\n",
    "sns.histplot(y_grain, kde=True)\n",
    "plt.show()\n",
    "train_df = pd.DataFrame(y_train)\n",
    "train_df['sample']='train'\n",
    "tir_df = pd.DataFrame(y_grain)\n",
    "tir_df['sample']='Tirage'\n",
    "sns.histplot(data=pd.concat([train_df,tir_df],axis=0),x=lab_Y,hue='sample',kde=True, stat=\"density\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(None,X_train, y_train, seed=graine, mode=\"kVAE\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_Train=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.abs((synth_Train-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(res_AE,X_train, y_train, seed=graine, mode=\"kAE\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_kAE=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.abs((synth_kAE-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0-VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(res_0VAE,X_train, y_train, seed=graine, mode=\"VAE\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_0VAE=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.abs((synth_0VAE-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# synth_X, synth_y = generationXy(res_VAE,X_train, y_train, seed=graine, mode=\"VAE\",N=N, hmult=hmult)\n",
    "# synth = pd.concat([synth_y,synth_X]],axis=1)\n",
    "# synth_VAE=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# if plotModel:sns.pairplot(synth)\n",
    "# sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### $\\beta$-VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(res_BVAE,X_train, y_train, seed=graine, mode=\"VAE\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_BVAE=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.abs((synth_BVAE-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### k$\\beta$-VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(res_BVAE,X_train, y_train, seed=graine, mode=\"kVAE\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_kBVAE=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.abs((synth_kBVAE-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### k$\\beta$-VAE-wMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(res_BVAEw,X_train, y_train, seed=graine, mode=\"kVAE\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_kBVAEw=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.abs((synth_kBVAEw-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(None,X_train, y_train, seed=graine, mode=\"kPCA\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_kPCA=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.abs((synth_kPCA-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KernelPCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "Xytrain=pd.concat([y_train,X_train],axis=1)\n",
    "pca = KernelPCA(n_components=Xytrain.shape[1], kernel='linear', fit_inverse_transform=True)\n",
    "scaler = StandardScaler()\n",
    "Xytrain_sc = scaler.fit_transform(Xytrain)\n",
    "Xyfact = pca.fit_transform(Xytrain_sc)\n",
    "inputs = Xyfact[graine,:]\n",
    "kde = gaussian_kde(Xyfact.T, bw_method = \"silverman\")\n",
    "H = kde.factor**2 * kde.covariance * hmult\n",
    "sim = np.random.multivariate_normal(mean=np.repeat(0,inputs.shape[1]),cov=H,size=N)\n",
    "synth = pd.DataFrame(inputs + sim)\n",
    "synth = pca.inverse_transform(synth)\n",
    "synth = pd.DataFrame(scaler.inverse_transform(synth))\n",
    "synth.columns=Xytrain.columns\n",
    "synth.set_index(Xytrain.iloc[graine,:].index,inplace=True)\n",
    "outputs_x = synth[X_train.columns]\n",
    "outputs_y = synth[y_train.columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "synth_X, synth_y = generationXy(None,X_train, y_train, seed=graine, mode=\"kKPCA\",N=N, hmult=hmult)\n",
    "synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "synth_kKPCA=synth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.abs((synth_kKPCA-Xy_train.iloc[graine,:])).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "if plotModel:sns.pairplot(synth)\n",
    "sns.histplot(synth[lab_Y])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prediction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "h2o.init()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "test_data = h2o.H2OFrame(X_test)\n",
    "X_col = X_test.columns\n",
    "y_col = X_test.columns[colY]\n",
    "X_col = X_col.drop(y_col).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def TirMix(train, synth, graine):\n",
    "    train_graine = train.iloc[graine,:]\n",
    "    doub=(train_graine.duplicated()*1).to_numpy()\n",
    "    synth_mix = synth.copy()\n",
    "    for i in range(synth_mix.shape[0]):\n",
    "        if doub[i]==0:\n",
    "            synth_mix.iloc[i,:]=train_graine.iloc[i,:]\n",
    "    return np.round(synth_mix,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def pred(train, test_data, Mix=False,Xy_train=None,graine=None):\n",
    "    if Mix:\n",
    "        print(\"mode TirMix\")\n",
    "        train_hat = TirMix(Xy_train, train, graine)\n",
    "    else:\n",
    "        train_hat=train.copy()\n",
    "    train_hat = np.round(train_hat,10)\n",
    "    w_test = IR_weighting(X_test[lab_Y])\n",
    "    train_hat = h2o.H2OFrame(train_hat)\n",
    "    aml = H2OAutoML(max_models=10, seed=42)\n",
    "    print(\"train_hat.shape\",train_hat.shape )\n",
    "    aml.train(x=X_col, y=y_col, training_frame=train_hat)\n",
    "    res_aml = aml.leaderboard\n",
    "    preds = aml.predict(test_data)\n",
    "    mse = mean_squared_error(preds.as_data_frame(),test_data[y_col].as_data_frame())\n",
    "    wmse=(w_test*(preds.as_data_frame().to_numpy()-test_data[y_col].as_data_frame().to_numpy())**2).mean()\n",
    "    mape = mean_absolute_percentage_error(preds.as_data_frame(),test_data[y_col].as_data_frame())\n",
    "    mae = median_absolute_error(preds.as_data_frame(),test_data[y_col].as_data_frame())\n",
    "    # egm = np.prod(np.abs(preds.as_data_frame().to_numpy()-test_data[y_col].as_data_frame().to_numpy()))**(1/test_data.shape[0])\n",
    "    return {\"res_aml\" : res_aml, \"preds\" : preds, \"mse\" : mse, 'mape':mape, 'mae':mae, 'wmse':wmse}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mix=True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Mix=True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Baseline: Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_baseline = pred(Xy_train, test_data, Mix=False)\n",
    "print(resPred_baseline['mse'])\n",
    "print(resPred_baseline['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Oversampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_baseline = pred(Xy_train.iloc[graine,:], test_data, Mix, Xy_train, graine)\n",
    "print(resPred_baseline['mse'])\n",
    "print(resPred_baseline['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### SB_Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "(synth_Train-Xy_train.iloc[graine,:]).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_Train = pred(synth_Train, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_Train['mse'])\n",
    "print(resPred_Train['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "(synth_kPCA-Xy_train.iloc[graine,:]).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_kPCA = pred(synth_kPCA, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_kPCA['mse'])\n",
    "print(resPred_kPCA['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### KernelPCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_kKPCA = pred(synth_kKPCA, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_kKPCA['mse'])\n",
    "print(resPred_kKPCA['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### AE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_kAE = pred(synth_kAE, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_kAE['mse'])\n",
    "print(resPred_kAE['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 0VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_0VAE = pred(synth_0VAE, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_0VAE['mse'])\n",
    "print(resPred_0VAE['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BVAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_BVAE = pred(synth_BVAE, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_BVAE['mse'])\n",
    "print(resPred_BVAE['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### kBVAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_kBVAE = pred(synth_kBVAE, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_kBVAE['mse'])\n",
    "print(resPred_kBVAE['wmse'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### kBVAE-wMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "resPred_kBVAEw = pred(synth_kBVAEw, test_data, Mix, Xy_train, graine)\n",
    "print(resPred_kBVAEw['mse'])\n",
    "print(resPred_kBVAEw['wmse'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Boucle runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_simus=10\n",
    "test_size=0.4\n",
    "train_size=0.6\n",
    "n = X_train.shape[0]\n",
    "N = n\n",
    "Mix=True\n",
    "hmult=0.1\n",
    "alpha=1\n",
    "epoch_simu=2000\n",
    "benchmark = True\n",
    "metadata = SingleTableMetadata()\n",
    "metadata.detect_from_dataframe(Xy_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_MSE_l =[]\n",
    "res_MAPE_l=[]\n",
    "res_MAE_l =[]\n",
    "res_wMSE_l=[]\n",
    "seed_samples=[860, 738, 945, 289, 473, 354, 945, 19, 30, 611] # seeds to get the same results than paper\n",
    "np.random.seed()\n",
    "for simu in range(n_simus):\n",
    "        \n",
    "    print(\"============================     simu : \",simu,\"          ======================\")\n",
    "    \n",
    "    res_MSE  =  []\n",
    "    res_wMSE =  []\n",
    "    res_MAE  =  []\n",
    "    res_MAPE =  []\n",
    "    \n",
    "    # Train-test sampling\n",
    "    w_Y=IR_weighting(data[lab_Y], alpha=alpha)\n",
    "    # seed_sample = np.random.randint(1000)\n",
    "    seed_sample =seed_samples[simu]\n",
    "    np.random.seed(seed_sample)\n",
    "    # seed_samples.append(seed_sample)\n",
    "    split = trainTest(data,test_size,seed_sample,w=w_Y,train_size=train_size)\n",
    "    X_train = split['X_train']\n",
    "    Xy_train = X_train.copy()\n",
    "    X_test = split['X_test']\n",
    "    y_train =  X_train[[lab_Y]]\n",
    "    X_train.drop(data.columns[colY], axis=1, inplace=True)\n",
    "    w_Y=IR_weighting(y_train[lab_Y])\n",
    "    test_data = h2o.H2OFrame(X_test)\n",
    "    X_col = X_test.columns\n",
    "    y_col = X_test.columns[colY]\n",
    "    X_col = X_col.drop(y_col).tolist()\n",
    "    \n",
    "    \n",
    "    # Model training\n",
    "    res_AE = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, betaKLD=0, epochs=epoch_simu, type_model=\"AE\")\n",
    "    res_0VAE = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, epochs=epoch_simu, betaKLD=0,type_model=\"VAE\")\n",
    "    res_BVAE = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, epochs=epoch_simu,type_model=\"VAE\")\n",
    "    res_BVAEw = train_VAEy(X_train=X_train,y_train=y_train, seed=seed_sample, stacked=None, epochs=epoch_simu,type_model=\"VAE\",w_Y=w_Y)\n",
    "    \n",
    "    \n",
    "    # Generation\n",
    "    ## DAVID\n",
    "    graine = np.random.choice(n, size=N, p=w_Y , replace=True)\n",
    "    synth_X, synth_y = generationXy(None,X_train, y_train, seed=graine, mode=\"kVAE\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_Train=synth\n",
    "    synth_X, synth_y = generationXy(res_AE,X_train, y_train, seed=graine, mode=\"kAE\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_kAE=synth\n",
    "    synth_X, synth_y = generationXy(res_0VAE,X_train, y_train, seed=graine, mode=\"VAE\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_0VAE=synth\n",
    "    synth_X, synth_y = generationXy(res_BVAE,X_train, y_train, seed=graine, mode=\"VAE\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_BVAE=synth\n",
    "    synth_X, synth_y = generationXy(res_BVAE,X_train, y_train, seed=graine, mode=\"kVAE\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_kBVAE=synth\n",
    "    synth_X, synth_y = generationXy(res_BVAEw,X_train, y_train, seed=graine, mode=\"VAE\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_BVAEw=synth\n",
    "    synth_X, synth_y = generationXy(res_BVAEw,X_train, y_train, seed=graine, mode=\"kVAE\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_kBVAEw=synth\n",
    "    synth_X, synth_y = generationXy(None,X_train, y_train, seed=graine, mode=\"kPCA\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_kPCA=synth\n",
    "    synth_X, synth_y = generationXy(None,X_train, y_train, seed=graine, mode=\"kKPCA\",N=N, hmult=hmult)\n",
    "    synth = pd.concat([synth_y,synth_X],axis=1)\n",
    "    synth_kKPCA=synth\n",
    "    \n",
    "    # Benchmark\n",
    "    if benchmark:\n",
    "        ## TVAE\n",
    "        warnings.simplefilter(\"ignore\")\n",
    "        synthesizer = TVAESynthesizer(metadata, epochs=epoch_simu)\n",
    "        synthesizer.fit(Xy_train)\n",
    "        synth_TVAE = synthesizer.sample(num_rows=N)\n",
    "        ## CTGAN\n",
    "        synthesizer = CTGANSynthesizer(metadata,epochs=epoch_simu)\n",
    "        synthesizer.fit(Xy_train)\n",
    "        synth_CTGAN = synthesizer.sample(num_rows=N)\n",
    "        ## CopulaGAN\n",
    "        synthesizer = CopulaGANSynthesizer(metadata,epochs=epoch_simu)\n",
    "        synthesizer.fit(Xy_train)\n",
    "        synth_CopGAN = synthesizer.sample(num_rows=N)\n",
    "        ## ILR\n",
    "        synth_ILRro = ro(data = Xy_train.reset_index().drop(columns=\"index\", axis=1), y = lab_Y)#.iloc[:N,:]\n",
    "        # synth_ILRro.set_index(Xy_train.index)\n",
    "        synth_ILRsmote = smote(data = Xy_train.reset_index().drop(columns=\"index\", axis=1), y = lab_Y,k=3)#.iloc[:N,:]\n",
    "        # synth_ILRsmote.set_index(Xy_train.index)\n",
    "        synth_ILRgn = gn(data = Xy_train.reset_index().drop(columns=\"index\", axis=1), y = lab_Y, pert=hmult)#.iloc[:N,:]\n",
    "        # synth_ILRgn.set_index(Xy_train.index)\n",
    "        # try:\n",
    "        #     synth_ILRadasyn = adasyn(data = Xy_train.reset_index().drop(columns=\"index\", axis=1), y = lab_Y)#.iloc[:N,:]\n",
    "        #     # synth_ILRadasyn.set_index(Xy_train.index)\n",
    "        # except Exception:\n",
    "        #     pass\n",
    "    \n",
    "    \n",
    "    # Prediction\n",
    "    resPred_baseline = pred(Xy_train, test_data, False, Xy_train, graine)\n",
    "    resPred_OS = pred(Xy_train, test_data, Mix, Xy_train, graine)\n",
    "    resPred_Train = pred(synth_Train, test_data, Mix, Xy_train, graine)\n",
    "    resPred_kAE = pred(synth_kAE, test_data, Mix, Xy_train, graine)\n",
    "    resPred_0VAE = pred(synth_0VAE, test_data, Mix, Xy_train, graine)\n",
    "    resPred_BVAE = pred(synth_BVAE, test_data, Mix, Xy_train, graine)\n",
    "    resPred_kBVAE = pred(synth_kBVAE, test_data, Mix, Xy_train, graine)\n",
    "    resPred_BVAEw = pred(synth_BVAEw, test_data, Mix, Xy_train, graine)\n",
    "    resPred_kBVAEw = pred(synth_kBVAEw, test_data, Mix, Xy_train, graine)\n",
    "    resPred_kPCA = pred(synth_kPCA, test_data, Mix, Xy_train, graine)\n",
    "    resPred_kKPCA = pred(synth_kKPCA, test_data, Mix, Xy_train, graine)\n",
    "    if benchmark:\n",
    "        resPred_TVAE = pred(synth_TVAE, test_data, Mix, Xy_train, graine)\n",
    "        resPred_CTGAN = pred(synth_CTGAN, test_data, Mix, Xy_train, graine)\n",
    "        resPred_CopGAN = pred(synth_CopGAN, test_data, Mix, Xy_train, graine)\n",
    "        resPred_ILRro = pred(synth_ILRro, test_data, Mix=False, Xy_train=Xy_train, graine=graine)\n",
    "        resPred_ILRsmote = pred(synth_ILRsmote, test_data, Mix=False, Xy_train=Xy_train, graine=graine)\n",
    "        resPred_ILRgn = pred(synth_ILRgn, test_data, Mix=False, Xy_train=Xy_train, graine=graine)\n",
    "        # resPred_ILRadasyn = pred(synth_ILRadasyn, test_data, Mix=False, Xy_train=Xy_train, graine=graine)\n",
    "    warnings.resetwarnings()\n",
    "    # Saving\n",
    "    # if (resPred_kBVAEw['mse'] == min(resPred_kBVAEw['mse'],resPred_baseline['mse'],resPred_OS['mse'],\n",
    "    #                                  resPred_Train['mse'],resPred_kAE['mse'],resPred_0VAE['mse'],resPred_BVAE['mse'],\n",
    "    #                                  resPred_kBVAE['mse'],resPred_kPCA['mse'],resPred_kKPCA['mse'])):#,\n",
    "    #                                  # resPred_TVAE['mse'], resPred_CTGAN['mse'],resPred_CopGAN['mse'],resPred_ILRro['mse'],resPred_ILRsmote['mse'],resPred_ILRgn['mse'])):\n",
    "    #     seed_samples.append(seed_sample)\n",
    "    res_MSE  =  pd.DataFrame({'train':\"Baseline\",'run_'+str(simu):[resPred_baseline['mse']]})\n",
    "    res_MAE  =  pd.DataFrame({'train':\"Baseline\",'run_'+str(simu):[resPred_baseline['mae']]})\n",
    "    res_MAPE =  pd.DataFrame({'train':\"Baseline\",'run_'+str(simu):[resPred_baseline['mape']]})\n",
    "    res_wMSE =  pd.DataFrame({'train':\"Baseline\",'run_'+str(simu):[resPred_baseline['wmse']]})    \n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"OS\",'run_'+str(simu):[resPred_OS['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"OS\",'run_'+str(simu):[resPred_OS['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"OS\",'run_'+str(simu):[resPred_OS['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"OS\",'run_'+str(simu):[resPred_OS['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"kTrain\",'run_'+str(simu):[resPred_Train['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"kTrain\",'run_'+str(simu):[resPred_Train['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"kTrain\",'run_'+str(simu):[resPred_Train['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"kTrain\",'run_'+str(simu):[resPred_Train['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"kAE\",'run_'+str(simu):[resPred_kAE['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"kAE\",'run_'+str(simu):[resPred_kAE['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"kAE\",'run_'+str(simu):[resPred_kAE['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"kAE\",'run_'+str(simu):[resPred_kAE['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"OVAE\",'run_'+str(simu):[resPred_0VAE['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"OVAE\",'run_'+str(simu):[resPred_0VAE['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"OVAE\",'run_'+str(simu):[resPred_0VAE['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"OVAE\",'run_'+str(simu):[resPred_0VAE['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"BVAE\",'run_'+str(simu):[resPred_BVAE['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"BVAE\",'run_'+str(simu):[resPred_BVAE['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"BVAE\",'run_'+str(simu):[resPred_BVAE['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"BVAE\",'run_'+str(simu):[resPred_BVAE['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"kBVAE\",'run_'+str(simu):[resPred_kBVAE['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"kBVAE\",'run_'+str(simu):[resPred_kBVAE['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"kBVAE\",'run_'+str(simu):[resPred_kBVAE['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"kBVAE\",'run_'+str(simu):[resPred_kBVAE['wmse']]})])\n",
    "    \n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"BVAEw\",'run_'+str(simu):[resPred_BVAEw['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"BVAEw\",'run_'+str(simu):[resPred_BVAEw['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"BVAEw\",'run_'+str(simu):[resPred_BVAEw['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"BVAEw\",'run_'+str(simu):[resPred_BVAEw['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"kBVAEw\",'run_'+str(simu):[resPred_kBVAEw['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"kBVAEw\",'run_'+str(simu):[resPred_kBVAEw['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"kBVAEw\",'run_'+str(simu):[resPred_kBVAEw['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"kBVAEw\",'run_'+str(simu):[resPred_kBVAEw['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"kPCA\",'run_'+str(simu):[resPred_kPCA['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"kPCA\",'run_'+str(simu):[resPred_kPCA['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"kPCA\",'run_'+str(simu):[resPred_kPCA['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"kPCA\",'run_'+str(simu):[resPred_kPCA['wmse']]})])\n",
    "\n",
    "    res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"kKPCA\",'run_'+str(simu):[resPred_kKPCA['mse']]})])\n",
    "    res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"kKPCA\",'run_'+str(simu):[resPred_kKPCA['mae']]})])\n",
    "    res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"kKPCA\",'run_'+str(simu):[resPred_kKPCA['mape']]})])\n",
    "    res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"kKPCA\",'run_'+str(simu):[resPred_kKPCA['wmse']]})])\n",
    "\n",
    "    if benchmark:\n",
    "        res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"TVAE\",'run_'+str(simu):[resPred_TVAE['mse']]})])\n",
    "        res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"TVAE\",'run_'+str(simu):[resPred_TVAE['mae']]})])\n",
    "        res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"TVAE\",'run_'+str(simu):[resPred_TVAE['mape']]})])\n",
    "        res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"TVAE\",'run_'+str(simu):[resPred_TVAE['wmse']]})])\n",
    "\n",
    "        res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"CTGAN\",'run_'+str(simu):[resPred_CTGAN['mse']]})])\n",
    "        res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"CTGAN\",'run_'+str(simu):[resPred_CTGAN['mae']]})])\n",
    "        res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"CTGAN\",'run_'+str(simu):[resPred_CTGAN['mape']]})])\n",
    "        res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"CTGAN\",'run_'+str(simu):[resPred_CTGAN['wmse']]})])\n",
    "\n",
    "        res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"CopGAN\",'run_'+str(simu):[resPred_CopGAN['mse']]})])\n",
    "        res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"CopGAN\",'run_'+str(simu):[resPred_CopGAN['mae']]})])\n",
    "        res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"CopGAN\",'run_'+str(simu):[resPred_CopGAN['mape']]})])\n",
    "        res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"CopGAN\",'run_'+str(simu):[resPred_CopGAN['wmse']]})])\n",
    "\n",
    "        res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"ILRro\",'run_'+str(simu):[resPred_ILRro['mse']]})])\n",
    "        res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"ILRro\",'run_'+str(simu):[resPred_ILRro['mae']]})])\n",
    "        res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"ILRro\",'run_'+str(simu):[resPred_ILRro['mape']]})])\n",
    "        res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"ILRro\",'run_'+str(simu):[resPred_ILRro['wmse']]})])\n",
    "\n",
    "        res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"ILRsmote\",'run_'+str(simu):[resPred_ILRsmote['mse']]})])\n",
    "        res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"ILRsmote\",'run_'+str(simu):[resPred_ILRsmote['mae']]})])\n",
    "        res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"ILRsmote\",'run_'+str(simu):[resPred_ILRsmote['mape']]})])\n",
    "        res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"ILRsmote\",'run_'+str(simu):[resPred_ILRsmote['wmse']]})])\n",
    "\n",
    "        res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"ILRgn\",'run_'+str(simu):[resPred_ILRgn['mse']]})])\n",
    "        res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"ILRgn\",'run_'+str(simu):[resPred_ILRgn['mae']]})])\n",
    "        res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"ILRgn\",'run_'+str(simu):[resPred_ILRgn['mape']]})])\n",
    "        res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"ILRgn\",'run_'+str(simu):[resPred_ILRgn['wmse']]})])\n",
    "\n",
    "        # res_MSE  =  pd.concat([res_MSE,pd.DataFrame({'train':\"ILRadasyn\",'run_'+str(simu):[resPred_ILRadasyn['mse']]})])\n",
    "        # res_MAE  =  pd.concat([res_MAE,pd.DataFrame({'train':\"ILRadasyn\",'run_'+str(simu):[resPred_ILRadasyn['mae']]})])\n",
    "        # res_MAPE =  pd.concat([res_MAPE,pd.DataFrame({'train':\"ILRadasyn\",'run_'+str(simu):[resPred_ILRadasyn['mape']]})])\n",
    "        # res_wMSE =  pd.concat([res_wMSE,pd.DataFrame({'train':\"ILRadasyn\",'run_'+str(simu):[resPred_ILRadasyn['wmse']]})])\n",
    "\n",
    "\n",
    "    if len(res_MSE_l)==0:\n",
    "        res_MSE_l  = res_MSE\n",
    "        res_MAE_l  =  res_MAE\n",
    "        res_MAPE_l =  res_MAPE\n",
    "        res_wMSE_l =  res_wMSE\n",
    "    else:\n",
    "        res_MSE_l  = res_MSE_l.merge(res_MSE)\n",
    "        res_MAE_l  = res_MAE_l.merge(res_MAE)\n",
    "        res_MAPE_l = res_MAPE_l.merge(res_MAPE)\n",
    "        res_wMSE_l  = res_wMSE_l.merge(res_wMSE)\n",
    "\n",
    "res_MSE_l.set_index('train', inplace=True)\n",
    "res_MAE_l.set_index('train', inplace=True)\n",
    "res_MAPE_l.set_index('train', inplace=True)\n",
    "res_wMSE_l.set_index('train', inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(5,5)})\n",
    "sns.heatmap(pd.DataFrame(res_MSE_l.mean(axis=1)), annot=True, cmap='RdYlGn_r',fmt='.2f')\n",
    "plt.show()\n",
    "sns.heatmap(pd.DataFrame(res_MSE_l.std(axis=1)), annot=True, cmap='RdYlGn_r',fmt='.2f')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "sns.boxplot(res_MSE_l.transpose(), orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sns.set(rc={'figure.figsize':(11.7,8.27)})\n",
    "sns.boxplot(res_wMSE_l.transpose(), orient='h')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "res_wMSE_l.mean(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "seed_samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_MSE_l.rank(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_MSE_l.to_csv(rep+\"/res_MSE_l.csv\")\n",
    "res_MAE_l.to_csv(rep+\"/res_MAE_l.csv\")\n",
    "res_MAPE_l.to_csv(rep+\"/res_MAPE_l.csv\")\n",
    "res_wMSE_l.to_csv(rep+\"/res_wMSE_l.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(seed_samples).to_csv(rep+\"/seed_samples.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res_MSE_l.rank(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "machine_shape": "hm",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "toc-showcode": false
 },
 "nbformat": 4,
 "nbformat_minor": 4
}